Skip to content

Commit

Permalink
feat(exc): only specific error will be caught
Browse files Browse the repository at this point in the history
  • Loading branch information
@JamzumSum
JamzumSum committed Mar 17, 2023
1 parent c63b9b0 commit f6fce07
Show file tree
Hide file tree
Showing 7 changed files with 215 additions and 101 deletions.
51 changes: 36 additions & 15 deletions doc/source/locale/zh_CN/LC_MESSAGES/api/feed.po
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: aioqzone-feed \n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-03-13 13:11+0800\n"
"POT-Creation-Date: 2023-03-17 22:36+0800\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language: zh_CN\n"
Expand Down Expand Up @@ -110,35 +110,46 @@ msgstr ""
msgid "not logined"
msgstr ""

#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_count:9
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:12
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_count:8
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_second:11 of
msgid "unexcpected error"
msgid "when code != -3000"
msgstr ""

#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_count:9
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:12
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_count:9
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_second:12 of
msgid "when code != 403"
msgstr ""

#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_count:11
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:14
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_count:10
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:13
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_count:10
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_second:13 of
msgid ":py:class:`int`"
msgid "max retry exceeds"
msgstr ""

#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_count:12
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:15
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_count:11
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_second:14 of
msgid "feeds num got actually."
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_count:12
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_second:15 of
msgid ":py:class:`int`"
msgstr ""

#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_count:14
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:17
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:16
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_count:13
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_second:16 of
msgid "feeds num got actually."
msgstr ""

#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_count:15
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:18
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_count:15
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_second:18 of
msgid "You may need :meth:`.new_batch` to generate a new batch id."
msgstr ""

#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_count:17 of
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_count:19 of
msgid "`FeedEvent.StopFeedFetch` works in this method as well."
msgstr ""

Expand All @@ -161,7 +172,7 @@ msgstr ""
msgid "another criterion to judge if the feed is out of range, defaults to None"
msgstr ""

#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_second:20 of
#: aioqzone_feed.api.feed.web.FeedWebApi.get_feeds_by_second:22 of
msgid "removed ``exceed_pred``, use `FeedEvent.StopFeedFetch` instead."
msgstr ""

Expand Down Expand Up @@ -203,9 +214,19 @@ msgstr ""

#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_count:8
#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:11 of
msgid "max error time exceeds"
msgid "when code not in -(3000, -10000)"
msgstr ""

#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_count:13 of
msgid "number of feeds that we got actually."
msgstr ""

#: aioqzone_feed.api.feed.h5.FeedH5Api.get_feeds_by_second:1 of
msgid "Get feeds by abstime (seconds). Range: [`start` - `seconds`, `start`]."
msgstr ""

#~ msgid "unexcpected error"
#~ msgstr ""

#~ msgid "max error time exceeds"
#~ msgstr ""
11 changes: 6 additions & 5 deletions poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "aioqzone-feed"
version = "0.13.2.dev1"
version = "0.13.2.dev2"
description = "aioqzone plugin providing higher level api for processing feed."
authors = ["aioqzone <[email protected]>"]
license = "AGPL-3.0"
Expand All @@ -14,7 +14,7 @@ documentation = "https://aioqzone.github.io/aioqzone-feed"

[tool.poetry.dependencies]
python = "^3.8"
aioqzone = { version = "^0.12.8.dev1", source = "PyPI", allow-prereleases = true }
aioqzone = { version = "^0.12.9.dev3", source = "PyPI", allow-prereleases = true }
exceptiongroup = { version = ">=1.0.0rc8", python = "<3.11" }

# dependency groups
Expand Down
112 changes: 60 additions & 52 deletions src/aioqzone_feed/api/feed/h5.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@

from aioqzone.api import Loginable
from aioqzone.api.h5 import QzoneH5API
from aioqzone.exception import LoginError, QzoneError
from aioqzone.exception import LoginError
from aioqzone.type.resp.h5 import FeedData
from httpx import HTTPStatusError
from aioqzone.utils.catch import HTTPStatusErrorDispatch, QzoneErrorDispatch
from qqqr.event import Emittable, hook_guard
from qqqr.exception import UserBreak
from qqqr.utils.net import ClientAdapter
Expand All @@ -17,14 +17,12 @@
from aioqzone_feed.event import FeedEvent
from aioqzone_feed.type import FeedContent

from .web import exc_stack

log = logging.getLogger(__name__)
login_exc = (LoginError, UserBreak, asyncio.CancelledError)


if sys.version_info < (3, 11):
from exceptiongroup import ExceptionGroup


class FeedH5Api(QzoneH5API, Emittable[FeedEvent]):
"""
.. versionadded:: 0.13.0
Expand Down Expand Up @@ -54,40 +52,46 @@ async def get_feeds_by_count(self, count: int = 10) -> int:
:param count: feeds count to get, max as 10, defaults to 10
:raises `qqqr.exception.UserBreak`: qr login canceled
:raises `aioqzone.exception.LoginError`: not logined
:raises `ExceptionGroup`: max error time exceeds
:raises `SystemExit`: unexcpected error
:raise `qqqr.exception.UserBreak`: qr login canceled
:raise `aioqzone.exception.LoginError`: not logined
:raise `QzoneError`: when code not in -(3000, -10000)
:raise `HTTPStatusError`: when code != 403
:raise `ExceptionGroup`: max retry exceeds
:return: feeds num got actually.
:return: number of feeds that we got actually.
.. note:: You may need :meth:`.new_batch` to generate a new batch id.
"""
exceed_pred = hook_guard(self.hook.StopFeedFetch)
stop_fetching = False
got = 0
aux = ""
exceed_pred = hook_guard(self.hook.StopFeedFetch)
err = []
for page in range(1000):
try:
resp = await self.get_active_feeds(aux)
except (QzoneError, HTTPStatusError) as e:
err.append(e)
log.warning(f"Error when fetching page. Skipped. {e}")
if len(err) >= 5:
raise ExceptionGroup("max error exceeds", err)
await asyncio.sleep(0.5)
continue

aux = resp.attachinfo
for fd in resp.vFeeds[: count - got]:
attach_info = ""
errs = exc_stack()
feeds = []

def error_dispatch(e):
feeds.clear()
errs.append(e)
log.warning(f"error fetching page: {e}")

while not stop_fetching:
with QzoneErrorDispatch() as qse, HTTPStatusErrorDispatch() as hse:
qse.dispatch(-3000, -10000, dispatcher=error_dispatch)
hse.dispatch(403, dispatcher=error_dispatch)
resp = await self.get_active_feeds(attach_info)
attach_info = resp.attachinfo
feeds = resp.vFeeds
stop_fetching = not resp.hasmore

for fd in feeds[: count - got]:
if await exceed_pred(fd):
stop_fetching = True
continue
if (got := got + 1) >= count:
stop_fetching = True
break
self._dispatch_feed(fd)
got += 1
if stop_fetching or got >= count or not resp.hasmore:
break

return got

async def get_feeds_by_second(
Expand All @@ -101,10 +105,11 @@ async def get_feeds_by_second(
:param start: start timestamp, defaults to None, means now.
:param exceed_pred: another criterion to judge if the feed is out of range, defaults to None
:raises `qqqr.exception.UserBreak`: qr login canceled
:raises `aioqzone.exception.LoginError`: not logined
:raises `ExceptionGroup`: max error time exceeds
:raises `SystemExit`: unexcpected error
:raise `qqqr.exception.UserBreak`: qr login canceled
:raise `aioqzone.exception.LoginError`: not logined
:raise `QzoneError`: when code not in -(3000, -10000)
:raise `HTTPStatusError`: when code != 403
:raise `ExceptionGroup`: max retry exceeds
:return: feeds num got actually.
Expand All @@ -114,31 +119,34 @@ async def get_feeds_by_second(
end = start - seconds
stop_fetching = False
got = 0
aux = ""
attach_info = ""
exceed_pred = hook_guard(self.hook.StopFeedFetch)
err = []
for page in range(1000):
try:
resp = await self.get_active_feeds(aux)
except (QzoneError, HTTPStatusError) as e:
err.append(e)
log.warning(f"Error when fetching page. Skipped. {e}")
if len(err) >= 5:
raise ExceptionGroup("max error exceeds", err)
await asyncio.sleep(0.5)
continue

aux = resp.attachinfo
for fd in resp.vFeeds:
errs = exc_stack()
feeds = []

def error_dispatch(e):
feeds.clear()
errs.append(e)
log.warning(f"error fetching page: {e}")

while not stop_fetching:
with QzoneErrorDispatch() as qse, HTTPStatusErrorDispatch() as hse:
qse.dispatch(-3000, -10000, dispatcher=error_dispatch)
hse.dispatch(403, dispatcher=error_dispatch)
resp = await self.get_active_feeds(attach_info)
attach_info = resp.attachinfo
feeds = resp.vFeeds
stop_fetching = not resp.hasmore

for fd in feeds:
if fd.abstime > start:
continue
if fd.abstime < end or await exceed_pred(fd):
stop_fetching = True
continue
self._dispatch_feed(fd)
got += 1
if stop_fetching or not resp.hasmore:
break
self._dispatch_feed(fd)

return got

def drop_rule(self, feed: FeedData) -> bool:
Expand Down
Loading

0 comments on commit f6fce07

Please sign in to comment.