Change all internal APIs to use timezone-aware datetimes (UTC).

Remove reader._types.fix_datetime_tzinfo().

For #321, #325

CHANGES.rst

@@ -13,6 +13,9 @@ Unreleased
 
 * Stop using deprecated :mod:`sqlite3` datetime converters/adapters.
   (:issue:`321`)
+* Change all :doc:`internal APIs <internal>` to use timezone-aware datetimes,
+  with the timezone set to UTC.
+  (:issue:`321`)
 * In the API documentation,
   fall back to type hints if hand-written parameter types are not available.
   Add relevant :ref:`documentation` guidelines to the dev documentation.

docs/dev.rst

@@ -639,6 +639,10 @@ Wrapping underlying storage exceptions
 
 Which exception to wrap, and which not: :issue:`21#issuecomment-365442439`.
 
+In version 3.10 (November 2023), all internal APIs were changed
+to use timezone-aware datetimes, with the timezone set to UTC,
+in preparation for support for any timezone.
+
 
 Timezone handling
 ~~~~~~~~~~~~~~~~~

scripts/bench.py

@@ -14,6 +14,7 @@
 from contextlib import ExitStack
 from datetime import datetime
 from datetime import timedelta
+from datetime import timezone
 from fnmatch import fnmatchcase
 from functools import partial
 
@@ -87,7 +88,7 @@ def make_reader_with_entries(path, num_entries, num_feeds=NUM_FEEDS, text=False)
     parser.tzinfo = None
 
     for i in range(num_feeds):
-        feed = parser.feed(i, datetime(2010, 1, 1))
+        feed = parser.feed(i, datetime(2010, 1, 1, tzinfo=timezone.utc))
         reader.add_feed(feed.url)
 
     random.seed(0)
@@ -98,7 +99,12 @@ def make_reader_with_entries(path, num_entries, num_feeds=NUM_FEEDS, text=False)
                 title=generate_lorem_ipsum(html=False, n=1, min=1, max=10),
                 summary=generate_lorem_ipsum(html=False),
             )
-        parser.entry(i % num_feeds, i, datetime(2010, 1, 1) + timedelta(i), **kwargs)
+        parser.entry(
+            i % num_feeds,
+            i,
+            datetime(2010, 1, 1, tzinfo=timezone.utc) + timedelta(i),
+            **kwargs,
+        )
 
     return reader
 

src/reader/_parser/feedparser.py

@@ -5,6 +5,7 @@
 import time
 import warnings
 from datetime import datetime
+from datetime import timezone
 from typing import Any
 from typing import IO
 from typing import TYPE_CHECKING
@@ -122,7 +123,7 @@ def _get_datetime_attr(thing: Any, key: str) -> datetime | None:
 
 
 def _datetime_from_timetuple(tt: time.struct_time) -> datetime:
-    return datetime.utcfromtimestamp(calendar.timegm(tt))
+    return datetime.fromtimestamp(calendar.timegm(tt), timezone.utc)
 
 
 def _process_entry(feed_url: str, entry: Any, is_rss: bool) -> EntryData:

src/reader/_parser/jsonfeed.py

@@ -189,4 +189,4 @@ def _parse_date(s: str) -> datetime | None:
     except iso8601.ParseError:
         return None
     assert isinstance(dt, datetime)
-    return dt.astimezone(timezone.utc).replace(tzinfo=None)
+    return dt.astimezone(timezone.utc)

src/reader/_plugins/sqlite_releases.py

@@ -24,6 +24,7 @@
 """
 import warnings
 from datetime import datetime
+from datetime import timezone
 from urllib.parse import urlparse
 from urllib.parse import urlunparse
 
@@ -71,7 +72,9 @@ def extract_text(soup):
 def make_entries(feed_url, url, soup):
     for title, fragment, content in extract_text(soup):
         try:
-            updated = datetime.strptime(title.split()[0], '%Y-%m-%d')
+            updated = datetime.strptime(title.split()[0], '%Y-%m-%d').replace(
+                tzinfo=timezone.utc
+            )
         except (ValueError, IndexError):
             continue
 

src/reader/_storage.py

@@ -8,6 +8,7 @@
 from collections.abc import Sequence
 from datetime import datetime
 from datetime import timedelta
+from datetime import timezone
 from functools import partial
 from typing import Any
 from typing import NamedTuple
@@ -1748,11 +1749,13 @@ def entry_update_intent_to_dict(intent: EntryUpdateIntent) -> Mapping[str, Any]:
 
 
 def adapt_datetime(val: datetime) -> str:
-    assert not val.tzinfo, val
+    assert val.tzinfo == timezone.utc, val
+    val = val.replace(tzinfo=None)
     return val.isoformat(" ")
 
 
 def convert_timestamp(val: str) -> datetime:
     rv = datetime.fromisoformat(val)
     assert not rv.tzinfo, val
+    rv = rv.replace(tzinfo=timezone.utc)
     return rv

src/reader/_types.py

@@ -235,7 +235,7 @@ def _getattr_optional_datetime(obj: object, name: str) -> datetime | None:
     value = _getattr_optional(obj, name, datetime)
     if value is None:
         return value
-    return value.astimezone(timezone.utc).replace(tzinfo=None)
+    return value.astimezone(timezone.utc)
 
 
 class ParsedFeed(NamedTuple):
@@ -549,34 +549,6 @@ def make_plugin_name(self, plugin_name: str, key: str | None = None) -> str:
 )
 
 
-_NT = TypeVar('_NT', bound=_namedtuple_compat)
-
-
-def fix_datetime_tzinfo(
-    obj: _NT,
-    *names: str,
-    _old: None | timezone | bool = None,
-    _new: None | timezone = timezone.utc,
-    **kwargs: Any,
-) -> _NT:
-    """For specific optional datetime attributes of an object,
-    and set their tzinfo to `_new`.
-
-    Build and return a new object, using the old ones _replace() method.
-    Pass any other kwargs to _replace().
-
-    If `_old` is not False, assert the old tzinfo is equal to it.
-
-    """
-    for name in names:
-        assert name not in kwargs, (name, list(kwargs))
-        value = getattr(obj, name)
-        if value:
-            assert _old is False or value.tzinfo == _old, value
-            kwargs[name] = value.replace(tzinfo=_new)
-    return obj._replace(**kwargs)
-
-
 UpdateHook = Callable[..., None]
 UpdateHookType = Literal[
     'before_feeds_update',
@@ -661,15 +633,12 @@ class StorageType(Protocol):  # pragma: no cover
       Closing the storage in one thread should not close it in another thread.
 
     All :class:`~datetime.datetime` attributes
-    of all parameters and return values are timezone-naive,
-    with the timezone assumed to be UTC by convention.
+    of all parameters and return values are timezone-aware,
+    with the timezone set to :attr:`~datetime.timezone.utc`.
 
     .. admonition:: Unstable
 
-        As part of :issue:`321`,
-        :class:`~datetime.datetime`\s will be required to be timezone-aware,
-        with the timezone set to :attr:`~datetime.timezone.utc`.
-        At some later point, implementations will be required
+        In the future, implementations will be required
         to accept datetimes with any timezone.
 
     Methods, grouped by topic:

src/reader/core.py

@@ -29,7 +29,6 @@
 from ._types import EntryFilter
 from ._types import EntryUpdateIntent
 from ._types import FeedFilter
-from ._types import fix_datetime_tzinfo
 from ._types import NameScheme
 from ._types import SearchType
 from ._types import StorageType
@@ -656,10 +655,7 @@ def get_feeds(
                 raise ValueError("limit should be a positive integer")
         starting_after = _feed_argument(starting_after) if starting_after else None
 
-        rv = self._storage.get_feeds(filter, sort, limit, starting_after)
-
-        for rv_feed in rv:
-            yield fix_datetime_tzinfo(rv_feed, 'updated', 'added', 'last_updated')
+        return self._storage.get_feeds(filter, sort, limit, starting_after)
 
     @overload
     def get_feed(self, feed: FeedInput, /) -> Feed:  # pragma: no cover
@@ -1085,7 +1081,7 @@ def update_feed(self, feed: FeedInput, /) -> UpdatedFeed | None:
 
     @staticmethod
     def _now() -> datetime:
-        return datetime.utcnow()
+        return datetime.now(timezone.utc)
 
     def get_entries(
         self,
@@ -1194,21 +1190,7 @@ def get_entries(
         if starting_after and sort == 'random':
             raise ValueError("using starting_after with sort='random' not supported")
 
-        rv = self._storage.get_entries(filter, sort, limit, starting_after)
-
-        for rv_entry in rv:
-            yield fix_datetime_tzinfo(
-                rv_entry,
-                'updated',
-                'published',
-                'added',
-                'last_updated',
-                'read_modified',
-                'important_modified',
-                feed=fix_datetime_tzinfo(
-                    rv_entry.feed, 'updated', 'added', 'last_updated'
-                ),
-            )
+        return self._storage.get_entries(filter, sort, limit, starting_after)
 
     @overload
     def get_entry(self, entry: EntryInput, /) -> Entry:  # pragma: no cover
@@ -1339,15 +1321,15 @@ def set_entry_read(
         if read not in (True, False):
             raise ValueError("read should be one of (True, False)")
 
-        modified_naive: datetime | None
+        modified_aware: datetime | None
         if isinstance(modified, MissingType):
-            modified_naive = self._now()
+            modified_aware = self._now()
         elif modified is None:
-            modified_naive = None
+            modified_aware = None
         else:
-            modified_naive = modified.astimezone(timezone.utc).replace(tzinfo=None)
+            modified_aware = modified.astimezone(timezone.utc)
 
-        self._storage.set_entry_read(_entry_argument(entry), read, modified_naive)
+        self._storage.set_entry_read(_entry_argument(entry), read, modified_aware)
 
     def mark_entry_as_read(self, entry: EntryInput, /) -> None:
         """Mark an entry as read.
@@ -1431,16 +1413,16 @@ def set_entry_important(
         if important not in (True, False, None):
             raise ValueError("important should be one of (True, False, None)")
 
-        modified_naive: datetime | None
+        modified_aware: datetime | None
         if isinstance(modified, MissingType):
-            modified_naive = self._now()
+            modified_aware = self._now()
         elif modified is None:
-            modified_naive = None
+            modified_aware = None
         else:
-            modified_naive = modified.astimezone(timezone.utc).replace(tzinfo=None)
+            modified_aware = modified.astimezone(timezone.utc)
 
         self._storage.set_entry_important(
-            _entry_argument(entry), important, modified_naive
+            _entry_argument(entry), important, modified_aware
         )
 
     def mark_entry_as_important(self, entry: EntryInput, /) -> None:

tests/data/full.atom.py

@@ -8,7 +8,7 @@
 
 feed = FeedData(
     url=f'{url_base}full.atom',
-    updated=datetime.datetime(2003, 12, 13, 18, 30, 2),
+    updated=datetime.datetime(2003, 12, 13, 18, 30, 2, tzinfo=datetime.timezone.utc),
     title='Example Feed',
     link='http://example.org/',
     author='John Doe',
@@ -20,11 +20,15 @@
     EntryData(
         feed_url=feed.url,
         id='urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a',
-        updated=datetime.datetime(2003, 12, 13, 18, 30, 2),
+        updated=datetime.datetime(
+            2003, 12, 13, 18, 30, 2, tzinfo=datetime.timezone.utc
+        ),
         title='Atom-Powered Robots Run Amok',
         link='http://example.org/2003/12/13/atom03',
         author='John Doe',
-        published=datetime.datetime(2003, 12, 13, 17, 17, 51),
+        published=datetime.datetime(
+            2003, 12, 13, 17, 17, 51, tzinfo=datetime.timezone.utc
+        ),
         summary='Some text.',
         content=(
             # the text/plain type comes from feedparser
@@ -59,7 +63,7 @@
     EntryData(
         feed_url=feed.url,
         id='urn:uuid:00000000-cfb8-4ebb-aaaa-00000000000',
-        updated=datetime.datetime(2003, 12, 13, 0, 0, 0),
+        updated=datetime.datetime(2003, 12, 13, 0, 0, 0, tzinfo=datetime.timezone.utc),
         title='Atom-Powered Robots Run Amok Again',
         # link comes from feedparser
         link='urn:uuid:00000000-cfb8-4ebb-aaaa-00000000000',

tests/data/full.json.py

@@ -20,11 +20,11 @@
     EntryData(
         feed_url=feed.url,
         id='2',
-        updated=datetime.datetime(2020, 1, 4, 0, 0),
+        updated=datetime.datetime(2020, 1, 4, 0, 0, tzinfo=datetime.timezone.utc),
         title="Title",
         link="https://example.org/second-item",
         author="mailto:[email protected]",
-        published=datetime.datetime(2020, 1, 2, 21, 0),
+        published=datetime.datetime(2020, 1, 2, 21, 0, tzinfo=datetime.timezone.utc),
         summary="A summary",
         content=(
             Content(
@@ -56,7 +56,7 @@
         title=None,
         link='https://example.org/initial-post',
         author='Jane',
-        published=datetime.datetime(2020, 1, 2, 12, 0),
+        published=datetime.datetime(2020, 1, 2, 12, 0, tzinfo=datetime.timezone.utc),
         summary=None,
         content=(
             Content(value='<p>Hello, world!</p>', type='text/html', language='en'),

tests/data/full.rss.py

@@ -8,7 +8,7 @@
 
 feed = FeedData(
     url=f'{url_base}full.rss',
-    updated=datetime.datetime(2010, 9, 6, 0, 1),
+    updated=datetime.datetime(2010, 9, 6, 0, 1, tzinfo=datetime.timezone.utc),
     title='RSS Title',
     link='http://www.example.com/main.html',
     author='Example editor ([email protected])',
@@ -24,7 +24,7 @@
         title='Example entry',
         link='http://www.example.com/blog/post/1',
         author='Example editor',
-        published=datetime.datetime(2009, 9, 6, 16, 20),
+        published=datetime.datetime(2009, 9, 6, 16, 20, tzinfo=datetime.timezone.utc),
         summary='Here is some text containing an interesting description.',
         content=(
             # the text/plain type comes from feedparser
@@ -41,7 +41,7 @@
         feed_url=feed.url,
         id='00000000-1655-4c27-aeee-00000000',
         updated=None,
-        published=datetime.datetime(2009, 9, 6, 0, 0, 0),
+        published=datetime.datetime(2009, 9, 6, 0, 0, 0, tzinfo=datetime.timezone.utc),
         title='Example entry, again',
     ),
 ]

tests/fakeparser.py

@@ -9,7 +9,6 @@
 from reader._parser import RetrieveResult
 from reader._types import EntryData
 from reader._types import FeedData
-from reader._types import fix_datetime_tzinfo
 from reader._types import ParsedFeed
 from reader.types import _entry_argument
 
@@ -84,12 +83,7 @@ def parse(self, url, result):
         else:
             raise RuntimeError(f"unkown feed: {url}")
 
-        feed = fix_datetime_tzinfo(feed, 'updated', _old=self.tzinfo, _new=None)
-
-        entries = [
-            fix_datetime_tzinfo(e, 'updated', 'published', _old=self.tzinfo, _new=None)
-            for e in self.entries[feed_number].values()
-        ]
+        entries = list(self.entries[feed_number].values())
 
         return ParsedFeed(
             feed,