From e252e12c546ca7fbc6263bae4491be4923f1e8e2 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 02:50:59 -0600 Subject: [PATCH 01/14] update .gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 3fafd07..22bd59d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ __pycache__ *.egg-info +build/ +dist/ +*.sublime-* +.coverage + From 0474ac4afbf0bd86a55cf89cd2c8b8863c38a3a6 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 03:00:28 -0600 Subject: [PATCH 02/14] fix: test had an issue with using IntEnum repr for column name --- tests/test_rawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_rawl.py b/tests/test_rawl.py index 5d3df6c..072e292 100644 --- a/tests/test_rawl.py +++ b/tests/test_rawl.py @@ -57,7 +57,7 @@ class TheCols(IntEnum): class TheModel(RawlBase): def __init__(self, dsn): # Generate column list from the Enum - columns = [str(col).split(".")[1] for col in TheCols] + columns = [col.name for col in TheCols] log.debug("columns: %s" % columns) From 97f6f54d51564352a9acaff2bf87bdfbd1291892 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 03:07:02 -0600 Subject: [PATCH 03/14] fix: use isinstance() over type() --- rawl/__init__.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/rawl/__init__.py b/rawl/__init__.py index d925dc1..d9d6e1c 100644 --- a/rawl/__init__.py +++ b/rawl/__init__.py @@ -150,6 +150,9 @@ def put_conn(self, conn): if conn.status in OPEN_TRANSACTION_STATES: conn.rollback() + # Silence mypy. Should always be setup in constructor + assert RawlConnection.pool is not None + RawlConnection.pool.putconn(conn) @@ -185,10 +188,10 @@ def __setstate__(self, state): def __getitem__(self, k): # If it's an int, use the int to lookup a column in the position of the # sequence provided. - if type(k) == int: + if isinstance(k, int): return dict.__getitem__(self._data, self.columns[k]) # If it's a string, it's a dict lookup - elif type(k) == str: + elif isinstance(k, str): return dict.__getitem__(self._data, k) # Anything else and we have no idea how to handle it. else: @@ -202,10 +205,10 @@ def __getitem__(self, k): def __setitem__(self, k, v): # If it's an int, use the int to lookup a column in the position of the # sequence provided. - if type(k) == int: + if isinstance(k, int): return dict.__setitem__(self._data, self.columns[k], v) # If it's a string, it's a dict lookup - elif type(k) == str: + elif isinstance(k, str): return dict.__setitem__(self._data, k, v) # Anything else and we have no idea how to handle it. else: @@ -411,12 +414,12 @@ def process_columns(self, columns): :columns: A sequence of columns for the table. Can be list, comma -delimited string, or IntEnum. """ - if type(columns) == list: + if isinstance(columns, list): self.columns = columns - elif type(columns) == str: + elif isinstance(columns, str): self.columns = [c.strip() for c in columns.split()] - elif type(columns) == IntEnum: - self.columns = [str(c) for c in columns] + elif isinstance(columns, IntEnum): + self.columns = [c.name for c in columns] else: raise RawlException("Unknown format for columns") @@ -528,7 +531,7 @@ def get(self, pk): :returns: List of single result """ - if type(pk) == str: + if isinstance(pk, str): # Probably an int, give it a shot try: pk = int(pk) @@ -604,8 +607,8 @@ class RawlJSONEncoder(JSONEncoder): """ def default(self, o): - if type(o) == datetime: + if isinstance(o, datetime): return o.isoformat() - elif type(o) == RawlResult: + elif isinstance(o, RawlResult): return o.to_dict() return super(RawlJSONEncoder, self).default(o) From 719bcc5bc9128475456201bc9f28137d8152d4e6 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 03:12:28 -0600 Subject: [PATCH 04/14] fix silence mypy --- rawl/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rawl/__init__.py b/rawl/__init__.py index d9d6e1c..ecd6aee 100644 --- a/rawl/__init__.py +++ b/rawl/__init__.py @@ -139,6 +139,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): def get_conn(self): log.debug("Retrieving connection from pool for %s" % self.dsn) + # Silence mypy. Should always be setup in constructor + assert RawlConnection.pool is not None + conn = RawlConnection.pool.getconn() if conn.status not in OPEN_TRANSACTION_STATES: conn.set_session(isolation_level=ISOLATION_LEVEL_READ_COMMITTED) From cedc5a82cf29984dc0d8b1470784aea63f822679 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 15:33:11 -0600 Subject: [PATCH 05/14] feat: type annotations and update to psycopg3 --- .flake8 | 3 + pyproject.toml | 8 ++ rawl/__init__.py | 188 +++++++++++++++++++++++++------------------ rawl/py.typed | 0 requirements.dev.txt | 1 + setup.py | 4 +- 6 files changed, 125 insertions(+), 79 deletions(-) create mode 100644 .flake8 create mode 100644 pyproject.toml create mode 100644 rawl/py.typed diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..c140601 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +exclude = .git,__pycache__,docs,old,build,dist +max-line-length = 100 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3bd55b2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[tool.mypy] +follow_imports = "silent" +warn_redundant_casts = true +warn_unused_ignores = true +disallow_any_generics = true +check_untyped_defs = true +no_implicit_reexport = true +disallow_untyped_defs = true diff --git a/rawl/__init__.py b/rawl/__init__.py index ecd6aee..5198d8a 100644 --- a/rawl/__init__.py +++ b/rawl/__init__.py @@ -56,28 +56,28 @@ def get_name(self, pk): import logging import random import warnings -from enum import IntEnum +from enum import Enum, EnumType, IntEnum from abc import ABC +from collections.abc import KeysView, ValuesView from json import JSONEncoder from datetime import datetime -from psycopg2 import sql -from psycopg2.pool import ThreadedConnectionPool -from psycopg2.extensions import ( - STATUS_IN_TRANSACTION, - STATUS_BEGIN, - STATUS_PREPARED, - ISOLATION_LEVEL_READ_COMMITTED, -) - -OPEN_TRANSACTION_STATES = (STATUS_IN_TRANSACTION, STATUS_BEGIN, STATUS_PREPARED) +from psycopg import Connection, Cursor, IsolationLevel, sql +from psycopg.pq import TransactionStatus +from psycopg_pool import ConnectionPool + +from types import TracebackType +from typing import Any, Dict, Iterator, List, Optional, Type, TypeVar, Union + +OPEN_TRANSACTION_STATES = (TransactionStatus.ACTIVE, TransactionStatus.INTRANS) POOL_MIN_CONN = 1 POOL_MAX_CONN = 25 log = logging.getLogger("rawl") +_IE = TypeVar("_IE", bound=IntEnum) -def pop_or_none(d, k): - """ Pop a value from a dict or return None if not exists """ +def pop_or_none(d: Dict[str, Any], k: str) -> Any: + """Pop a value from a dict or return None if not exists""" try: return d.pop(k) except KeyError: @@ -88,7 +88,7 @@ class RawlException(Exception): pass -class RawlConnection(object): +class RawlConnection: """ Connection handling for rawl @@ -100,10 +100,9 @@ class RawlConnection(object): results = cursor.fetchall() """ - pool = None - - def __init__(self, dsn_string, close_on_exit=True): + pool: Optional[ConnectionPool] = None + def __init__(self, dsn_string: str, close_on_exit: bool = True) -> None: log.debug("Connection init") self.dsn = dsn_string @@ -111,46 +110,53 @@ def __init__(self, dsn_string, close_on_exit=True): # Create the pool if it doesn't exist already if RawlConnection.pool is None: - RawlConnection.pool = ThreadedConnectionPool( - POOL_MIN_CONN, POOL_MAX_CONN, self.dsn + RawlConnection.pool = ConnectionPool( + self.dsn, min_size=POOL_MIN_CONN, max_size=POOL_MAX_CONN ) log.debug("Created connection pool ({})".format(id(RawlConnection.pool))) else: log.debug("Reusing connection pool ({})".format(id(RawlConnection.pool))) - def __enter__(self): + def __enter__(self) -> Connection[Any]: conn = None try: conn = self.get_conn() return conn - except Exception: + except Exception as e: log.exception("Connection failure") + raise e finally: - self.put_conn(conn) - - def __exit__(self, exc_type, exc_val, exc_tb): + if conn is not None: + self.put_conn(conn) + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> bool: if exc_val: self.entrance = False return True - def get_conn(self): + def get_conn(self) -> Connection[Any]: log.debug("Retrieving connection from pool for %s" % self.dsn) # Silence mypy. Should always be setup in constructor assert RawlConnection.pool is not None conn = RawlConnection.pool.getconn() - if conn.status not in OPEN_TRANSACTION_STATES: - conn.set_session(isolation_level=ISOLATION_LEVEL_READ_COMMITTED) + if conn.info.transaction_status not in OPEN_TRANSACTION_STATES: + conn.isolation_level = IsolationLevel.READ_COMMITTED return conn - def put_conn(self, conn): + def put_conn(self, conn: Connection[Any]) -> None: if self.close_on_exit: # Assume rolled back if uncommitted - if conn.status in OPEN_TRANSACTION_STATES: + if conn.info.transaction_status in OPEN_TRANSACTION_STATES: conn.rollback() # Silence mypy. Should always be setup in constructor @@ -159,36 +165,35 @@ def put_conn(self, conn): RawlConnection.pool.putconn(conn) -class RawlResult(object): - """ Represents a row of results retreived from the DB """ +class RawlResult: + """Represents a row of results retreived from the DB""" - def __init__(self, columns, data_dict): + def __init__(self, columns: List[str], data_dict: Dict[str, Any]) -> None: self._data = data_dict self.columns = columns - def __str__(self): + def __str__(self) -> str: return str(self._data) - def __getattribute__(self, name): + def __getattribute__(self, name: str) -> Any: # Try for the local objects actual attributes first try: return object.__getattribute__(self, name) # Then resort to the data dict except AttributeError: - if name in self._data: return self._data[name] else: raise AttributeError("%s is not available" % name) - def __getstate__(self): + def __getstate__(self) -> Dict[str, Any]: return self._data - def __setstate__(self, state): + def __setstate__(self, state: Dict[str, Any]) -> None: self._data = state - def __getitem__(self, k): + def __getitem__(self, k: str) -> Any: # If it's an int, use the int to lookup a column in the position of the # sequence provided. if isinstance(k, int): @@ -205,7 +210,7 @@ def __getitem__(self, k): except IndexError: raise IndexError("Unknown index value %s" % k) - def __setitem__(self, k, v): + def __setitem__(self, k: str, v: Any) -> None: # If it's an int, use the int to lookup a column in the position of the # sequence provided. if isinstance(k, int): @@ -222,37 +227,43 @@ def __setitem__(self, k, v): except IndexError: raise IndexError("Unknown index value %s" % k) - def __len__(self): + def __len__(self) -> int: return len(self._data) - def __iter__(self): + def __iter__(self) -> Iterator[Any]: things = self._data.values() for x in things: yield x - def keys(self): + def keys(self) -> KeysView[str]: return self._data.keys() - def values(self): + def values(self) -> ValuesView[Any]: return self._data.values() - def to_dict(self): + def to_dict(self) -> Dict[str, Any]: return self._data - def to_list(self): + def to_list(self) -> List[Any]: return list(self.values()) class RawlBase(ABC): - """ And abstract class for creating models out of raw SQL queries """ - - def __init__(self, dsn, columns, table_name, pk_name=None): + """And abstract class for creating models out of raw SQL queries""" + + def __init__( + self, + dsn: str, + columns: List[str], + table_name: str, + pk_name: Optional[str] = None, + ) -> None: self.dsn = dsn self.table = table_name - self.columns = [] + self.columns: List[str] = [] self._connection_manager = RawlConnection(dsn) - self._open_transaction = None - self._open_cursor = None + self._open_transaction: Optional[Connection[Any]] = None + self._open_cursor: Optional[Cursor[Any]] = None # Process the provided columns into a list self.process_columns(columns) @@ -264,7 +275,13 @@ def __init__(self, dsn, columns, table_name, pk_name=None): else: self.pk = columns[0] - def _assemble_with_columns(self, sql_str, columns, *args, **kwargs): + def _assemble_with_columns( + self, + sql_str: str, + columns: List[str], + *args: Optional[Any], + **kwargs: Optional[Any] + ) -> sql.Composed: """ Format a select statement with specific columns @@ -275,7 +292,7 @@ def _assemble_with_columns(self, sql_str, columns, *args, **kwargs): """ # Handle any aliased columns we get (e.g. table_alias.column) - qcols = [] + qcols: List[Union[sql.Composed, sql.Identifier]] = [] for col in columns: if "." in col: # Explodeded it @@ -292,7 +309,13 @@ def _assemble_with_columns(self, sql_str, columns, *args, **kwargs): return query_string - def _assemble_select(self, sql_str, columns, *args, **kwargs): + def _assemble_select( + self, + sql_str: str, + columns: List[str], + *args: Optional[Any], + **kwargs: Optional[Any] + ) -> sql.Composed: """Alias for _assemble_with_columns""" warnings.warn( "_assemble_select has been depreciated for _assemble_with_columns. It will be removed in a future version.", @@ -300,7 +323,9 @@ def _assemble_select(self, sql_str, columns, *args, **kwargs): ) return self._assemble_with_columns(sql_str, columns, *args, **kwargs) - def _assemble_simple(self, sql_str, *args, **kwargs): + def _assemble_simple( + self, sql_str: str, *args: Optional[Any], **kwargs: Optional[Any] + ) -> sql.Composed: """ Format a select statement with specific columns @@ -313,7 +338,13 @@ def _assemble_simple(self, sql_str, *args, **kwargs): return query_string - def _execute(self, query, commit=True, working_columns=None, read_only=False): + def _execute( + self, + query: sql.Composed, + commit: bool = True, + working_columns: Optional[List[str]] = None, + read_only: bool = False, + ) -> List[RawlResult]: """ Execute a query with provided parameters @@ -346,7 +377,7 @@ def _execute(self, query, commit=True, working_columns=None, read_only=False): else: curs = conn.cursor() - def _clean_up(): + def _clean_up() -> None: if not self._open_cursor: log.debug("Closing cursor") curs.close() @@ -409,7 +440,7 @@ def _clean_up(): return result - def process_columns(self, columns): + def process_columns(self, columns: Union[List[str], str, Type[_IE]]) -> None: """ Handle provided columns and if necessary, convert columns to a list for internal strage. @@ -421,12 +452,14 @@ def process_columns(self, columns): self.columns = columns elif isinstance(columns, str): self.columns = [c.strip() for c in columns.split()] - elif isinstance(columns, IntEnum): + elif type(columns) == EnumType: # noqa: E721 self.columns = [c.name for c in columns] else: raise RawlException("Unknown format for columns") - def query(self, sql_string, *args, **kwargs): + def query( + self, sql_string: str, *args: Optional[Any], **kwargs: Optional[Any] + ) -> List[RawlResult]: """ Execute a DML query @@ -435,16 +468,19 @@ def query(self, sql_string, *args, **kwargs): :commit: Whether or not to commit the transaction after the query :returns: Psycopg2 result """ - commit = pop_or_none(kwargs, "commit") + commit = bool(kwargs.pop("commit", self._open_transaction is None)) columns = pop_or_none(kwargs, "columns") - if commit is None and self._open_transaction is not None: - commit = False - query = self._assemble_simple(sql_string, *args, **kwargs) return self._execute(query, commit=commit, working_columns=columns) - def select(self, sql_string, cols, *args, **kwargs): + def select( + self, + sql_string: str, + cols: List[str], + *args: Optional[Any], + **kwargs: Optional[Any] + ) -> List[RawlResult]: """ Execute a SELECT statement @@ -454,17 +490,15 @@ def select(self, sql_string, cols, *args, **kwargs): :returns: Psycopg2 result """ - commit = pop_or_none(kwargs, "commit") + commit = bool(kwargs.pop("commit", self._open_transaction is None)) working_columns = pop_or_none(kwargs, "columns") - if commit is None and self._open_transaction is not None: - commit = False - query = self._assemble_with_columns(sql_string, cols, *args, *kwargs) - return self._execute(query, working_columns=working_columns, commit=commit) - def insert_dict(self, value_dict, commit=True): + def insert_dict( + self, value_dict: Dict[str, Any], commit: bool = True + ) -> Optional[Union[RawlResult, Any]]: """ Execute an INSERT statement using a python dict @@ -525,7 +559,7 @@ def insert_dict(self, value_dict, commit=True): else: return None - def get(self, pk): + def get(self, pk: Union[str, int]) -> List[RawlResult]: """ Retreive a single record from the table. Lots of reasons this might be best implemented in the model @@ -547,7 +581,7 @@ def get(self, pk): pk, ) - def all(self): + def all(self) -> List[RawlResult]: """ Retreive all single record from the table. Should be implemented but not required. @@ -556,14 +590,14 @@ def all(self): return self.select("SELECT {0} FROM " + self.table + ";", self.columns) - def start_transaction(self): + def start_transaction(self) -> Connection[Any]: """ Initiate a connection to use as a transaction """ self._open_transaction = self._connection_manager.get_conn() return self._open_transaction - def rollback(self): + def rollback(self) -> None: """ Initiate a connection to use as a transaction """ @@ -581,7 +615,7 @@ def rollback(self): else: log.warning("Cannot rollback, no open transaction") - def commit(self): + def commit(self) -> None: """ Commit an already open transaction """ @@ -609,7 +643,7 @@ class RawlJSONEncoder(JSONEncoder): json.dumps(cls=RawlJSONEncoder) """ - def default(self, o): + def default(self, o: Any) -> Any: if isinstance(o, datetime): return o.isoformat() elif isinstance(o, RawlResult): diff --git a/rawl/py.typed b/rawl/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/requirements.dev.txt b/requirements.dev.txt index d5f5102..f17bb9e 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -5,3 +5,4 @@ mypy>=0.720 pytest>=3.2.3 pytest-logging>=2015.11.4 pytest-dependency>=0.2 +types-psycopg2~=2.9.21.13 diff --git a/setup.py b/setup.py index 7913759..a3a6ffa 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="rawl", - version="0.3.5", + version="0.4.0", description="An ugly raw SQL postgresql db layer", url="https://github.com/mikeshultz/rawl", author="Mike Shultz", @@ -22,6 +22,6 @@ packages=find_packages(exclude=["build", "dist"]), package_data={"": ["README.md"]}, install_requires=[ - "psycopg2>=2.7.3.2", + "psycopg[pool]~=3.1.11", ], ) From 120e9d733575aa8d0a08a4dceb06260f8621e24f Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 16:26:17 -0600 Subject: [PATCH 06/14] fix: some typing was wrong, return of insert_dict was too variable, annotate tests --- rawl/__init__.py | 35 +++++++++++--------- tests/test_rawl.py | 81 ++++++++++++++++++++++++---------------------- 2 files changed, 63 insertions(+), 53 deletions(-) diff --git a/rawl/__init__.py b/rawl/__init__.py index 5198d8a..ad6e8ef 100644 --- a/rawl/__init__.py +++ b/rawl/__init__.py @@ -193,7 +193,7 @@ def __getstate__(self) -> Dict[str, Any]: def __setstate__(self, state: Dict[str, Any]) -> None: self._data = state - def __getitem__(self, k: str) -> Any: + def __getitem__(self, k: Any) -> Any: # If it's an int, use the int to lookup a column in the position of the # sequence provided. if isinstance(k, int): @@ -210,7 +210,7 @@ def __getitem__(self, k: str) -> Any: except IndexError: raise IndexError("Unknown index value %s" % k) - def __setitem__(self, k: str, v: Any) -> None: + def __setitem__(self, k: Any, v: Any) -> None: # If it's an int, use the int to lookup a column in the position of the # sequence provided. if isinstance(k, int): @@ -254,7 +254,7 @@ class RawlBase(ABC): def __init__( self, dsn: str, - columns: List[str], + columns: Union[List[str], Type[_IE]], table_name: str, pk_name: Optional[str] = None, ) -> None: @@ -273,14 +273,19 @@ def __init__( self.pk = pk_name # Otherwise, assume first column else: - self.pk = columns[0] + if type(columns) == EnumType: # noqa: E721 + self.pk = columns(0).name + elif isinstance(columns, list) and len(columns) > 0: + self.pk = columns[0] + else: + raise ValueError(f"Unexpected columns type: {type(columns)}") def _assemble_with_columns( self, sql_str: str, columns: List[str], *args: Optional[Any], - **kwargs: Optional[Any] + **kwargs: Optional[Any], ) -> sql.Composed: """ Format a select statement with specific columns @@ -314,7 +319,7 @@ def _assemble_select( sql_str: str, columns: List[str], *args: Optional[Any], - **kwargs: Optional[Any] + **kwargs: Optional[Any], ) -> sql.Composed: """Alias for _assemble_with_columns""" warnings.warn( @@ -453,7 +458,8 @@ def process_columns(self, columns: Union[List[str], str, Type[_IE]]) -> None: elif isinstance(columns, str): self.columns = [c.strip() for c in columns.split()] elif type(columns) == EnumType: # noqa: E721 - self.columns = [c.name for c in columns] + # trailing _ can be used to avoid conflict with Enum members + self.columns = [c.name.rstrip("_") for c in columns] else: raise RawlException("Unknown format for columns") @@ -479,7 +485,7 @@ def select( sql_string: str, cols: List[str], *args: Optional[Any], - **kwargs: Optional[Any] + **kwargs: Optional[Any], ) -> List[RawlResult]: """ Execute a SELECT statement @@ -496,9 +502,7 @@ def select( query = self._assemble_with_columns(sql_string, cols, *args, *kwargs) return self._execute(query, working_columns=working_columns, commit=commit) - def insert_dict( - self, value_dict: Dict[str, Any], commit: bool = True - ) -> Optional[Union[RawlResult, Any]]: + def insert_dict(self, value_dict: Dict[str, Any], commit: bool = True) -> int: """ Execute an INSERT statement using a python dict @@ -543,7 +547,7 @@ def insert_dict( + """ """, insert_cols, - *value_set + *value_set, ) result = self._execute(query, commit=commit) @@ -553,11 +557,12 @@ def insert_dict( # Return the pk if we can if hasattr(result[0], self.pk): return getattr(result[0], self.pk) - # Otherwise, the full result + # Otherwise, the first col of result else: - return result[0] + # If it's an int + return result[0] if isinstance(result[0], int) else -1 else: - return None + return 0 def get(self, pk: Union[str, int]) -> List[RawlResult]: """ diff --git a/tests/test_rawl.py b/tests/test_rawl.py index 072e292..529373b 100644 --- a/tests/test_rawl.py +++ b/tests/test_rawl.py @@ -4,10 +4,12 @@ import pickle import json from enum import IntEnum -from psycopg2 import connect -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +from psycopg import Connection, connect + from rawl import POOL_MAX_CONN, RawlBase, RawlResult, RawlJSONEncoder +from typing import Any, List, Optional + log = logging.getLogger(__name__) DROP_TEST_DB = "DROP DATABASE IF EXISTS rawl_test;" @@ -28,9 +30,11 @@ @pytest.fixture(scope="module") -def pgdb(): - pgconn = connect(os.environ.get("PG_DSN", "postgresql://localhost:5432/postgres")) - pgconn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) +def pgdb() -> Connection[Any]: + pgconn = connect( + os.environ.get("PG_DSN", "postgresql://localhost:5432/postgres"), + autocommit=True, + ) cur = pgconn.cursor() cur.execute(DROP_TEST_DB) cur.execute(CREATE_TEST_DB) @@ -50,21 +54,22 @@ def pgdb(): class TheCols(IntEnum): rawl_id = 0 stamp = 1 - name = 2 + # name conflicts with a member of Enum + name_ = 2 # Test rawl query class TheModel(RawlBase): - def __init__(self, dsn): + def __init__(self, dsn: str) -> None: # Generate column list from the Enum - columns = [col.name for col in TheCols] + columns = [col.name.rstrip("_") for col in TheCols] log.debug("columns: %s" % columns) # Init the parent super(TheModel, self).__init__(dsn, columns=columns, table_name="rawl") - def select_rawls_with_extra_column(self, rawl_id): + def select_rawls_with_extra_column(self, rawl_id: int) -> Optional[RawlResult]: """Return the rawls from the rawl table but in a way to test more stuff""" # We're adding an arbitrary column in @@ -72,7 +77,7 @@ def select_rawls_with_extra_column(self, rawl_id): cols.append("foo") res = self.select( - "SELECT {0}, TRUE" " FROM rawl" " WHERE rawl_id={1}", + "SELECT {0}, TRUE FROM rawl WHERE rawl_id={1}", self.columns, rawl_id, columns=cols, @@ -83,7 +88,7 @@ def select_rawls_with_extra_column(self, rawl_id): else: return None - def query_rawls_with_asterisk(self, rawl_id): + def query_rawls_with_asterisk(self, rawl_id: int) -> Optional[RawlResult]: """Test out self.query directly using columns""" res = self.query( @@ -95,13 +100,13 @@ def query_rawls_with_asterisk(self, rawl_id): else: return None - def delete_rawl(self, rawl_id): - """ Test a delete """ + def delete_rawl(self, rawl_id: int) -> List[RawlResult]: + """Test a delete""" return self.query("DELETE FROM rawl WHERE rawl_id={0};", rawl_id, commit=True) - def delete_rawl_without_commit(self, rawl_id): - """ Test a delete """ + def delete_rawl_without_commit(self, rawl_id: int) -> List[RawlResult]: + """Test a delete""" self.start_transaction() @@ -116,8 +121,8 @@ def delete_rawl_without_commit(self, rawl_id): class TestRawl(object): @pytest.mark.dependency() - def test_all(self, pgdb): - """ Test out a basic SELECT statement """ + def test_all(self, pgdb: Connection[Any]) -> None: + """Test out a basic SELECT statement""" mod = TheModel(RAWL_DSN) @@ -133,8 +138,8 @@ def test_all(self, pgdb): assert "I am row one." in result[0] @pytest.mark.dependency() - def test_get_single_rawl(self, pgdb): - """ Test a SELECT WHERE """ + def test_get_single_rawl(self, pgdb: Connection[Any]) -> None: + """Test a SELECT WHERE""" RAWL_ID = 2 @@ -144,7 +149,7 @@ def test_get_single_rawl(self, pgdb): assert result is not None assert type(result) == RawlResult - assert result[TheCols.name] == "I am row two." + assert result[TheCols.name_] == "I am row two." assert result.rawl_id == RAWL_ID assert result["rawl_id"] == RAWL_ID assert result[0] == RAWL_ID @@ -152,8 +157,8 @@ def test_get_single_rawl(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_delete_rawl(self, pgdb): - """ Test a DELETE """ + def test_delete_rawl(self, pgdb: Connection[Any]) -> None: + """Test a DELETE""" RAWL_ID = 2 @@ -167,8 +172,8 @@ def test_delete_rawl(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_rollback_without_commit(self, pgdb): - """ Test a DELETE without a commit """ + def test_rollback_without_commit(self, pgdb: Connection[Any]) -> None: + """Test a DELETE without a commit""" RAWL_ID = 3 @@ -183,7 +188,7 @@ def test_rollback_without_commit(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_access_invalid_attribute(self, pgdb): + def test_access_invalid_attribute(self, pgdb: Connection[Any]) -> None: """ Test that an invalid attribute on the result object throws an exception. @@ -205,7 +210,7 @@ def test_access_invalid_attribute(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_access_invalid_index(self, pgdb): + def test_access_invalid_index(self, pgdb: Connection[Any]) -> None: """ Test that an invalid column index(in bytes string form) on the result object throws an exception. @@ -230,7 +235,7 @@ def test_access_invalid_index(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_insert_dict(self, pgdb): + def test_insert_dict(self, pgdb: Connection[Any]) -> None: """ Test that a new rawl entry can be created with insert_dict """ @@ -250,7 +255,7 @@ def test_insert_dict(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_insert_dict_without_commit(self, pgdb): + def test_insert_dict_without_commit(self, pgdb: Connection[Any]) -> None: """ Test that multople insert_dicts can happen in one transaction """ @@ -309,7 +314,7 @@ def test_insert_dict_without_commit(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_insert_dict_with_invalid_column(self, pgdb): + def test_insert_dict_with_invalid_column(self, pgdb: Connection[Any]) -> None: """ Test case that an insert_dict with an invalid column fails """ @@ -325,7 +330,7 @@ def test_insert_dict_with_invalid_column(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_serialization(self, pgdb): + def test_serialization(self, pgdb: Connection[Any]) -> None: """ Test that a RawlResult object can be serialized properly. """ @@ -355,7 +360,7 @@ def test_serialization(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_json_serialization(self, pgdb): + def test_json_serialization(self, pgdb: Connection[Any]) -> None: """ Test that a RawlResult object can be serialized properly. """ @@ -376,7 +381,7 @@ def test_json_serialization(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_select_with_columns(self, pgdb): + def test_select_with_columns(self, pgdb: Connection[Any]) -> None: """ Test a case with a select query with different columns that given for formatting. @@ -396,7 +401,7 @@ def test_select_with_columns(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_query_with_columns(self, pgdb): + def test_query_with_columns(self, pgdb: Connection[Any]) -> None: """ Test a case with a query with an asterisk for columns so result columns must be specified @@ -416,7 +421,7 @@ def test_query_with_columns(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_get_with_string_pk(self, pgdb): + def test_get_with_string_pk(self, pgdb: Connection[Any]) -> None: """ Test case that covers if a string is given as pk to get() """ @@ -432,7 +437,7 @@ def test_get_with_string_pk(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_single_line_call(self, pgdb): + def test_single_line_call(self, pgdb: Connection[Any]) -> None: """ Test single line calls where the model is instantiated and a method is called at the same time. @@ -447,7 +452,7 @@ def test_single_line_call(self, pgdb): @pytest.mark.dependency( depends=["TestRawl::test_all", "TestRawl::test_get_single_rawl"] ) - def test_dict_assignment(self, pgdb): + def test_dict_assignment(self, pgdb: Connection[Any]) -> None: """ This test tries to assign something to RawlResult as if it were a dict """ @@ -468,7 +473,7 @@ def test_dict_assignment(self, pgdb): "TestRawl::test_insert_dict_with_invalid_column", ] ) - def test_many_insert(self, pgdb): + def test_many_insert(self, pgdb: Connection[Any]) -> None: """ Test more inserts than POOL_MAX_CONN. Tests against "connection pool exhausted" errors @@ -487,7 +492,7 @@ def test_many_insert(self, pgdb): "TestRawl::test_insert_dict_with_invalid_column", ] ) - def test_many_insert_in_transaction(self, pgdb): + def test_many_insert_in_transaction(self, pgdb: Connection[Any]) -> None: """ Test more inserts than POOL_MAX_CONN. Tests against "connection pool exhausted" errors From 64705445e113ec7fd41471385a959aa6683a1eb5 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 16:29:03 -0600 Subject: [PATCH 07/14] chore: only test python 3.8-3.11 --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index abdd70a..ba7c84a 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -5,7 +5,7 @@ jobs: name: Test (pytest) strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9"] + python-version: ["3.8", "3.9", "3.10", "3.11"] runs-on: ubuntu-latest container: python:${{ matrix.python-version }} services: From 55971862b2cbd91cdb0c57c5968869f31f8db271 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 16:35:04 -0600 Subject: [PATCH 08/14] fix: EnumType is not available before 3.11 --- rawl/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rawl/__init__.py b/rawl/__init__.py index ad6e8ef..cf8084b 100644 --- a/rawl/__init__.py +++ b/rawl/__init__.py @@ -56,7 +56,9 @@ def get_name(self, pk): import logging import random import warnings -from enum import Enum, EnumType, IntEnum + +# EnumMeta is an alias to EnumType as of 3.11 - to be depreciated +from enum import EnumMeta, IntEnum from abc import ABC from collections.abc import KeysView, ValuesView from json import JSONEncoder @@ -273,7 +275,7 @@ def __init__( self.pk = pk_name # Otherwise, assume first column else: - if type(columns) == EnumType: # noqa: E721 + if type(columns) == EnumMeta: # noqa: E721 self.pk = columns(0).name elif isinstance(columns, list) and len(columns) > 0: self.pk = columns[0] @@ -457,7 +459,7 @@ def process_columns(self, columns: Union[List[str], str, Type[_IE]]) -> None: self.columns = columns elif isinstance(columns, str): self.columns = [c.strip() for c in columns.split()] - elif type(columns) == EnumType: # noqa: E721 + elif type(columns) == EnumMeta: # noqa: E721 # trailing _ can be used to avoid conflict with Enum members self.columns = [c.name.rstrip("_") for c in columns] else: From f7b44392b0e1bcf5f4aa226557bf42bfefa3dc45 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 16:56:34 -0600 Subject: [PATCH 09/14] chore: drop python 3.8 support for collections type generics --- .github/workflows/pytest.yaml | 2 +- setup.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index ba7c84a..9f9bb86 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -5,7 +5,7 @@ jobs: name: Test (pytest) strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] runs-on: ubuntu-latest container: python:${{ matrix.python-version }} services: diff --git a/setup.py b/setup.py index a3a6ffa..0041333 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,9 @@ "Intended Audience :: Developers", "Topic :: Database", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], keywords="postgresql database bindings sql", packages=find_packages(exclude=["build", "dist"]), From d1f1a2f7342040da4d60391a8abd570c663fe912 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 17:01:42 -0600 Subject: [PATCH 10/14] fix: coverage workflow ref to pytest module --- .github/workflows/coverage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index baa33d0..e2479f7 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -44,7 +44,7 @@ jobs: fi - name: Run Coverage - run: coverage run --source rawl -m py.test + run: coverage run --source rawl -m pytest env: PG_DSN: postgresql://rawl:s3cretpass@postgres:5432/postgres RAWL_DSN: postgresql://rawl:s3cretpass@postgres:5432/rawl_test From aa6c89c6778f346304af93d58e12611589209981 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 17:12:51 -0600 Subject: [PATCH 11/14] chore: isort --- rawl/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/rawl/__init__.py b/rawl/__init__.py index cf8084b..325886f 100644 --- a/rawl/__init__.py +++ b/rawl/__init__.py @@ -56,20 +56,19 @@ def get_name(self, pk): import logging import random import warnings - -# EnumMeta is an alias to EnumType as of 3.11 - to be depreciated -from enum import EnumMeta, IntEnum from abc import ABC from collections.abc import KeysView, ValuesView -from json import JSONEncoder from datetime import datetime +# EnumMeta is an alias to EnumType as of 3.11 - to be depreciated +from enum import EnumMeta, IntEnum +from json import JSONEncoder +from types import TracebackType +from typing import Any, Dict, Iterator, List, Optional, Type, TypeVar, Union + from psycopg import Connection, Cursor, IsolationLevel, sql from psycopg.pq import TransactionStatus from psycopg_pool import ConnectionPool -from types import TracebackType -from typing import Any, Dict, Iterator, List, Optional, Type, TypeVar, Union - OPEN_TRANSACTION_STATES = (TransactionStatus.ACTIVE, TransactionStatus.INTRANS) POOL_MIN_CONN = 1 POOL_MAX_CONN = 25 From 3ce74fa818fc8ab7e6b788b7b81145de3953d411 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Sat, 23 Sep 2023 17:21:25 -0600 Subject: [PATCH 12/14] lint: isort and black disagree --- rawl/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rawl/__init__.py b/rawl/__init__.py index 325886f..54e6754 100644 --- a/rawl/__init__.py +++ b/rawl/__init__.py @@ -59,8 +59,10 @@ def get_name(self, pk): from abc import ABC from collections.abc import KeysView, ValuesView from datetime import datetime -# EnumMeta is an alias to EnumType as of 3.11 - to be depreciated -from enum import EnumMeta, IntEnum +from enum import ( + EnumMeta, # EnumMeta is an alias to EnumType as of 3.11 - to be depreciated + IntEnum, +) from json import JSONEncoder from types import TracebackType from typing import Any, Dict, Iterator, List, Optional, Type, TypeVar, Union From 6f8e13173c159860f6b4547ad82dbc41484c4b61 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Tue, 26 Sep 2023 18:24:48 -0600 Subject: [PATCH 13/14] feat: implements RawlBase.count() --- rawl/__init__.py | 8 ++++++++ tests/test_rawl.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/rawl/__init__.py b/rawl/__init__.py index 54e6754..ade2589 100644 --- a/rawl/__init__.py +++ b/rawl/__init__.py @@ -598,6 +598,14 @@ def all(self) -> List[RawlResult]: return self.select("SELECT {0} FROM " + self.table + ";", self.columns) + def count(self) -> int: + """ + Get a count of all records in the table. + :returns: List of results + """ + + return self.query("SELECT COUNT(*) FROM " + self.table + ";")[0][0] + def start_transaction(self) -> Connection[Any]: """ Initiate a connection to use as a transaction diff --git a/tests/test_rawl.py b/tests/test_rawl.py index 529373b..2bd1c65 100644 --- a/tests/test_rawl.py +++ b/tests/test_rawl.py @@ -120,6 +120,14 @@ def delete_rawl_without_commit(self, rawl_id: int) -> List[RawlResult]: class TestRawl(object): + @pytest.mark.dependency() + def test_count(self, pgdb: Connection[Any]) -> None: + """Test out count() statement""" + + mod = TheModel(RAWL_DSN) + + assert mod.count() == 4 + @pytest.mark.dependency() def test_all(self, pgdb: Connection[Any]) -> None: """Test out a basic SELECT statement""" From 35bf09dcac0d4171973f773684906db1fcc61a05 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Tue, 26 Sep 2023 18:30:52 -0600 Subject: [PATCH 14/14] feat: implements RawBase.exists() --- rawl/__init__.py | 14 ++++++++++++++ tests/test_rawl.py | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/rawl/__init__.py b/rawl/__init__.py index ade2589..e9db865 100644 --- a/rawl/__init__.py +++ b/rawl/__init__.py @@ -606,6 +606,20 @@ def count(self) -> int: return self.query("SELECT COUNT(*) FROM " + self.table + ";")[0][0] + def exists(self, pk: int) -> bool: + """ + Check of a record with the given pk exists + :returns: List of results + """ + + return ( + self.query( + "SELECT COUNT(*) FROM " + self.table + " WHERE " + self.pk + " = {0};", + pk, + )[0][0] + > 0 + ) + def start_transaction(self) -> Connection[Any]: """ Initiate a connection to use as a transaction diff --git a/tests/test_rawl.py b/tests/test_rawl.py index 2bd1c65..d255d8b 100644 --- a/tests/test_rawl.py +++ b/tests/test_rawl.py @@ -128,6 +128,17 @@ def test_count(self, pgdb: Connection[Any]) -> None: assert mod.count() == 4 + @pytest.mark.dependency() + def test_exists(self, pgdb: Connection[Any]) -> None: + """Test out count() statement""" + + mod = TheModel(RAWL_DSN) + + assert mod.exists(1) is True + assert mod.exists(2) is True + assert mod.exists(3) is True + assert mod.exists(4) is True + @pytest.mark.dependency() def test_all(self, pgdb: Connection[Any]) -> None: """Test out a basic SELECT statement"""