From 525af7213cc2b8fc97a8ad35112e4e0cedd7e5f2 Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 30 Jan 2024 17:07:52 +1300 Subject: [PATCH] Update pre-commit config Remove Black Update requirements Change reference_period to time_period in variables but not db tables (since schema is not yet changed) --- .config/black.toml | 2 - .config/pre-commit-config.yaml | 19 ++++----- pyproject.toml | 22 ++++------ requirements.txt | 41 +++++++++++-------- src/hapi/pipelines/database/admins.py | 14 +++---- src/hapi/pipelines/database/food_security.py | 22 +++++----- .../pipelines/database/humanitarian_needs.py | 8 ++-- src/hapi/pipelines/database/national_risk.py | 12 +++--- .../database/operational_presence.py | 12 +++--- src/hapi/pipelines/database/org.py | 8 ++-- src/hapi/pipelines/database/population.py | 8 ++-- 11 files changed, 80 insertions(+), 88 deletions(-) delete mode 100644 .config/black.toml diff --git a/.config/black.toml b/.config/black.toml deleted file mode 100644 index a8f43fef..00000000 --- a/.config/black.toml +++ /dev/null @@ -1,2 +0,0 @@ -[tool.black] -line-length = 79 diff --git a/.config/pre-commit-config.yaml b/.config/pre-commit-config.yaml index 9b45c2e4..8aaefe05 100644 --- a/.config/pre-commit-config.yaml +++ b/.config/pre-commit-config.yaml @@ -2,23 +2,22 @@ default_language_version: python: python3.11 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-ast - - repo: https://github.com/psf/black - rev: 23.3.0 - hooks: - - id: black - args: [--config, .config/black.toml] - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.267 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.14 hooks: + # Run the linter. - id: ruff - args: [--config, .config/ruff.toml, --fix, --exit-non-zero-on-fix] + args: [--config, .config/ruff.toml, --fix] + # Run the formatter. + - id: ruff-format + args: [--config, .config/ruff.toml] - repo: https://github.com/jazzband/pip-tools - rev: 6.13.0 + rev: 7.3.0 hooks: - id: pip-compile name: pip-compile requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 346f12e9..bc1fc3cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,10 +34,10 @@ classifiers = [ requires-python = ">=3.8" dependencies = [ - "hapi-schema>= 0.5.0", - "hdx-python-country>= 3.6.3", + "hapi-schema>= 0.6.0", + "hdx-python-country>= 3.6.4", "hdx-python-database[postgresql]>= 1.2.9", - "hdx-python-scraper>= 2.3.2", + "hdx-python-scraper>= 2.3.3", "libhxl", "sqlalchemy" ] @@ -92,22 +92,14 @@ test = """ """ [[tool.hatch.envs.test.matrix]] -python = ["3.11"] +python = ["3.12"] [tool.hatch.envs.lint] detached = true -dependencies = [ - "black", - "ruff", -] +dependencies = ["ruff"] [tool.hatch.envs.lint.scripts] style = [ - "ruff --config .config/ruff.toml {args:.}", - "black --config .config/black.toml --check --diff {args:.}", -] -# Not used for anything at the moment -fmt = [ - "black --config .config/black.toml {args:.}", - "ruff --config .config/ruff.toml --fix {args:.}", + "ruff check --config .config/ruff.toml --diff {args:.}", + "ruff format --config .config/ruff.toml --diff {args:.}", ] diff --git a/requirements.txt b/requirements.txt index 68492d7b..7dd74767 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --all-extras --output-file=requirements.txt --resolver=backtracking pyproject.toml +# pip-compile --all-extras --output-file=requirements.txt pyproject.toml # annotated-types==0.6.0 # via pydantic @@ -29,9 +29,11 @@ click==8.1.7 # via typer colorama==0.4.6 # via typer -coverage[toml]==7.4.0 - # via pytest-cov -cryptography==42.0.0 +coverage[toml]==7.4.1 + # via + # coverage + # pytest-cov +cryptography==42.0.1 # via pyopenssl defopt==6.4.0 # via hdx-python-api @@ -51,9 +53,9 @@ et-xmlfile==1.1.0 # via openpyxl filelock==3.13.1 # via virtualenv -frictionless==5.16.0 +frictionless==5.16.1 # via hdx-python-utilities -google-auth==2.26.2 +google-auth==2.27.0 # via # google-auth-oauthlib # gspread @@ -61,7 +63,7 @@ google-auth-oauthlib==1.2.0 # via gspread greenlet==3.0.3 # via sqlalchemy -gspread==5.12.4 +gspread==6.0.0 # via hdx-python-scraper hapi-schema==0.6.0 # via hapi-pipelines (pyproject.toml) @@ -73,8 +75,10 @@ hdx-python-country==3.6.4 # hdx-python-api # hdx-python-scraper hdx-python-database[postgresql]==1.2.9 - # via hapi-pipelines (pyproject.toml) -hdx-python-scraper==2.3.2 + # via + # hapi-pipelines (pyproject.toml) + # hdx-python-database +hdx-python-scraper==2.3.3 # via hapi-pipelines (pyproject.toml) hdx-python-utilities==3.6.4 # via @@ -139,7 +143,7 @@ petl==1.7.14 # via frictionless platformdirs==4.1.0 # via virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via pytest ply==3.11 # via @@ -163,11 +167,11 @@ pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pydantic==2.5.3 +pydantic==2.6.0 # via # frictionless # inflect -pydantic-core==2.14.6 +pydantic-core==2.16.1 # via pydantic pygments==2.17.2 # via rich @@ -179,7 +183,7 @@ pyphonetics==0.5.3 # via hdx-python-country pyrsistent==0.20.0 # via jsonschema -pytest==7.4.4 +pytest==8.0.0 # via # hapi-pipelines (pyproject.toml) # pytest-cov @@ -192,7 +196,7 @@ python-dateutil==2.8.2 # libhxl python-io-wrapper==0.3.1 # via libhxl -python-slugify==8.0.1 +python-slugify==8.0.2 # via # ckanapi # frictionless @@ -215,7 +219,7 @@ requests==2.31.0 # libhxl # requests-file # requests-oauthlib -requests-file==1.5.1 +requests-file==2.0.0 # via hdx-python-utilities requests-oauthlib==1.3.1 # via google-auth-oauthlib @@ -239,7 +243,6 @@ six==1.16.0 # isodate # pockets # python-dateutil - # requests-file # sphinxcontrib-napoleon sphinxcontrib-napoleon==0.7 # via defopt @@ -248,6 +251,8 @@ sqlalchemy==2.0.25 # hapi-pipelines (pyproject.toml) # hapi-schema # hdx-python-database +strenum==0.4.15 + # via gspread stringcase==1.2.0 # via frictionless structlog==24.1.0 @@ -259,7 +264,9 @@ tabulate==0.9.0 text-unidecode==1.3 # via python-slugify typer[all]==0.9.0 - # via frictionless + # via + # frictionless + # typer typing-extensions==4.9.0 # via # frictionless diff --git a/src/hapi/pipelines/database/admins.py b/src/hapi/pipelines/database/admins.py index 35a654b0..4dae0f36 100644 --- a/src/hapi/pipelines/database/admins.py +++ b/src/hapi/pipelines/database/admins.py @@ -71,7 +71,7 @@ def _update_admin_table( for i, row in enumerate(admin_filter): code = row.get("#adm+code") name = row.get("#adm+name") - reference_period_start = parse_date(row.get("#date+start")) + time_period_start = parse_date(row.get("#date+start")) parent = row.get("#adm+code+parent") parent_ref = parent_dict.get(parent) if not parent_ref: @@ -92,14 +92,14 @@ def _update_admin_table( location_ref=parent_ref, code=code, name=name, - reference_period_start=reference_period_start, + reference_period_start=time_period_start, ) elif desired_admin_level == "2": admin_row = DBAdmin2( admin1_ref=parent_ref, code=code, name=name, - reference_period_start=reference_period_start, + reference_period_start=time_period_start, ) self._session.add(admin_row) if i % self._limit == 0: @@ -108,7 +108,7 @@ def _update_admin_table( def _add_admin1_connector_rows(self): for location_code, location_ref in self._locations.data.items(): - reference_period_start = ( + time_period_start = ( self._session.query(DBLocation) .filter(DBLocation.id == location_ref) .one() @@ -121,14 +121,14 @@ def _add_admin1_connector_rows(self): ), name="UNSPECIFIED", is_unspecified=True, - reference_period_start=reference_period_start, + reference_period_start=time_period_start, ) self._session.add(admin_row) self._session.commit() def _add_admin2_connector_rows(self): for admin1_code, admin1_ref in self.admin1_data.items(): - reference_period_start = ( + time_period_start = ( self._session.query(DBAdmin1) .filter(DBAdmin1.id == admin1_ref) .one() @@ -141,7 +141,7 @@ def _add_admin2_connector_rows(self): ), name="UNSPECIFIED", is_unspecified=True, - reference_period_start=reference_period_start, + reference_period_start=time_period_start, ) self._session.add(admin_row) self._session.commit() diff --git a/src/hapi/pipelines/database/food_security.py b/src/hapi/pipelines/database/food_security.py index bd228ff0..5345ab81 100644 --- a/src/hapi/pipelines/database/food_security.py +++ b/src/hapi/pipelines/database/food_security.py @@ -73,9 +73,9 @@ def populate(self): ][irow] ) ( - reference_period_start, - reference_period_end, - ) = _get_reference_period( + time_period_start, + time_period_end, + ) = _get_time_period( month_range=values[reference_period_months_column][ admin_code ][irow], @@ -101,8 +101,8 @@ def populate(self): admin2_ref=admin2_ref, ipc_phase_code=ipc_phase_code, ipc_type_code=ipc_type_code, - reference_period_start=reference_period_start, - reference_period_end=reference_period_end, + reference_period_start=time_period_start, + reference_period_end=time_period_end, population_in_phase=population_in_phase, population_fraction_in_phase=population_in_phase / population_total @@ -128,11 +128,11 @@ def _get_ipc_type_code_from_data(ipc_type_from_data: str) -> str: ) from e -def _get_reference_period(month_range: str, year: str) -> (datetime, datetime): - reference_period_start = parse_date_range( +def _get_time_period(month_range: str, year: str) -> (datetime, datetime): + time_period_start = parse_date_range( f"{year} {month_range.split('-')[0]}" )[0] - reference_period_end = parse_date_range( - f"{year} {month_range.split('-')[1]}" - )[1] - return reference_period_start, reference_period_end + time_period_end = parse_date_range(f"{year} {month_range.split('-')[1]}")[ + 1 + ] + return time_period_start, time_period_end diff --git a/src/hapi/pipelines/database/humanitarian_needs.py b/src/hapi/pipelines/database/humanitarian_needs.py index 198177eb..d8e3e585 100644 --- a/src/hapi/pipelines/database/humanitarian_needs.py +++ b/src/hapi/pipelines/database/humanitarian_needs.py @@ -57,8 +57,8 @@ def match_column(col, pattern_to_code): return None for dataset in self._results.values(): - reference_period_start = dataset["reference_period"]["startdate"] - reference_period_end = dataset["reference_period"]["enddate"] + time_period_start = dataset["time_period"]["start"] + time_period_end = dataset["time_period"]["end"] for admin_level, admin_results in dataset["results"].items(): resource_id = admin_results["hapi_resource_metadata"]["hdx_id"] @@ -115,8 +115,8 @@ def match_column(col, pattern_to_code): age_range_code=age_range_code, disabled_marker=disabled_marker, population=value, - reference_period_start=reference_period_start, - reference_period_end=reference_period_end, + reference_period_start=time_period_start, + reference_period_end=time_period_end, # TODO: For v2+, add to scraper (HAPI-199) source_data="not yet implemented", ) diff --git a/src/hapi/pipelines/database/national_risk.py b/src/hapi/pipelines/database/national_risk.py index affd89c2..9c9bb9be 100644 --- a/src/hapi/pipelines/database/national_risk.py +++ b/src/hapi/pipelines/database/national_risk.py @@ -32,12 +32,10 @@ def populate(self): datasetinfo = self._metadata.runner.scrapers[ "national_risk_national" ].datasetinfo - reference_period_start = datasetinfo["source_date"][ - "default_date" - ]["start"] - reference_period_end = datasetinfo["source_date"]["default_date"][ - "end" + time_period_start = datasetinfo["source_date"]["default_date"][ + "start" ] + time_period_end = datasetinfo["source_date"]["default_date"]["end"] for admin_level, admin_results in dataset["results"].items(): resource_id = admin_results["hapi_resource_metadata"]["hdx_id"] hxl_tags = admin_results["headers"][1] @@ -73,8 +71,8 @@ def populate(self): meta_avg_recentness_years=values[ "#meta+recentness+avg" ].get(location), - reference_period_start=reference_period_start, - reference_period_end=reference_period_end, + reference_period_start=time_period_start, + reference_period_end=time_period_end, # TODO: For v2+, add to scraper (HAPI-199) source_data="not yet implemented", ) diff --git a/src/hapi/pipelines/database/operational_presence.py b/src/hapi/pipelines/database/operational_presence.py index d88d941e..791d9b0d 100644 --- a/src/hapi/pipelines/database/operational_presence.py +++ b/src/hapi/pipelines/database/operational_presence.py @@ -41,8 +41,8 @@ def populate(self): rows = [] number_duplicates = 0 for dataset in self._results.values(): - reference_period_start = dataset["reference_period"]["startdate"] - reference_period_end = dataset["reference_period"]["enddate"] + time_period_start = dataset["time_period"]["start"] + time_period_end = dataset["time_period"]["end"] for admin_level, admin_results in dataset["results"].items(): resource_id = admin_results["hapi_resource_metadata"]["hdx_id"] hxl_tags = admin_results["headers"][1] @@ -104,9 +104,7 @@ def populate(self): acronym=org_acronym, org_name=org_name, org_type=org_type_code, - reference_period_start=parse_date( - "2023-11-21" - ), + time_period_start=parse_date("2023-11-21"), ) sector = values[sector_index][admin_code][i] sector_code = self._sector.get_sector_code(sector) @@ -134,8 +132,8 @@ def populate(self): admin2_ref=admin2_ref, org_ref=org_ref, sector_code=sector_code, - reference_period_start=reference_period_start, - reference_period_end=reference_period_end, + reference_period_start=time_period_start, + reference_period_end=time_period_end, # TODO: Add to scraper (HAPI-199) source_data="not yet implemented", ) diff --git a/src/hapi/pipelines/database/org.py b/src/hapi/pipelines/database/org.py index 83d68588..464c440d 100644 --- a/src/hapi/pipelines/database/org.py +++ b/src/hapi/pipelines/database/org.py @@ -44,16 +44,16 @@ def populate_single( acronym, org_name, org_type, - reference_period_start, - reference_period_end=None, + time_period_start, + time_period_end=None, ): logger.info(f"Adding org {org_name}") org_row = DBOrg( acronym=acronym, name=org_name, org_type_code=org_type, - reference_period_start=reference_period_start, - reference_period_end=reference_period_end, + reference_period_start=time_period_start, + reference_period_end=time_period_end, ) self._session.add(org_row) self._session.commit() diff --git a/src/hapi/pipelines/database/population.py b/src/hapi/pipelines/database/population.py index a73a2b36..3bb29b77 100644 --- a/src/hapi/pipelines/database/population.py +++ b/src/hapi/pipelines/database/population.py @@ -40,8 +40,8 @@ def __init__( def populate(self): logger.info("Populating population table") for dataset in self._results.values(): - reference_period_start = dataset["reference_period"]["startdate"] - reference_period_end = dataset["reference_period"]["enddate"] + time_period_start = dataset["time_period"]["start"] + time_period_end = dataset["time_period"]["end"] for admin_level, admin_results in dataset["results"].items(): resource_id = admin_results["hapi_resource_metadata"]["hdx_id"] @@ -81,8 +81,8 @@ def populate(self): gender_code=gender_code, age_range_code=age_range_code, population=int(value), - reference_period_start=reference_period_start, - reference_period_end=reference_period_end, + reference_period_start=time_period_start, + reference_period_end=time_period_end, # TODO: For v2+, add to scraper (HAPI-199) source_data="not yet implemented", )