Merge pull request #193 from OCHA-DAP/bugfix/idps-test

Bugfix/idps test

CHANGELOG.md

@@ -4,11 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## [0.10.15] - 2024-10-27
+## [0.10.16] - 2024-10-28
 
 ### Fixed
 
 - Second Haiti population update
+- Date issue in IDPs pipeline
 
 ## [0.10.14] - 2024-10-26
 

requirements.txt

@@ -22,7 +22,7 @@ ckanapi==4.8
     # via hdx-python-api
 click==8.1.7
     # via typer
-coverage==7.6.3
+coverage==7.6.4
     # via pytest-cov
 defopt==6.4.0
     # via hdx-python-api
@@ -38,7 +38,7 @@ docutils==0.21.2
     # via defopt
 email-validator==2.2.0
     # via hdx-python-api
-et-xmlfile==1.1.0
+et-xmlfile==2.0.0
     # via openpyxl
 filelock==3.16.1
     # via virtualenv
@@ -52,9 +52,9 @@ google-auth-oauthlib==1.2.1
     # via gspread
 greenlet==3.1.1
     # via sqlalchemy
-gspread==6.1.3
+gspread==6.1.4
     # via hdx-python-scraper
-hapi-schema==0.9.2
+hapi-schema==0.9.3
     # via hapi-pipelines (pyproject.toml)
 hdx-python-api==6.3.4
     # via
@@ -103,7 +103,7 @@ jsonschema==4.23.0
     #   tableschema-to-template
 jsonschema-specifications==2024.10.1
     # via jsonschema
-libhxl==5.2.1
+libhxl==5.2.2
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -116,7 +116,7 @@ markdown-it-py==3.0.0
     # via rich
 marko==2.1.2
     # via frictionless
-markupsafe==3.0.1
+markupsafe==3.0.2
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py
@@ -134,6 +134,8 @@ packaging==24.1
     # via pytest
 petl==1.7.15
     # via frictionless
+pip==24.3.1
+    # via simpleeval
 platformdirs==4.3.6
     # via virtualenv
 pluggy==1.5.0
@@ -216,7 +218,7 @@ requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
-rich==13.9.2
+rich==13.9.3
     # via typer
 rpds-py==0.20.0
     # via
@@ -226,13 +228,13 @@ rsa==4.9
     # via google-auth
 ruamel-yaml==0.18.6
     # via hdx-python-utilities
-ruamel-yaml-clib==0.2.8
+ruamel-yaml-clib==0.2.12
     # via ruamel-yaml
 setuptools==75.2.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
-simpleeval==1.0.0
+simpleeval==1.0.1
     # via frictionless
 simplejson==3.19.3
     # via ckanapi
@@ -259,7 +261,7 @@ tabulate==0.9.0
     # via frictionless
 text-unidecode==1.3
     # via python-slugify
-typeguard==4.3.0
+typeguard==4.4.0
     # via inflect
 typer==0.12.5
     # via frictionless
@@ -282,7 +284,7 @@ urllib3==2.2.3
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.26.6
+virtualenv==20.27.0
     # via pre-commit
 wheel==0.44.0
     # via libhxl

src/hapi/pipelines/database/idps.py

@@ -4,6 +4,7 @@
 from typing import Dict
 
 from hapi_schema.db_idps import DBIDPs
+from hdx.utilities.dateparse import parse_date
 from sqlalchemy.orm import Session
 
 from ..utilities.logging_helpers import add_message
@@ -97,8 +98,8 @@ def populate(self) -> None:
                         reporting_round=reporting_round,
                         operation=operation,
                         population=row[hxl_tags.index("#affected+idps")],
-                        reference_period_start=date_reported,
-                        reference_period_end=date_reported,
+                        reference_period_start=parse_date(date_reported),
+                        reference_period_end=parse_date(date_reported),
                     )
                     self._session.add(idps_row)
                     duplicate_rows.add(duplicate_row_check)