Skip to content

Commit

Permalink
Try dev instead of demo
Browse files Browse the repository at this point in the history
  • Loading branch information
mcarans committed Jan 14, 2025
1 parent 434dba7 commit 01a07ba
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 13 deletions.
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ dependencies = [
"defopt>=6.4.0",
"email_validator",
"hdx-python-country>=3.8.6",
"hdx-python-utilities>=3.8.0",
"hdx-python-utilities>=3.8.2",
"libhxl>=5.2.2",
"makefun",
"quantulum3",
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ gspread==6.1.4
# via hdx-python-api (pyproject.toml)
hdx-python-country==3.8.6
# via hdx-python-api (pyproject.toml)
hdx-python-utilities==3.8.0
hdx-python-utilities==3.8.2
# via
# hdx-python-api (pyproject.toml)
# hdx-python-country
Expand Down
48 changes: 39 additions & 9 deletions src/hdx/api/utilities/hdx_error_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,18 @@ class HDXErrorHandler(ErrorHandler):
sorted.
Args:
should_exit_on_error (bool): Whether to exit with a 1 code if there are errors. Default is True.
should_exit_on_error (bool): Whether to exit with a 1 code if there are errors. Default is False.
write_to_hdx (bool): Whether to write errors to HDX resources. Default is False.
"""

def __init__(
self,
should_exit_on_error: bool = True,
should_exit_on_error: bool = False,
write_to_hdx: bool = False,
):
super().__init__(should_exit_on_error)
self._write_to_hdx = write_to_hdx
self.shared_errors["hdx_error"] = {}

@staticmethod
Expand Down Expand Up @@ -81,6 +84,7 @@ def add_message(
fixed format:
pipeline - identifier - {text}
identifier is usually a dataset name.
Args:
pipeline (str): Name of the pipeline originating the error
identifier (str): Identifier e.g. dataset name
Expand Down Expand Up @@ -111,6 +115,7 @@ def add_missing_value_message(
to a dictionary of messages in a fixed format:
pipeline - identifier - {text}
identifier is usually a dataset name.
Args:
pipeline (str): Name of the scaper originating the error
identifier (str): Identifier e.g. dataset name
Expand Down Expand Up @@ -148,6 +153,7 @@ def add_multi_valued_message(
pipeline - identifier - n {text}. First 10 values: n1,n2,n3...
If less than 10 values, ". First 10 values" is omitted. identifier is usually
a dataset name.
Args:
pipeline (str): Name of the scaper originating the error
identifier (str): Identifier e.g. dataset name
Expand All @@ -172,22 +178,46 @@ def add_multi_valued_message(
)
return True

def output_errors(self, err_to_hdx: bool) -> None:
def write_errors_to_hdx(self) -> None:
"""
Write to HDX resources corresponding errors that have been flagged by
setting err_to_hdx True when adding messages
Returns:
None
"""
logger.info("Writing errors to HDX")
for identifier, errors in self.shared_errors["hdx_error"].items():
write_errors_to_resource(identifier, errors)

def output_errors(self) -> None:
"""
Log errors and warnings by category and sorted. Also write to HDX
resources corresponding errors that have been flagged by setting
err_to_hdx True when adding messages.
Returns:
None
"""
self.log()
if err_to_hdx:
logger.info("Writing errors to HDX")
for identifier, errors in self.shared_errors["hdx_error"].items():
write_error_to_resource(identifier, errors)
if self._write_to_hdx:
self.write_errors_to_hdx()

def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
super().__exit__(exc_type, exc_value, traceback)
if self._write_to_hdx:
self.write_errors_to_hdx()

def write_error_to_resource(

def write_errors_to_resource(
identifier: Tuple[str, str, str], errors: set[str]
) -> bool:
"""
Writes error messages to a resource on HDX. If the resource already has an
error message, it is only overwritten if the two messages are different.
Args:
identifier (Tuple[str, str, str]): The scraper, dataset, and resource names that the message applies to
identifier (Tuple[str, str, str]): Scraper, dataset, and resource names that the message applies to
errors (set[str]): Set of errors to use e.g. "negative values removed"
Returns:
bool: True if a message was added, False if not
Expand Down
4 changes: 2 additions & 2 deletions tests/hdx/utilities/test_hdx_error_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ class TestHDXErrorHandler:
def test_hdx_error_handler(self, caplog):
with pytest.raises(SystemExit):
with caplog.at_level(logging.ERROR):
with HDXErrorHandler() as errors:
with HDXErrorHandler(should_exit_on_error=True) as errors:
errors.add_message(
"pipeline1",
"dataset1",
Expand Down Expand Up @@ -54,5 +54,5 @@ def test_hdx_error_handler(self, caplog):
)
== 3
)
errors.output_errors(False)
errors.output_errors()
assert "following values changed" in caplog.text

0 comments on commit 01a07ba

Please sign in to comment.