Expose --abort-threshold on CLI (#147)

Author: oyvindwe

exabel_data_sdk/client/api/data_classes/prediction_model_run.py

@@ -114,7 +114,7 @@ def __repr__(self) -> str:
             f"PredictionModelRun(name='{self.name}', description='{self.description}', "
             f"configuration={self.configuration}, "
             f"configuration_source='{self.configuration_source}', "
-            f"auto_activate={self.auto_activate}"
+            f"auto_activate={self.auto_activate})"
         )
 
     def __lt__(self, other: object) -> bool:

exabel_data_sdk/scripts/csv_script.py

@@ -8,6 +8,7 @@
 from exabel_data_sdk.scripts.actions import DeprecatedArgumentAction
 from exabel_data_sdk.scripts.base_script import BaseScript
 from exabel_data_sdk.services.csv_loading_constants import (
+    DEFAULT_ABORT_THRESHOLD,
     DEFAULT_NUMBER_OF_RETRIES,
     DEFAULT_NUMBER_OF_THREADS,
 )
@@ -76,6 +77,15 @@ def __init__(self, argv: Sequence[str], description: str):
             help=f"The maximum number of retries to make for each failed request. Defaults to "
             f"{DEFAULT_NUMBER_OF_RETRIES}.",
         )
+        self.parser.add_argument(
+            "--abort-threshold",
+            required=False,
+            type=abort_threshold,
+            metavar="[0.0-1.0]",
+            default=DEFAULT_ABORT_THRESHOLD,
+            help=f"The threshold for the proportion of failed requests that will cause the "
+            f"upload to be aborted. Defaults to {DEFAULT_ABORT_THRESHOLD}.",
+        )
 
     def read_csv(
         self, args: argparse.Namespace, string_columns: Optional[Collection[Union[str, int]]] = None
@@ -85,3 +95,14 @@ def read_csv(
         if string_columns:
             dtype = {column: str for column in string_columns}
         return pd.read_csv(args.filename, header=0, sep=args.sep, dtype=dtype)
+
+
+def abort_threshold(val: str) -> float:
+    """Convert argument to a valid value for abort_threshold."""
+    try:
+        f = float(val)
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError(f"{val} not a floating-point literal") from exc
+    if not 0.0 <= f <= 1.0:
+        raise argparse.ArgumentTypeError(f"{val} not in range [0.0, 1.0]")
+    return f

exabel_data_sdk/scripts/load_entities_from_csv.py

@@ -136,6 +136,7 @@ def run_script(self, client: ExabelClient, args: argparse.Namespace) -> None:
                 dry_run=args.dry_run,
                 retries=args.retries,
                 batch_size=args.batch_size,
+                abort_threshold=args.abort_threshold,
             )
         except (FileLoadingException, ParsePropertyColumnsError) as e:
             print(e)

exabel_data_sdk/scripts/load_relationships_from_csv.py

@@ -208,6 +208,7 @@ def run_script(self, client: ExabelClient, args: argparse.Namespace) -> None:
                 dry_run=args.dry_run,
                 retries=args.retries,
                 batch_size=args.batch_size,
+                abort_threshold=args.abort_threshold,
             )
         except FileLoadingException as e:
             print(e)

exabel_data_sdk/scripts/load_time_series_from_file.py

@@ -149,6 +149,7 @@ def run_script(self, client: ExabelClient, args: argparse.Namespace) -> None:
                 batch_size=args.batch_size,
                 skip_validation=args.skip_validation,
                 case_sensitive_signals=args.case_sensitive_signals,
+                abort_threshold=args.abort_threshold,
             )
         except FileLoadingException as e:
             print(e)

exabel_data_sdk/services/csv_loading_constants.py

@@ -1,3 +1,4 @@
+DEFAULT_ABORT_THRESHOLD = 0.5
 DEFAULT_BULK_LOAD_CHECKPOINTS = 20
 DEFAULT_NUMBER_OF_THREADS = 40
 DEFAULT_NUMBER_OF_THREADS_FOR_IMPORT = 4