Updates for Databricks SQL Package and API Date validation (rtdip#733)

* Updates for Packages and API dates Signed-off-by: GBBBAS <[email protected]> * Remove test Signed-off-by: GBBBAS <[email protected]> --------- Signed-off-by: GBBBAS <[email protected]>
GBBBAS · May 3, 2024 · 59da351 · 59da351
1 parent 4917027
commit 59da351
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 9 deletions.
diff --git a/environment.yml b/environment.yml
@@ -39,7 +39,7 @@ dependencies:
     - fastapi==0.110.0
     - httpx==0.24.1
     - pyspark>=3.3.0,<3.6.0
-    - delta-spark>=2.2.0,<3.2.0
+    - delta-spark>=2.2.0,<3.3.0
     - grpcio>=1.48.1
     - grpcio-status>=1.48.1
     - googleapis-common-protos>=1.56.4
@@ -52,7 +52,7 @@ dependencies:
     - mkdocs-macros-plugin==1.0.1
     - pygments==2.16.1
     - pymdown-extensions==10.8.1
-    - databricks-sql-connector==3.1.0
+    - databricks-sql-connector==2.9.3
     - semver==3.0.0
     - xlrd==2.0.1
     - pygithub==1.59.0
@@ -81,4 +81,4 @@ dependencies:
         - trio==0.22.1
         - sqlparams==5.1.0
         - entsoe-py==0.5.10
-        - eth-typing==4.2.1
+        - eth-typing>=4.2.2
diff --git a/setup.py b/setup.py
@@ -59,6 +59,7 @@
     "hvac==1.1.1",
     "azure-keyvault-secrets==4.7.0",
     "web3==6.16.0",
+    "eth-typing>=4.2.2",
     "polars[deltalake]==0.18.8",
     "delta-sharing==1.0.0",
     "xarray>=2023.1.0,<2023.8.0",

diff --git a/src/api/requirements.txt b/src/api/requirements.txt
@@ -5,7 +5,7 @@ pydantic==2.6.0
 # turbodbc==4.11.0
 pyodbc==4.0.39
 importlib_metadata>=1.0.0
-databricks-sql-connector==3.1.0
+databricks-sql-connector==2.9.3
 azure-identity==1.15.0
 oauthlib>=3.2.2
 pandas>=2.0.1,<3.0.0

diff --git a/src/api/v1/models.py b/src/api/v1/models.py
@@ -15,8 +15,15 @@
 import os
 from datetime import datetime
 from tracemalloc import start
-from pydantic import BaseModel, ConfigDict, Field, field_serializer
-from typing import List, Union, Dict, Any
+from pydantic import (
+    BaseModel,
+    BeforeValidator,
+    ConfigDict,
+    Field,
+    Strict,
+    field_serializer,
+)
+from typing import Annotated, List, Union, Dict, Any
 from fastapi import Query, Header, Depends
 from datetime import date
 from src.api.auth.azuread import oauth2_scheme
@@ -244,6 +251,13 @@ def __init__(
         self.tag_name = tag_name
 
 
+def check_date(v: str) -> str:
+    assert (
+        len(v) == 10 or len(v) == 15 or len(v) == 16
+    )  # "Date must be in format YYYY-MM-DD, YYYY-MM-DD+zzzz or YYYY-MM-DD+zz:zz"
+    return v
+
+
 class RawQueryParams:
     def __init__(
         self,
@@ -255,12 +269,14 @@ def __init__(
         include_bad_data: bool = Query(
             ..., description="Include or remove Bad data points"
         ),
-        start_date: Union[datetime, date] = Query(
+        start_date: Union[
+            Annotated[date, BeforeValidator(check_date)], datetime
+        ] = Query(
             ...,
             description="Start Date in format YYYY-MM-DD or YYYY-MM-DDTHH:mm:ss or YYYY-MM-DDTHH:mm:ss+zz:zz",
             examples=[EXAMPLE_DATE, EXAMPLE_DATETIME, EXAMPLE_DATETIME_TIMEZOME],
         ),
-        end_date: Union[datetime, date] = Query(
+        end_date: Union[Annotated[date, BeforeValidator(check_date)], datetime] = Query(
             ...,
             description="End Date in format YYYY-MM-DD or YYYY-MM-DDTHH:mm:ss or YYYY-MM-DDTHH:mm:ss+zz:zz",
             examples=[EXAMPLE_DATE, EXAMPLE_DATETIME, EXAMPLE_DATETIME_TIMEZOME],

diff --git a/src/sdk/python/rtdip_sdk/queries/_utilities_query_builder.py b/src/sdk/python/rtdip_sdk/queries/_utilities_query_builder.py
@@ -29,7 +29,7 @@ def _is_date_format(dt, format):
 
 def _parse_date(dt, is_end_date=False, exclude_date_format=False):  # NOSONAR
     if isinstance(dt, datetime):
-        if dt.time() == time.min:
+        if dt.time() == time.min and not is_end_date:
             if dt.tzinfo is not None:
                 dt = datetime.strftime(dt, "%Y-%m-%d%z")
             else: