Update SEM Transformer Column and Datatype Mappings (rtdip#503)

* update sem transformer columns Signed-off-by: cching95 <[email protected]> * update unit test Signed-off-by: cching95 <[email protected]> --------- Signed-off-by: cching95 <[email protected]>
GBBBAS · Sep 22, 2023 · 998364d · 998364d
1 parent 159e181
commit 998364d
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 39 deletions.
diff --git a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/obc_field_mappings.py b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/obc_field_mappings.py
@@ -13,54 +13,54 @@
 # limitations under the License.
 
 OBC_FIELD_MAPPINGS_V10 = {
-    "0": {"TagName": "obc_timeStamp", "ValueType": "double"},
-    "1": {"TagName": "obc_errorState", "ValueType": "integer"},
+    "0": {"TagName": "obc_timeStamp", "ValueType": "float"},
+    "1": {"TagName": "obc_errorState", "ValueType": "float"},
     "2": {"TagName": "obc_energyThreshold", "ValueType": "float"},
     "3": {"TagName": "obc_energyLevel", "ValueType": "float"},
-    "4": {"TagName": "gps_errorState", "ValueType": "integer"},
+    "4": {"TagName": "gps_errorState", "ValueType": "float"},
     "5": {"TagName": "gps_utcTime", "ValueType": "string"},
-    "6": {"TagName": "gps_latitude", "ValueType": "double"},
-    "7": {"TagName": "gps_longitude", "ValueType": "double"},
-    "8": {"TagName": "gps_fixQuality", "ValueType": "integer"},
+    "6": {"TagName": "gps_latitude", "ValueType": "float"},
+    "7": {"TagName": "gps_longitude", "ValueType": "float"},
+    "8": {"TagName": "gps_fixQuality", "ValueType": "float"},
     "9": {"TagName": "gps_dilution", "ValueType": "float"},
     "10": {"TagName": "gps_speed", "ValueType": "float"},
-    "11": {"TagName": "jm_errorState", "ValueType": "integer"},
-    "12": {"TagName": "jm_posJoule", "ValueType": "integer"},
-    "13": {"TagName": "jm_negJoule", "ValueType": "integer"},
-    "14": {"TagName": "jm_netJoule", "ValueType": "integer"},
+    "11": {"TagName": "jm_errorState", "ValueType": "float"},
+    "12": {"TagName": "jm_posJoule", "ValueType": "float"},
+    "13": {"TagName": "jm_negJoule", "ValueType": "float"},
+    "14": {"TagName": "jm_netJoule", "ValueType": "float"},
     "15": {"TagName": "jm_globalTime", "ValueType": "string"},
-    "16": {"TagName": "jm_voltage", "ValueType": "integer"},
-    "17": {"TagName": "jm_current", "ValueType": "integer"},
+    "16": {"TagName": "jm_voltage", "ValueType": "float"},
+    "17": {"TagName": "jm_current", "ValueType": "float"},
     "18": {"TagName": "jm_battVolt", "ValueType": "float"},
-    "19": {"TagName": "lfm_errorState", "ValueType": "integer"},
-    "20": {"TagName": "lfm_msSinceReset", "ValueType": "integer"},
+    "19": {"TagName": "lfm_errorState", "ValueType": "float"},
+    "20": {"TagName": "lfm_msSinceReset", "ValueType": "float"},
     "21": {"TagName": "lfm_temperature", "ValueType": "float"},
     "22": {"TagName": "lfm_integratedCorrFlow", "ValueType": "float"},
     "23": {"TagName": "lfm_instantFlow", "ValueType": "float"},
-    "24": {"TagName": "gfs_errorState", "ValueType": "integer"},
+    "24": {"TagName": "gfs_errorState", "ValueType": "float"},
     "25": {"TagName": "gfs_gasFlow", "ValueType": "float"},
     "26": {"TagName": "gfs_gasTemp", "ValueType": "float"},
     "27": {"TagName": "gfs_netGasVolume", "ValueType": "float"},
-    "28": {"TagName": "do_errorState", "ValueType": "integer"},
-    "29": {"TagName": "do_state", "ValueType": "integer"},
-    "30": {"TagName": "jm3_errorState", "ValueType": "integer"},
-    "31": {"TagName": "jm3_ms_since_reset", "ValueType": "integer"},
-    "32": {"TagName": "jm3_netjoule", "ValueType": "integer"},
-    "33": {"TagName": "jm3_voltage", "ValueType": "integer"},
-    "34": {"TagName": "jm3_current", "ValueType": "integer"},
-    "35": {"TagName": "sup_errorState", "ValueType": "integer"},
-    "36": {"TagName": "sup_VoltageIn", "ValueType": "integer"},
-    "37": {"TagName": "all_msSinceReset", "ValueType": "integer"},
-    "38": {"TagName": "iot_errorState", "ValueType": "integer"},
-    "39": {"TagName": "iot_route", "ValueType": "text"},
-    "40": {"TagName": "cell_mode", "ValueType": "text"},
-    "41": {"TagName": "cell_rssi", "ValueType": "integer"},
-    "42": {"TagName": "wifi_power", "ValueType": "integer"},
+    "28": {"TagName": "do_errorState", "ValueType": "float"},
+    "29": {"TagName": "do_state", "ValueType": "float"},
+    "30": {"TagName": "jm3_errorState", "ValueType": "float"},
+    "31": {"TagName": "jm3_ms_since_reset", "ValueType": "float"},
+    "32": {"TagName": "jm3_netjoule", "ValueType": "float"},
+    "33": {"TagName": "jm3_voltage", "ValueType": "float"},
+    "34": {"TagName": "jm3_current", "ValueType": "float"},
+    "35": {"TagName": "sup_errorState", "ValueType": "float"},
+    "36": {"TagName": "sup_VoltageIn", "ValueType": "float"},
+    "37": {"TagName": "all_msSinceReset", "ValueType": "float"},
+    "38": {"TagName": "iot_errorState", "ValueType": "float"},
+    "39": {"TagName": "iot_route", "ValueType": "string"},
+    "40": {"TagName": "cell_mode", "ValueType": "string"},
+    "41": {"TagName": "cell_rssi", "ValueType": "float"},
+    "42": {"TagName": "wifi_power", "ValueType": "float"},
     "43": {"TagName": "message_id", "ValueType": "string"},
-    "44": {"TagName": "volt_errorState", "ValueType": "integer"},
-    "45": {"TagName": "volt_ms_since_reset", "ValueType": "integer"},
-    "46": {"TagName": "volt_voltage", "ValueType": "integer"},
-    "47": {"TagName": "volt_difference", "ValueType": "integer"},
-    "48": {"TagName": "sb_errorState", "ValueType": "integer"},
-    "49": {"TagName": "sb_isAutonomous", "ValueType": "integer"},
+    "44": {"TagName": "volt_errorState", "ValueType": "float"},
+    "45": {"TagName": "volt_ms_since_reset", "ValueType": "float"},
+    "46": {"TagName": "volt_voltage", "ValueType": "float"},
+    "47": {"TagName": "volt_difference", "ValueType": "float"},
+    "48": {"TagName": "sb_errorState", "ValueType": "float"},
+    "49": {"TagName": "sb_isAutonomous", "ValueType": "float"},
 }
diff --git a/src/sdk/python/rtdip_sdk/pipelines/transformers/spark/sem_json_to_pcdm.py b/src/sdk/python/rtdip_sdk/pipelines/transformers/spark/sem_json_to_pcdm.py
@@ -129,7 +129,7 @@ def transform(self) -> DataFrame:
                     "TagName",
                     concat(
                         col("dID"),
-                        lit("_"),
+                        lit(":"),
                         udf(lambda row: mapping[row]["TagName"])(col("pos")),
                     ),
                 )

diff --git a/tests/sdk/python/rtdip_sdk/pipelines/transformers/spark/test_sem_json_to_pcdm.py b/tests/sdk/python/rtdip_sdk/pipelines/transformers/spark/test_sem_json_to_pcdm.py
@@ -50,10 +50,10 @@ def test_sem_json_to_pcdm(spark_session: SparkSession):
     expected_data = [
         {
             "EventTime": datetime.fromisoformat("2023-09-18T14:30:39.192+00:00"),
-            "TagName": "502_obc_timeStamp",
+            "TagName": "502:obc_timeStamp",
             "Status": "Good",
             "Value": "1685025760.46",
-            "ValueType": "double",
+            "ValueType": "float",
             "ChangeType": "insert",
         }
     ]