Fix load credentials from string (#6)

* correct type info for `service_account_json_str` * shorten schema definitions
haleemur · Feb 27, 2024 · 3bbb5d4 · 3bbb5d4
1 parent e99fd73
commit 3bbb5d4
Show file tree

Hide file tree

Showing 22 changed files with 1,116 additions and 157 deletions.
diff --git a/meltano.yml b/meltano.yml
@@ -17,7 +17,7 @@ plugins:
     - stream-maps
     config:
       start_date: '2024-01-01T00:00:00Z'
-      service_account_json_file: $GOOGLE_PLAY_CREDENTIALS_FILE
+      service_account_json_str: $GOOGLE_PLAY_CREDENTIALS
       bucket_name: $GOOGLE_PLAY_BUCKET
     settings:
     - name: service_account_json_file
@@ -28,7 +28,8 @@ plugins:
       kind: string
     - name: start_date
       kind: string
-
+    select:
+      - reviews
   loaders:
   - name: target-jsonl
     variant: andyh1203

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tap-playstore"
-version = "0.1.2"
+version = "0.1.3"
 description = "`tap-playstore` is a Singer tap for Google PlayStore Console Reports, built with the Meltano Singer SDK."
 readme = "README.md"
 authors = ["Haleemur Ali <[email protected]>"]
@@ -23,7 +23,7 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8"
+python = ">=3.8,<3.13"
 importlib-resources = { version = "==6.1.*", python = "<3.9" }
 singer-sdk = { version="~=0.35.2" }
 fs-s3fs = { version = "~=1.1.1", optional = true }
@@ -34,6 +34,7 @@ google-cloud-storage = "^2.14.0"
 pytest = ">=7.4.0"
 singer-sdk = { version="~=0.35.2", extras = ["testing"] }
 pre-commit = "2.21.0"
+meltano = "^3.3.1"
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

diff --git a/tap_playstore/client.py b/tap_playstore/client.py
@@ -32,7 +32,9 @@ class PlayStoreStream(Stream):
     """
 
     replication_method = REPLICATION_INCREMENTAL
+
     replication_key = "_file_updated_at"
+    primary_keys = ("_bucket", "_file_path", "_file_lineno")
 
     def _yield_rows_from_file(
         self,
@@ -42,23 +44,18 @@ def _yield_rows_from_file(
     ) -> Iterable[dict]:
         """Yield processed rows from csv file as dict."""
         fields = [re.sub(r"(\W|\s)+", "_", fn.lower()).rstrip("_") for fn in header]
+        static_ = static_fields.copy()
+        typemap = {
+            f: t
+            for f, t in self._typemap_from_string.items()
+            if f in fields and t not in ["string", "null-string"]
+        }
         for i, row in enumerate(reader):
-            context = {
-                "fields": set(fields),
-                "static_fields": static_fields,
-                "_file_lineno": i,
-            }
-            rec = self.convert_types_and_add_metadata(dict(zip(fields, row)), context)
-            yield rec
-
-    def convert_types_and_add_metadata(
-        self, row: dict[str, Any], context: dict
-    ) -> dict[str, Any]:
-        """Post process string values read from the CSV files."""
-        fields = context["fields"]
-        static_fields = context["static_fields"]
-        file_lineno = {"_file_lineno": context["_file_lineno"]}
-        typemap = {f: t for f, t in self._typemap_from_string.items() if f in fields}
+            static_["_file_lineno"] = i
+            yield self.convert_types(dict(zip(fields, row)), typemap) | static_
+
+    def convert_types(self, row: dict[str, Any], typemap: dict) -> dict[str, Any]:
+        """Convert string values read from the CSV files."""
         for field, type_ in typemap.items():
             if type_ == "integer":
                 row[field] = int(row[field])
@@ -68,11 +65,9 @@ def convert_types_and_add_metadata(
                 row[field] = float(row["field"])
             elif type_ == "null-number":
                 row[field] = None if row[field] == "" else float(row[field])
-            elif type_ in {"string", "null-string"}:
-                continue
             else:
                 raise NotImplementedError
-        return row | static_fields | file_lineno
+        return row
 
     @property
     def _typemap_from_string(self) -> dict[str, str]:

diff --git a/tap_playstore/schemas/earnings.json b/tap_playstore/schemas/earnings.json
@@ -1,11 +1,4 @@
 {
-    "tap_stream_id": "earnings",
-    "stream": "earnings",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
     "properties": {
         "description": {
             "type": ["null", "string"]

diff --git a/tap_playstore/schemas/play_balance_krw.json b/tap_playstore/schemas/play_balance_krw.json
@@ -1,11 +1,4 @@
 {
-    "tap_stream_id": "reviews",
-    "stream": "reviews",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
     "type": "object",
     "properties": {
         "order_number": {

diff --git a/tap_playstore/schemas/reviews.json b/tap_playstore/schemas/reviews.json
@@ -1,6 +1,4 @@
 {
-    "tap_stream_id": "reviews",
-    "stream": "reviews",
     "key_properties": [
         "_bucket",
         "_file_path",
@@ -23,13 +21,13 @@
             "type": ["null", "string"]
         },
         "review_submit_date_and_time": {
-            "type": ["null", "string"]
+            "type": ["null", "string"], "format": "date-time"
         },
         "review_submit_millis_since_epoch": {
             "type": ["null", "integer"]
         },
         "review_last_update_date_and_time": {
-            "type": ["null", "string"]
+            "type": ["null", "string"], "format": "date-time"
         },
         "review_last_update_millis_since_epoch": {
             "type": ["null", "integer"]

diff --git a/tap_playstore/schemas/sales_reports.json b/tap_playstore/schemas/sales_reports.json
@@ -1,11 +1,4 @@
 {
-    "tap_stream_id": "sales_reports",
-    "stream": "sales_reports",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
     "properties": {
         "order_number": {
             "type": ["null", "string"]

diff --git a/tap_playstore/schemas/stats_by_dimension/buyers_7d.json b/tap_playstore/schemas/stats_by_dimension/buyers_7d.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "stats_by_dimension_buyers_7d",
-    "stream": "stats_by_dimension_buyers_7d",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/stats_by_dimension/crashes.json b/tap_playstore/schemas/stats_by_dimension/crashes.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "stats_by_dimension_crashes",
-    "stream": "stats_by_dimension_crashes",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/stats_by_dimension/gcm.json b/tap_playstore/schemas/stats_by_dimension/gcm.json
@@ -1,13 +1,4 @@
 {
-    "tap_stream_id": "stats_by_dimension_gcm",
-    "stream": "stats_by_dimension_gcm",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
-
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/stats_by_dimension/installs.json b/tap_playstore/schemas/stats_by_dimension/installs.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "stats_by_dimension_installs",
-    "stream": "stats_by_dimension_installs",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/stats_by_dimension/ratings.json b/tap_playstore/schemas/stats_by_dimension/ratings.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "stats_by_dimension_ratings",
-    "stream": "stats_by_dimension_ratings",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/stats_by_dimension/retained_installers.json b/tap_playstore/schemas/stats_by_dimension/retained_installers.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "stats_by_dimension_retained_installers",
-    "stream": "stats_by_dimension_retained_installers",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/stats_overview/crashes.json b/tap_playstore/schemas/stats_overview/crashes.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "crashes_stats_by_overview",
-    "stream": "crashes_stats_by_overview",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/stats_overview/gcm.json b/tap_playstore/schemas/stats_overview/gcm.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "gcm_stats_by_overview",
-    "stream": "gcm_stats_by_overview",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/stats_overview/installs.json b/tap_playstore/schemas/stats_overview/installs.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "stats_installs",
-    "stream": "stats_installs",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/stats_overview/ratings.json b/tap_playstore/schemas/stats_overview/ratings.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "ratings_stats_by_overview",
-    "stream": "ratings_stats_by_overview",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/store_performance_country.json b/tap_playstore/schemas/store_performance_country.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "stats_by_dimension_store_performance",
-    "stream": "stats_by_dimension_store_performance",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/store_performance_traffic_source.json b/tap_playstore/schemas/store_performance_traffic_source.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "stats_by_dimension_store_performance",
-    "stream": "stats_by_dimension_store_performance",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "properties": {
         "date": {
             "type": ["null", "string"], "format": "date"

diff --git a/tap_playstore/schemas/subscriptions_country.json b/tap_playstore/schemas/subscriptions_country.json
@@ -1,12 +1,4 @@
 {
-    "tap_stream_id": "subscriptions_country",
-    "stream": "subscriptions_country",
-    "key_properties": [
-        "_bucket",
-        "_file_path",
-        "_file_lineno"
-    ],
-    "replication_key": "_file_updated_at",
     "type": "object",
     "properties": {
         "date": {

diff --git a/tap_playstore/tap.py b/tap_playstore/tap.py
@@ -28,7 +28,7 @@ class TapPlayStore(Tap):
         ),
         th.Property(
             "service_account_json_str",
-            th.ArrayType(th.StringType),
+            th.StringType,
             required=False,
             secret=True,
             description="Google Cloud Service Account JSON string",
@@ -45,7 +45,10 @@ class TapPlayStore(Tap):
             th.DateTimeType,
             required=False,
             secret=False,
-            description="The most recent record date to sync",
+            description=(
+                "The most recent record date to sync. "
+                "NOTE: this has not been implemented yet."
+            ),
         ),
         th.Property(
             "bucket_name",