Fix Merging of variables (#128)

* update code pylint and flake8

* remove print

* add harmony deployment

* update harmony deployment url

* update changelog

* update changelog

.github/workflows/build-pipeline.yml

@@ -243,6 +243,19 @@ jobs:
           pull: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Deploy Harmony
+        env:
+            ENV: ${{ env.venue }}
+            CMR_USER: ${{ secrets.CMR_USER }}
+            CMR_PASS: ${{ secrets.CMR_PASS }}
+        if: |
+          github.ref == 'refs/heads/main'    ||
+          startsWith(github.ref, 'refs/heads/release')
+        working-directory: deployment
+        run: 
+          poetry run python harmony_deploy.py --tag ${{ env.software_version }} 
+
 # As of 2023/01/23 these steps below for scanning the Docker image with Snyk are failing. I've tried both the official Snyk
 # action https://github.com/snyk/actions/tree/master/docker and this method below of manually calling the CLI.
 # The error when using the official Snyk action is

CHANGELOG.md

@@ -7,11 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
-### Changed
+  - Update Github Actions
+    - Added harmony deployment into github actions.
 ### Changed
 ### Deprecated
 ### Removed
 ### Fixed
+  - Variable Merging
+    - Fixed way we merge variables when granules in a collection have varying variables.
 
 
 ## [0.9.0]

deployment/harmony_deploy.py

@@ -0,0 +1,67 @@
+import os
+import requests
+import json
+import logging
+import argparse
+from requests.auth import HTTPBasicAuth
+
+# Environment variables
+ENV = os.getenv('ENV')
+CMR_USER = os.getenv('CMR_USER')
+CMR_PASS = os.getenv('CMR_PASS')
+
+def bearer_token() -> str:
+    tokens = []
+    headers = {'Accept': 'application/json'}
+    url = f"https://{'uat.' if ENV == 'uat' else ''}urs.earthdata.nasa.gov/api/users"
+
+    # First just try to get a token that already exists
+    try:
+        resp = requests.get(url + "/tokens", headers=headers, auth=HTTPBasicAuth(CMR_USER, CMR_PASS))
+        response_content = json.loads(resp.content)
+
+        for x in response_content:
+            tokens.append(x['access_token'])
+
+    except Exception:  # noqa E722
+        logging.warning("Error getting the token - check user name and password", exc_info=True)
+
+    # No tokens exist, try to create one
+    if not tokens:
+        try:
+            resp = requests.post(url + "/token", headers=headers, auth=HTTPBasicAuth(CMR_USER, CMR_PASS))
+            response_content = json.loads(resp.content)
+            tokens.append(response_content['access_token'])
+        except Exception:  # noqa E722
+            logging.warning("Error getting the token - check user name and password", exc_info=True)
+
+    # If still no token, then we can't do anything
+    if not tokens:
+        raise RuntimeError("Unable to get bearer token from EDL")
+
+    return next(iter(tokens))
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description="Update the service image tag.")
+    parser.add_argument("--tag", help="The new tag version to update.", required=True)
+    args = parser.parse_args()
+
+    url = f"https://harmony.{'uat.' if ENV == 'uat' else ''}earthdata.nasa.gov/service-image-tag/podaac-concise"
+    token = bearer_token()
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-type": "application/json"
+    }
+    data = {
+        "tag": args.tag
+    }
+
+    response = requests.put(url, headers=headers, json=data)
+
+    print(response.status_code)
+    try:
+        print(response.json())
+    except json.JSONDecodeError:
+        print("Response content is not in JSON format")

podaac/merger/preprocess_worker.py

@@ -236,14 +236,10 @@ def _run_multi_core(file_list: list[Path],
         if var_info is None:
             var_info = result['var_info']
         elif var_info != result['var_info']:
-            if set(var_info.keys()).difference(result['var_info']):
-                # If not all variables match, only compare variables that intersect
-                intersecting_vars = set(var_info).intersection(result['var_info'])
-                if list(
-                        map(var_info.get, intersecting_vars)
-                ) != list(map(result['var_info'].get, intersecting_vars)):
-                    raise RuntimeError('Variable schemas are inconsistent between granules')
-                var_info.update(result['var_info'])
+            intersecting_vars = set(var_info).intersection(result['var_info'])
+            if any(var_info[var] != result['var_info'][var] for var in intersecting_vars):
+                raise RuntimeError('Variable schemas are inconsistent between granules')
+            var_info.update(result['var_info'])
 
         # The following data requires accumulation methods
         merge_max_dims(max_dims, result['max_dims'])