Added CWL examples for stage-in and stage-out

cwl/stage-in-daac/README.md

@@ -0,0 +1,7 @@
+# Test stage in data from DAAC
+
+Update `stac_json` value in stage-in-job-01.yml
+
+Run the following command with a CWL runner.
+
+`cwltool stage-in-workflow.cwl stage-in-job-01.yml`

cwl/stage-in-daac/stage-in-job-01.yml

@@ -0,0 +1,2 @@
+download_dir: "granules"
+stac_json: "https://cmr.earthdata.nasa.gov/stac/LPCLOUD/collections/EMITL1BRAD_001/items?limit=2"

cwl/stage-in-daac/stage-in-workflow.cwl

@@ -0,0 +1,31 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: Workflow
+label: Workflow that executes the Sounder SIPS end-to-end chirp rebinngin workflow
+
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+
+requirements:
+  SubworkflowFeatureRequirement: {}
+  StepInputExpressionRequirement: {}
+  InlineJavascriptRequirement: {}
+  NetworkAccess:
+    networkAccess: true
+
+inputs:
+  download_dir: string
+  stac_json: string
+
+outputs:
+  data:
+    type: Directory
+    outputSource: stage-in/download_dir
+
+steps:
+    stage-in:
+      run: "stage-in.cwl"
+      in:
+        stac_json: stac_json
+        download_dir: download_dir
+      out: [download_dir]

cwl/stage-in-daac/stage-in.cwl

@@ -0,0 +1,39 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+baseCommand: ["DOWNLOAD"]
+
+requirements:
+  DockerRequirement:
+    dockerPull: ghcr.io/unity-sds/unity-data-services:9.4.0
+  NetworkAccess:
+    networkAccess: true
+  EnvVarRequirement:
+    envDef:
+      DOWNLOAD_DIR: $(runtime.outdir)/$(inputs.download_dir)
+      STAC_JSON: $(inputs.stac_json)
+      LOG_LEVEL: '10'
+      PARALLEL_COUNT: '-1'
+      DOWNLOAD_RETRY_WAIT_TIME: '30'
+      DOWNLOAD_RETRY_TIMES: '5'
+      DOWNLOADING_ROLES: 'data'
+
+      EDL_BASE_URL: 'https://urs.earthdata.nasa.gov/'
+      EDL_USERNAME: '/sps/processing/workflows/edl_username'
+      EDL_PASSWORD: '/sps/processing/workflows/edl_password'
+      EDL_PASSWORD_TYPE: 'PARAM_STORE'
+
+      VERIFY_SSL: 'TRUE'
+      STAC_AUTH_TYPE: 'NONE'
+
+inputs:
+  download_dir:
+    type: string
+  stac_json:
+    type: string
+
+outputs:
+  download_dir:
+    type: Directory
+    outputBinding:
+      glob: "$(inputs.download_dir)"

cwl/stage-in-json-str/README.md

@@ -0,0 +1,7 @@
+# Test stage in data from DAAC
+
+Update `stac_json` value in stage-in-job-01.yml
+
+Run the following command with a CWL runner.
+
+`cwltool stage-in-workflow.cwl stage-in-job-01.yml`

cwl/stage-in-json-str/stage-in-job-01.yml

@@ -0,0 +1,2 @@
+download_dir: "granules"
+stac_json: '{"type":"FeatureCollection","stac_version":"1.0.0","numberMatched":2,"numberReturned":2,"features":[{"properties":{"datetime":"2016-08-22T00:05:22.000Z","start_datetime":"2016-08-22T00:05:22.000Z","end_datetime":"2016-08-22T00:11:22.000Z"},"bbox":[-7.02,-60.32,26.31,-36.16],"assets":{"data":{"title":"Download SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc","href":"https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc"}},"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[6.18,-36.16],[-7.02,-56.04],[23.24,-60.32],[26.31,-38.94],[6.18,-36.16]]]},"stac_extensions":[],"id":"G2040068613-GES_DISC","stac_version":"1.0.0","collection":"C2011289787-GES_DISC","links":[{"rel":"self","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.stac"},{"rel":"parent","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"},{"rel":"collection","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"},{"rel":"root","href":"https://cmr.earthdata.nasa.gov:443/search/"},{"rel":"via","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.json"},{"rel":"via","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.umm_json"}]},{"properties":{"datetime":"2016-08-22T00:11:22.000Z","start_datetime":"2016-08-22T00:11:22.000Z","end_datetime":"2016-08-22T00:17:22.000Z"},"bbox":[-43.78,-81.77028018298317,23.22,-56.18],"assets":{"data":{"title":"Download SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc","href":"https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc"}},"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-7.16,-56.18],[-43.78,-71.72],[20.73,-81.77],[23.22,-60.47],[-7.16,-56.18]]]},"stac_extensions":[],"id":"G2040068619-GES_DISC","stac_version":"1.0.0","collection":"C2011289787-GES_DISC","links":[{"rel":"self","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.stac"},{"rel":"parent","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"},{"rel":"collection","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"},{"rel":"root","href":"https://cmr.earthdata.nasa.gov:443/search/"},{"rel":"via","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.json"},{"rel":"via","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.umm_json"}]}],"links":[{"rel":"self","href":"https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&temporal%5B%5D=2016-08-22T00%3A10%3A00%2C2016-08-22T00%3A15%3A00&page_num=1"},{"rel":"root","href":"https://cmr.earthdata.nasa.gov:443/search/"}],"context":{"returned":2,"limit":1000000,"matched":2}}'

cwl/stage-in-json-str/stage-in-workflow.cwl

@@ -0,0 +1,31 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: Workflow
+label: Workflow that executes the Sounder SIPS end-to-end chirp rebinngin workflow
+
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+
+requirements:
+  SubworkflowFeatureRequirement: {}
+  StepInputExpressionRequirement: {}
+  InlineJavascriptRequirement: {}
+  NetworkAccess:
+    networkAccess: true
+
+inputs:
+  download_dir: string
+  stac_json: string
+
+outputs:
+  data:
+    type: Directory
+    outputSource: stage-in/download_dir
+
+steps:
+    stage-in:
+      run: "stage-in.cwl"
+      in:
+        stac_json: stac_json
+        download_dir: download_dir
+      out: [download_dir]

cwl/stage-in-json-str/stage-in.cwl

@@ -0,0 +1,26 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+baseCommand: ["DOWNLOAD"]
+
+requirements:
+  DockerRequirement:
+    dockerPull: ghcr.io/unity-sds/unity-data-services:9.4.0
+  NetworkAccess:
+    networkAccess: true
+  EnvVarRequirement:
+    envDef:
+      DOWNLOAD_DIR: $(runtime.outdir)/$(inputs.download_dir)
+      STAC_JSON: $(inputs.stac_json)
+
+inputs:
+  download_dir:
+    type: string
+  stac_json:
+    type: string
+
+outputs:
+  download_dir:
+    type: Directory
+    outputBinding:
+      glob: "$(inputs.download_dir)"

cwl/stage-in-unity/README.md

@@ -0,0 +1,7 @@
+# Test stage in data from MDPS
+
+Update `stac_json` and `unity_client_id` values in stage-in-job-01.yml
+
+Run the following command with a CWL runner.
+
+`cwltool stage-in-workflow.cwl stage-in-job-01.yml`

cwl/stage-in-unity/stage-in-job-01.yml

@@ -0,0 +1,3 @@
+download_dir: "granules"
+stac_json: "https://d3vc8w9zcq658.cloudfront.net/am-uds-dapa/collections/urn:nasa:unity:unity:dev:SBG-L2A_RSRFL___1/items"
+unity_client_id: "40c2s0ulbhp9i0fmaph3su9jch"

cwl/stage-in-unity/stage-in-workflow.cwl

@@ -0,0 +1,33 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: Workflow
+label: Workflow that executes the Sounder SIPS end-to-end chirp rebinngin workflow
+
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+
+requirements:
+  SubworkflowFeatureRequirement: {}
+  StepInputExpressionRequirement: {}
+  InlineJavascriptRequirement: {}
+  NetworkAccess:
+    networkAccess: true
+
+inputs:
+  download_dir: string
+  stac_json: string
+  unity_client_id: string
+
+outputs:
+  data:
+    type: Directory
+    outputSource: stage-in/download_dir
+
+steps:
+    stage-in:
+      run: "stage-in.cwl"
+      in:
+        stac_json: stac_json
+        download_dir: download_dir
+        unity_client_id: unity_client_id
+      out: [download_dir]

cwl/stage-in-unity/stage-in.cwl

@@ -0,0 +1,45 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+baseCommand: ["DOWNLOAD"]
+
+requirements:
+  DockerRequirement:
+    dockerPull: ghcr.io/unity-sds/unity-data-services:9.4.0
+  NetworkAccess:
+    networkAccess: true
+  EnvVarRequirement:
+    envDef:
+      DOWNLOAD_DIR: $(runtime.outdir)/$(inputs.download_dir)
+      STAC_JSON: $(inputs.stac_json)
+      LOG_LEVEL: '10'
+      PARALLEL_COUNT: '-1'
+      DOWNLOAD_RETRY_WAIT_TIME: '30'
+      DOWNLOAD_RETRY_TIMES: '5'
+      DOWNLOADING_ROLES: 'data'
+
+      CLIENT_ID: $(inputs.unity_client_id)
+      COGNITO_URL: $(inputs.unity_cognito_url)
+      USERNAME: '/sps/processing/workflows/unity_username'
+      PASSWORD: '/sps/processing/workflows/unity_password'
+      PASSWORD_TYPE: 'PARAM_STORE'
+
+      VERIFY_SSL: 'TRUE'
+      STAC_AUTH_TYPE: 'UNITY'
+
+inputs:
+  download_dir:
+    type: string
+  stac_json:
+    type: string
+  unity_client_id:
+    type: string
+  unity_cognito_url:
+    type: string
+    default: https://cognito-idp.us-west-2.amazonaws.com
+
+outputs:
+  download_dir:
+    type: Directory
+    outputBinding:
+      glob: "$(inputs.download_dir)"

cwl/stage-out-auxiliary/README.md

@@ -0,0 +1,7 @@
+# Test stage out and catalog sample auxiliary data
+
+Update `collection_id` and `staging_bucket` values in stage-out-job-01.yml
+
+Run the following command with a CWL runner.
+
+`cwltool stage-out-workflow.cwl stage-out-job-01.yml`

cwl/stage-out-auxiliary/stage-out-job-01.yml

@@ -0,0 +1,5 @@
+collection_id: "urn:nasa:unity:unity:dev:unity-dev-auxiliary___1"
+staging_bucket: "unity-dev-unity-unity-data"
+sample_output_data:
+  class: Directory
+  path: outputs

cwl/stage-out-auxiliary/stage-out-workflow.cwl

@@ -0,0 +1,34 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: Workflow
+label: Workflow that executes the Sounder SIPS end-to-end chirp rebinngin workflow
+
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+
+requirements:
+  SubworkflowFeatureRequirement: {}
+  StepInputExpressionRequirement: {}
+  InlineJavascriptRequirement: {}
+
+inputs:
+  collection_id: string
+  staging_bucket: string
+  sample_output_data: Directory
+
+outputs:
+  successful_features:
+    type: File
+    outputSource: stage-out/successful_features
+  failed_features:
+    type: File
+    outputSource: stage-out/failed_features
+
+steps:
+    stage-out:
+      run: "stage-out.cwl"
+      in:
+        collection_id: collection_id
+        staging_bucket: staging_bucket
+        sample_output_data: sample_output_data
+      out: [successful_features, failed_features]

cwl/stage-out-auxiliary/stage-out.cwl

@@ -0,0 +1,44 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+baseCommand: ["UPLOAD"]
+
+requirements:
+  DockerRequirement:
+    dockerPull: ghcr.io/unity-sds/unity-data-services:9.4.0
+  NetworkAccess:
+    networkAccess: true
+  InitialWorkDirRequirement:
+    listing:
+    - entry: $(inputs.sample_output_data)
+      entryname: /tmp/outputs
+
+  EnvVarRequirement:
+    envDef:
+      COLLECTION_ID: $(inputs.collection_id)
+      STAGING_BUCKET: $(inputs.staging_bucket)
+      LOG_LEVEL: '10'
+      PARALLEL_COUNT: '-1'
+      OUTPUT_DIRECTORY: $(runtime.outdir)
+      RESULT_PATH_PREFIX: 'stage_out/$(inputs.collection_id)'
+      GRANULES_UPLOAD_TYPE: 'UPLOAD_AUXILIARY_FILE_AS_GRANULE'
+      BASE_DIRECTORY: '/tmp/outputs'
+
+inputs:
+  collection_id:
+    type: string
+  staging_bucket:
+    type: string
+  sample_output_data:
+    type: Directory
+
+outputs:
+  successful_features:
+    type: File
+    outputBinding:
+      glob: "$(runtime.outdir)/successful_features.json"
+  failed_features:
+    type: File
+    outputBinding:
+      glob: "$(runtime.outdir)/failed_features.json"
+

cwl/stage-out-stac-catalog/README.md

@@ -0,0 +1,7 @@
+# Test stage out and catalog sample output data
+
+Update `collection_id` `staging_bucket` and `sample_output_data` values in stage-out-job-01.yml
+
+Run the following command with a CWL runner.
+
+`cwltool stage-out-workflow.cwl stage-out-job-01.yml`

cwl/stage-out-stac-catalog/outputs/catalog.json

@@ -0,0 +1,24 @@
+{
+  "type": "Catalog",
+  "id": "NA",
+  "stac_version": "1.0.0",
+  "description": "NA",
+  "links": [
+    {
+      "rel": "root",
+      "href": "./catalog.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "item",
+      "href": "./output_1.nc.stac.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "item",
+      "href": "./output_2.nc.stac.json",
+      "type": "application/json"
+    }
+  ],
+  "stac_extensions": []
+}