diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 92935a0..fbd6cdf 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -1,4 +1,4 @@ -name: Deploy modules to prod project +name: Deploy modules to test project on: push: branches: @@ -19,13 +19,15 @@ jobs: CDF_PROJECT: ${{ vars.CDF_PROJECT }} IDP_CLIENT_ID: ${{ vars.IDP_CLIENT_ID }} IDP_TOKEN_URL: ${{ vars.IDP_TOKEN_URL }} - DATA_DEVELOPER_OBJECT_ID: ${{ vars.DATA_DEVELOPER_OBJECT_ID }} - DATA_PIPELINE_OEE_OBJECT_ID: ${{ vars.DATA_PIPELINE_OEE_OBJECT_ID }} - ICAPI_EXTRACTORS_OBJECT_ID: ${{ vars.ICAPI_EXTRACTORS_OBJECT_ID }} DATA_PIPELINE_OEE_CLIENT_ID: ${{ vars.DATA_PIPELINE_OEE_CLIENT_ID }} ICAPI_EXTRACTORS_CLIENT_ID: ${{ vars.ICAPI_EXTRACTORS_CLIENT_ID }} CDF_CLUSTER: ${{ vars.CDF_CLUSTER }} IDP_TENANT_ID: ${{ secrets.IDP_TENANT_ID }} + + data_developer_source_id: ${{ vars.DATA_DEVELOPER_OBJECT_ID }} + data_pipeline_oee_source_id: ${{ vars.DATA_PIPELINE_OEE_OBJECT_ID }} + icapi_extractors_source_id: ${{ vars.ICAPI_EXTRACTORS_OBJECT_ID }} + IDP_CLIENT_SECRET: ${{ secrets.IDP_CLIENT_SECRET }} DATA_PIPELINE_OEE_CLIENT_SECRET: ${{ secrets.DATA_PIPELINE_OEE_CLIENT_SECRET }} ICAPI_EXTRACTORS_CLIENT_SECRET: ${{ secrets.ICAPI_EXTRACTORS_CLIENT_SECRET }} diff --git a/ice-cream-dataops/build_info.test.yaml b/ice-cream-dataops/build_info.test.yaml deleted file mode 100644 index ea7b3ee..0000000 --- a/ice-cream-dataops/build_info.test.yaml +++ /dev/null @@ -1,435 +0,0 @@ -# DO NOT MODIFY THIS FILE MANUALLY. IT IS AUTO-GENERATED BY THE COGNITE TOOLKIT. -modules: - version: 0.3.2 - modules: - - name: data_foundation - location: - path: modules/bootcamp/data_foundation - hash: f3f72fe1 - build_variables: - - key: CDF_PROJECT - value: ${CDF_PROJECT} - is_selected: true - location: modules/bootcamp - - key: IDP_CLIENT_ID - value: ${IDP_CLIENT_ID} - is_selected: true - location: modules/bootcamp - - key: IDP_CLIENT_SECRET - value: ${IDP_CLIENT_SECRET} - is_selected: true - location: modules/bootcamp - - key: IDP_TENANT_ID - value: ${IDP_TENANT_ID} - is_selected: true - location: modules/bootcamp - - key: CDF_URL - value: https://westeurope-1.cognitedata.com - is_selected: true - location: modules/bootcamp - - key: IDP_SCOPES - value: https://westeurope-1.cognitedata.com/.default - is_selected: true - location: modules/bootcamp - - key: IDP_TOKEN_URL - value: ${IDP_TOKEN_URL} - is_selected: true - location: modules/bootcamp - - key: data_developer_source_id - value: ${DATA_DEVELOPER_OBJECT_ID} - is_selected: true - location: modules/bootcamp - - key: data_pipeline_oee_source_id - value: ${DATA_PIPELINE_OEE_OBJECT_ID} - is_selected: true - location: modules/bootcamp - - key: readwrite_source_id - value: a9bb9da4-5416-4390-8fd7-ccbf24af4b3d - is_selected: true - location: modules/bootcamp - - key: readonly_source_id - value: a9bb9da4-5416-4390-8fd7-ccbf24af4b3d - is_selected: true - location: modules/bootcamp - - key: icapi_extractors_source_id - value: ${ICAPI_EXTRACTORS_OBJECT_ID} - is_selected: true - location: modules/bootcamp - - key: data_pipeline_oee_client_id - value: ${DATA_PIPELINE_OEE_CLIENT_ID} - is_selected: true - location: modules/bootcamp - - key: data_pipeline_oee_client_secret - value: ${DATA_PIPELINE_OEE_CLIENT_SECRET} - is_selected: true - location: modules/bootcamp - - key: icapi_extractors_client_id - value: ${ICAPI_EXTRACTORS_CLIENT_ID} - is_selected: true - location: modules/bootcamp - - key: icapi_extractors_client_secret - value: ${ICAPI_EXTRACTORS_CLIENT_SECRET} - is_selected: true - location: modules/bootcamp - - key: icapi_ds_external_id - value: ds_icapi - is_selected: true - location: modules/bootcamp - - key: uc_oee_ds_external_id - value: ds_uc_oee - is_selected: true - location: modules/bootcamp - resources: - auth: - - identifier: - name: gp_admin_readonly - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/data_foundation/auth/admin.readonly.group.yaml - hash: e0b16472 - kind: Group - - identifier: - name: gp_admin_read_write - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/data_foundation/auth/admin.readwrite.group.yaml - hash: 5d76eb9d - kind: Group - - identifier: - name: data_developer - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/data_foundation/auth/data_developer.yaml - hash: 4da5fb8a - kind: Group - warning_count: 0 - status: Success - - name: ice_cream_api - location: - path: modules/bootcamp/ice_cream_api - hash: b1381b08 - build_variables: - - key: CDF_PROJECT - value: ${CDF_PROJECT} - is_selected: true - location: modules/bootcamp - - key: IDP_CLIENT_ID - value: ${IDP_CLIENT_ID} - is_selected: true - location: modules/bootcamp - - key: IDP_CLIENT_SECRET - value: ${IDP_CLIENT_SECRET} - is_selected: true - location: modules/bootcamp - - key: IDP_TENANT_ID - value: ${IDP_TENANT_ID} - is_selected: true - location: modules/bootcamp - - key: CDF_URL - value: https://westeurope-1.cognitedata.com - is_selected: true - location: modules/bootcamp - - key: IDP_SCOPES - value: https://westeurope-1.cognitedata.com/.default - is_selected: true - location: modules/bootcamp - - key: IDP_TOKEN_URL - value: ${IDP_TOKEN_URL} - is_selected: true - location: modules/bootcamp - - key: data_developer_source_id - value: ${DATA_DEVELOPER_OBJECT_ID} - is_selected: true - location: modules/bootcamp - - key: data_pipeline_oee_source_id - value: ${DATA_PIPELINE_OEE_OBJECT_ID} - is_selected: true - location: modules/bootcamp - - key: readwrite_source_id - value: a9bb9da4-5416-4390-8fd7-ccbf24af4b3d - is_selected: true - location: modules/bootcamp - - key: readonly_source_id - value: a9bb9da4-5416-4390-8fd7-ccbf24af4b3d - is_selected: true - location: modules/bootcamp - - key: icapi_extractors_source_id - value: ${ICAPI_EXTRACTORS_OBJECT_ID} - is_selected: true - location: modules/bootcamp - - key: data_pipeline_oee_client_id - value: ${DATA_PIPELINE_OEE_CLIENT_ID} - is_selected: true - location: modules/bootcamp - - key: data_pipeline_oee_client_secret - value: ${DATA_PIPELINE_OEE_CLIENT_SECRET} - is_selected: true - location: modules/bootcamp - - key: icapi_extractors_client_id - value: ${ICAPI_EXTRACTORS_CLIENT_ID} - is_selected: true - location: modules/bootcamp - - key: icapi_extractors_client_secret - value: ${ICAPI_EXTRACTORS_CLIENT_SECRET} - is_selected: true - location: modules/bootcamp - - key: icapi_ds_external_id - value: ds_icapi - is_selected: true - location: modules/bootcamp - - key: uc_oee_ds_external_id - value: ds_uc_oee - is_selected: true - location: modules/bootcamp - resources: - extraction_pipelines: - - identifier: - externalId: ep_icapi_assets - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/extraction_pipelines/ep_icapi_assets.yaml - hash: 5f0dd030 - kind: ExtractionPipeline - - identifier: - externalId: ep_icapi_datapoints - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/extraction_pipelines/ep_icapi_datapoints.yaml - hash: 81abe813 - kind: ExtractionPipeline - - identifier: - externalId: ep_icapi_timeseries - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/extraction_pipelines/ep_icapi_timeseries.yaml - hash: 0dd35906 - kind: ExtractionPipeline - auth: - - identifier: - name: icapi_extractor - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/auth/icapi_extractors.yaml - hash: abe83767 - kind: Group - functions: - - identifier: - externalId: icapi_assets_extractor - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/functions/functions.yaml - hash: 7a3f72cf - kind: Function - - identifier: - externalId: icapi_datapoints_extractor - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/functions/functions.yaml - hash: 7a3f72cf - kind: Function - - identifier: - externalId: icapi_timeseries_extractor - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/functions/functions.yaml - hash: 7a3f72cf - kind: Function - - identifier: - functionExternalId: icapi_datapoints_extractor - name: Frontfill every 10 minutes last hour of data (streamer) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/functions/schedules.yaml - hash: a151e5eb - kind: Schedule - - identifier: - functionExternalId: icapi_datapoints_extractor - name: Backfill once a day for the last 30 days (gap filling) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/functions/schedules.yaml - hash: a151e5eb - kind: Schedule - data_sets: - - identifier: - externalId: ds_icapi - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/data_sets/icapi.DataSet.yaml - hash: 252ee3a7 - kind: DataSet - raw: - - identifier: - dbName: State Store - tableName: icapi_datapoints_extractor - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/raw/extraction_sources.Table.yaml - hash: fe459b6c - kind: Table - - identifier: - dbName: ice_cream_api - tableName: icapi_assets_extractor - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/ice_cream_api/raw/extraction_sources.Table.yaml - hash: fe459b6c - kind: Table - warning_count: 0 - status: Success - - name: oee - location: - path: modules/bootcamp/use_cases/oee - hash: 06d09ad5 - build_variables: - - key: CDF_PROJECT - value: ${CDF_PROJECT} - is_selected: true - location: modules/bootcamp - - key: IDP_CLIENT_ID - value: ${IDP_CLIENT_ID} - is_selected: true - location: modules/bootcamp - - key: IDP_CLIENT_SECRET - value: ${IDP_CLIENT_SECRET} - is_selected: true - location: modules/bootcamp - - key: IDP_TENANT_ID - value: ${IDP_TENANT_ID} - is_selected: true - location: modules/bootcamp - - key: CDF_URL - value: https://westeurope-1.cognitedata.com - is_selected: true - location: modules/bootcamp - - key: IDP_SCOPES - value: https://westeurope-1.cognitedata.com/.default - is_selected: true - location: modules/bootcamp - - key: IDP_TOKEN_URL - value: ${IDP_TOKEN_URL} - is_selected: true - location: modules/bootcamp - - key: data_developer_source_id - value: ${DATA_DEVELOPER_OBJECT_ID} - is_selected: true - location: modules/bootcamp - - key: data_pipeline_oee_source_id - value: ${DATA_PIPELINE_OEE_OBJECT_ID} - is_selected: true - location: modules/bootcamp - - key: readwrite_source_id - value: a9bb9da4-5416-4390-8fd7-ccbf24af4b3d - is_selected: true - location: modules/bootcamp - - key: readonly_source_id - value: a9bb9da4-5416-4390-8fd7-ccbf24af4b3d - is_selected: true - location: modules/bootcamp - - key: icapi_extractors_source_id - value: ${ICAPI_EXTRACTORS_OBJECT_ID} - is_selected: true - location: modules/bootcamp - - key: data_pipeline_oee_client_id - value: ${DATA_PIPELINE_OEE_CLIENT_ID} - is_selected: true - location: modules/bootcamp - - key: data_pipeline_oee_client_secret - value: ${DATA_PIPELINE_OEE_CLIENT_SECRET} - is_selected: true - location: modules/bootcamp - - key: icapi_extractors_client_id - value: ${ICAPI_EXTRACTORS_CLIENT_ID} - is_selected: true - location: modules/bootcamp - - key: icapi_extractors_client_secret - value: ${ICAPI_EXTRACTORS_CLIENT_SECRET} - is_selected: true - location: modules/bootcamp - - key: icapi_ds_external_id - value: ds_icapi - is_selected: true - location: modules/bootcamp - - key: uc_oee_ds_external_id - value: ds_uc_oee - is_selected: true - location: modules/bootcamp - resources: - auth: - - identifier: - name: oee_pipeline - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/auth/data_pipeline_oee.yaml - hash: 1b5b11c6 - kind: Group - functions: - - identifier: - externalId: oee_timeseries - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/functions.yaml - hash: 3579da40 - kind: Function - - identifier: - functionExternalId: oee_timeseries - name: Run calculations every 10 minutes for last hour of data - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - - identifier: - functionExternalId: oee_timeseries - name: Run calculations once a day for 30 days history (Oslo) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - - identifier: - functionExternalId: oee_timeseries - name: Run calculations once a day for 30 days history (Nuremberg) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - - identifier: - functionExternalId: oee_timeseries - name: Run calculations once a day for 30 days history (Marseille) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - - identifier: - functionExternalId: oee_timeseries - name: Run calculations once a day for 30 days history (Houston) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - - identifier: - functionExternalId: oee_timeseries - name: Run calculations once a day for 30 days history (Sao Paulo) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - - identifier: - functionExternalId: oee_timeseries - name: Run calculations once a day for 30 days history (Kuala Lumpur) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - - identifier: - functionExternalId: oee_timeseries - name: Run calculations once a day for 30 days history (Chicago) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - - identifier: - functionExternalId: oee_timeseries - name: Run calculations once a day for 30 days history (Rotterdam) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - - identifier: - functionExternalId: oee_timeseries - name: Run calculations once a day for 30 days history (London) - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/functions/schedules.yaml - hash: 1dd72b15 - kind: Schedule - data_sets: - - identifier: - externalId: ds_uc_oee - source: - path: /Users/fredrik.larsen@cognitedata.com/Bootcamp/ice-cream-dataops/modules/bootcamp/use_cases/oee/data_sets/data_sets.yaml - hash: 3d953d35 - kind: DataSet - warning_count: 0 - status: Success diff --git a/ice-cream-dataops/config.test.yaml b/ice-cream-dataops/config.test.yaml index eb77c6f..d149b35 100644 --- a/ice-cream-dataops/config.test.yaml +++ b/ice-cream-dataops/config.test.yaml @@ -14,21 +14,19 @@ variables: IDP_CLIENT_ID: ${IDP_CLIENT_ID} IDP_CLIENT_SECRET: ${IDP_CLIENT_SECRET} IDP_TENANT_ID: ${IDP_TENANT_ID} + # This is related to the cluster where the CDF project is hosted. CDF_URL: https://westeurope-1.cognitedata.com IDP_SCOPES: https://westeurope-1.cognitedata.com/.default IDP_TOKEN_URL: ${IDP_TOKEN_URL} - # Groups - # OBJECT ID FOR TEST/PROD DATA DEVELOPER ENTRA GROUP + + # Groups: OBJECT ID FOR TEST/PROD DATA ENTRA GROUP data_developer_source_id: ${DATA_DEVELOPER_OBJECT_ID} - # OBJECT ID FOR TEST/PROD DATA PIPELINE OEE ENTRA GROUP data_pipeline_oee_source_id: ${DATA_PIPELINE_OEE_OBJECT_ID} - # OBJECT ID FOR TEST/PROD ADMIN TK ENTRA GROUP readwrite_source_id: a9bb9da4-5416-4390-8fd7-ccbf24af4b3d - # OBJECT ID FOR TEST/PROD ADMIN TK ENTRA GROUP readonly_source_id: a9bb9da4-5416-4390-8fd7-ccbf24af4b3d - # OBJECT ID FOR TEST/PROD ADMIN TK ENTRA GROUP icapi_extractors_source_id: ${ICAPI_EXTRACTORS_OBJECT_ID} + # Client Ids data_pipeline_oee_client_id: ${DATA_PIPELINE_OEE_CLIENT_ID} data_pipeline_oee_client_secret: ${DATA_PIPELINE_OEE_CLIENT_SECRET}