Extract Assets replication into Flex template

.github/workflows/pr.yaml

@@ -2,9 +2,16 @@ name: Java CI
 
 on:
   - push
-
+env:
+  PROJECT_ID: cognite-inso-sandbox
+
 jobs:
   build:
+    strategy:
+      matrix:
+        module:
+          - replicate-assets
+
     runs-on: ubuntu-latest
 
     steps:
@@ -16,4 +23,24 @@ jobs:
           distribution: 'adopt'
 
       - name: Build with Maven
-        run: mvn --batch-mode --update-snapshots verify
+        run: mvn --batch-mode --update-snapshots verify
+
+      - name: Login with specific service account
+        uses: GoogleCloudPlatform/github-actions/[email protected]
+        with:
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+          export_default_credentials: true
+
+      - name: Build and push image to Google Container Repository
+        env:
+          TARGET_GCR_IMAGE: eu.gcr.io/${{ env.PROJECT_ID }}/${{ matrix.module }}
+          BASE_CONTAINER_IMAGE: gcr.io/dataflow-templates-base/java11-template-launcher-base
+          BASE_CONTAINER_IMAGE_VERSION: latest
+          APP_ROOT: /template/${{ matrix.module }}
+        run: mvn clean package -Dimage=${TARGET_GCR_IMAGE} \
+          -Dbase-container-image=${BASE_CONTAINER_IMAGE} \
+          -Dbase-container-image.version=${BASE_CONTAINER_IMAGE_VERSION} \
+          -Dapp-root=${APP_ROOT} \
+          -Dcommand-spec=${APP_ROOT}/resources/${{ matrix.module }}-command-spec.json \
+          -am -pl templates/${{ matrix.module }}

build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+mvn clean package -Dimage=eu.gcr.io/cognitedata-development/replicate-assets \
+                  -Dbase-container-image=gcr.io/dataflow-templates-base/java11-template-launcher-base \
+                  -Dbase-container-image.version=latest \
+                  -Dapp-root=/template/replicate-assets \
+                  -Dcommand-spec=/template/replicate-assets/resources/replicate-assets-command-spec.json \
+                  -am -pl templates/replicate-assets

legacy/pom.xml

@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.cognite.data</groupId>
+    <artifactId>beam-components-legacy</artifactId>
+    <version>0.2.0-SNAPSHOT</version>
+
+    <properties>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <java.version>11</java.version>
+
+        <beam.version>2.28.0</beam.version>
+        <beam.connector.version>0.9.20-SNAPSHOT</beam.connector.version>
+        <junit.version>5.7.0</junit.version>
+        <logback.classic.version>1.2.3</logback.classic.version>
+        <maven-jar-plugin.version>3.0.0</maven-jar-plugin.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.cognite</groupId>
+            <artifactId>beam-connector-java</artifactId>
+            <version>${beam.connector.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>beam-sdks-java-core</artifactId>
+            <version>${beam.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
+            <version>${beam.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>beam-runners-direct-java</artifactId>
+            <version>${beam.version}</version>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
+            <version>${beam.version}</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.beam</groupId>
+            <artifactId>beam-sdks-java-extensions-sql</artifactId>
+            <version>${beam.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>${logback.classic.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <version>3.0.0-M4</version>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+</project>

src/main/java/com/cognite/sa/beam/bq/CdfTsPointsBQ.java → legacy/src/main/java/com/cognite/sa/beam/bq/CdfTsPointsBQ.java

@@ -37,12 +37,10 @@
 import org.apache.beam.sdk.options.*;
 import org.apache.beam.sdk.transforms.*;
 import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.TypeDescriptor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.time.Duration;
 import java.time.Instant;
 import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoUnit;

src/main/java/com/cognite/sa/beam/replicate/ReplicateRelationships.java → legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateRelationships.java

@@ -26,7 +26,6 @@
 import com.cognite.client.dto.Relationship;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.Int64Value;
-import com.google.protobuf.StringValue;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.metrics.Counter;
@@ -41,7 +40,6 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;

legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateAssets-Dataflow_template.ps1

@@ -0,0 +1,10 @@
+. $PSScriptRoot'\..\..\variables.ps1'
+. $PSScriptRoot'\..\base-config.ps1'
+$postfix = Get-Postfix
+$postfix = ''
+# Default variable values are picked up from the above files.
+# You can override them with local variable definitions in this file.
+#
+#-------------------------------------------------------------------
+
+mvn compile exec:java -D exec.mainClass=com.cognite.sa.beam.ReplicateAssets -D exec.args="--project=$gcpProject --runner=DataFlowRunner --gcpTempLocation=$gcpBucketPrefix$postfix/temp --stagingLocation=$gcpBucketPrefix$postfix/template-stage/replicate/replicate-assets --templateLocation=$gcpBucketPrefix$postfix/template/replicate/replicate-assets --region=europe-west1 --experiments=shuffle_mode=service --numWorkers=1 --maxNumWorkers=2 --experiments=enable_stackdriver_agent_metrics --workerMachineType=e2-standard-2"

src/scripts/ps1/replicate/RunReplicateAssets-Dataflow.ps1 → legacy/src/scripts/ps1/replicate/RunReplicateAssets-Dataflow.ps1

@@ -35,7 +35,7 @@
 #---------------------------------------------------------------------
 $jobConfigFile = $gcpBucketPrefix + '-test/config/var/job-config-assets-replication.toml'
 
-mvn compile exec:java -D exec.mainClass=com.cognite.sa.beam.replicate.ReplicateAssets -D exec.args="--cdfInputSecret=$cdfInputSecret --cdfInputHost=$cdfInputHost --cdfOutputSecret=$cdfOutputSecret --cdfOutputHost=$cdfOutputHost --jobConfigFile=$jobConfigFile --project=$gcpProject --runner=DataFlowRunner --gcpTempLocation=$gcpBucketPrefix-test/temp --stagingLocation=$gcpBucketPrefix-test/stage/replicate/replicate-assets --region=europe-west1 --experiments=shuffle_mode=service --numWorkers=1 --maxNumWorkers=2 --experiments=enable_stackdriver_agent_metrics,enable_execution_details_collection,use_monitoring_state_manager --workerMachineType=e2-standard-2"
+mvn compile exec:java -D exec.mainClass=com.cognite.sa.beam.ReplicateAssets -D exec.args="--cdfInputSecret=$cdfInputSecret --cdfInputHost=$cdfInputHost --cdfOutputSecret=$cdfOutputSecret --cdfOutputHost=$cdfOutputHost --jobConfigFile=$jobConfigFile --project=$gcpProject --runner=DataFlowRunner --gcpTempLocation=$gcpBucketPrefix-test/temp --stagingLocation=$gcpBucketPrefix-test/stage/replicate/replicate-assets --region=europe-west1 --experiments=shuffle_mode=service --numWorkers=1 --maxNumWorkers=2 --experiments=enable_stackdriver_agent_metrics,enable_execution_details_collection,use_monitoring_state_manager --workerMachineType=e2-standard-2"
 
 # In order to add profiling:
 # --profilingAgentConfiguration='{\""APICurated\"" : true}'

pom.xml

@@ -1,88 +1,14 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
-
     <groupId>com.cognite.data</groupId>
     <artifactId>beam-components</artifactId>
     <version>0.2.0-SNAPSHOT</version>
+    <packaging>pom</packaging>
+    <name>beam-components</name>
 
-    <properties>
-        <maven.compiler.source>11</maven.compiler.source>
-        <maven.compiler.target>11</maven.compiler.target>
-        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <java.version>11</java.version>
-
-        <beam.version>2.28.0</beam.version>
-        <beam.connector.version>0.9.19</beam.connector.version>
-        <junit.version>5.7.0</junit.version>
-        <logback.classic.version>1.2.3</logback.classic.version>
-        <maven-jar-plugin.version>3.0.0</maven-jar-plugin.version>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.cognite</groupId>
-            <artifactId>beam-connector-java</artifactId>
-            <version>${beam.connector.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.beam</groupId>
-            <artifactId>beam-sdks-java-core</artifactId>
-            <version>${beam.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.beam</groupId>
-            <artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
-            <version>${beam.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.beam</groupId>
-            <artifactId>beam-runners-direct-java</artifactId>
-            <version>${beam.version}</version>
-            <scope>runtime</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.beam</groupId>
-            <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
-            <version>${beam.version}</version>
-            <scope>compile</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.beam</groupId>
-            <artifactId>beam-sdks-java-extensions-sql</artifactId>
-            <version>${beam.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter-api</artifactId>
-            <version>${junit.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter-engine</artifactId>
-            <version>${junit.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-classic</artifactId>
-            <version>${logback.classic.version}</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-surefire-plugin</artifactId>
-                    <version>3.0.0-M4</version>
-                </plugin>
-            </plugins>
-        </pluginManagement>
-    </build>
+    <modules>
+        <module>templates</module>
+        <module>legacy</module>
+    </modules>
 </project>