Merge pull request #4 from dagster-io/benpankow/docker-target-stage

Allowed specifying Docker target stage in locations.yaml

.github/workflows/example-docker-target-stage.yml

@@ -0,0 +1,33 @@
+name: Docker Target Stage Example
+on: push
+
+jobs:
+  docker-target-stage-example:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v1
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Login to ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Cache Docker builds
+        uses: satackey/[email protected]
+        continue-on-error: true
+
+      - name: Run Dagster Cloud CI/CD action
+        # Here, use `uses: dagster-io/dagster-cloud-cicd-action@{version}`
+        # e.g. `uses: dagster-io/[email protected]`
+        uses: ./
+        with:
+          location-file: ./example/docker-target-stage/locations.yaml
+          dagit-url: https://elementl.dogfood.dagster.cloud/dev
+          api-token: ${{ secrets.DAGSTER_AGENT_TOKEN }}

README.md

@@ -70,13 +70,13 @@ locations:
     registry: dagster-io/foo
 
     # Python file containing the job repo
-    # Can alternatively supply python_module, as below
+    # Can alternatively supply package_name, as below
     python_file: repo.py
 
   bar:
     build: ./bar_src
     registry: dagster-io/bar
-    python_module: bar
+    package_name: bar
 ```
 
 ### More Examples
@@ -93,6 +93,18 @@ More examples are provided in the [`example` folder](./example).
 | `image-tag`                     | Tag for the built Docker images, defaults to the first 6 chars of git hash.                  |
 | `parallel`                      | Whether to build and push Docker images in parallel. Defaults to `true`.                     |
 
+### `locations.yaml` Properties
+Each location specified in the `locations.yaml` can have the following properties:
+
+| Name                            | Description                                                                                  |
+|---------------------------------|----------------------------------------------------------------------------------------------|
+| `build`                         | **(Required)** Path to the build directory relative to the `locations.yaml`'s folder. Build directory must contain a `Dockerfile` or `requirements.txt` file.|
+| `registry`                      | **(Required)** Docker registry to push the built Docker image to.                            |
+| `package_name`                  | Python module containing the [Dagster Repository](https://docs.dagster.io/concepts/repositories-workspaces/repositories). Can alternatively use `python_file` below.|
+| `python_file`                   | Python file containing the [Dagster Repository](https://docs.dagster.io/concepts/repositories-workspaces/repositories). Can alternatively use `package_name` above.|
+| `base_image`                    | If the build directory only contains a `requirements.txt` file and no `Dockerfile`, specifies the base Docker image to use to build the code location.|
+| `target`                        | If providing a multistage `Dockerfile`, can be used to specify the [target stage](https://docs.docker.com/develop/develop-images/multistage-build/) to build.|
+
 
 ## Developing the CI/CD Action
 

dist/index.js

@@ -21737,13 +21737,19 @@ async function run() {
 
         const imageName = `${location['registry']}:${imageTag}`;
 
+        let dockerArguments = [
+          'build', '.',
+          '--label', `sha=${github.context.sha}`,
+          '-f', dockerfile,
+          '-t', imageName
+        ];
+
+        if (location['target']) {
+          dockerArguments = dockerArguments.concat(['--target', location['target']]);
+        }
+
         await exec.exec('docker',
-          [
-            'build', '.',
-            '--label', `sha=${github.context.sha}`,
-            '-f', dockerfile,
-            '-t', imageName
-          ],
+          dockerArguments,
           options = {'cwd': buildPath}
         );
       });

example/README.md

@@ -6,4 +6,5 @@ This folder contains a number of examples of potential use cases of the Dagster
 - [`requirements-txt`](./requirements-txt): A lightweight example which builds from a `requirements.txt`
 file instead of a Dockerfile.
 - [`multiple-locations`](./multiple-locations): An example containing multiple code locations,
-which are built and pushed separately.
+which are built and pushed separately.
+- [`docker-target-stage`](./docker-target-stage): An example which specifies the [Docker target stage](https://docs.docker.com/develop/develop-images/multistage-build/) to build in a multistage Dockerfile.

example/docker-target-stage/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.8-slim AS first-stage
+
+ENV DAGSTER_VERSION=0.12.14
+
+RUN pip install \
+    dagster==${DAGSTER_VERSION} \
+    dagster-cloud==${DAGSTER_VERSION}
+
+WORKDIR /opt/dagster/app
+
+COPY repo.py /opt/dagster/app
+
+FROM python:3.8-slim AS second-stage

example/docker-target-stage/README.md

@@ -0,0 +1,6 @@
+# Docker Target Stage Example
+
+This example specifies a [Docker target stage](https://docs.docker.com/develop/develop-images/multistage-build/) as part of the image build process.
+
+The GitHub actions workflow for this example can be found at
+[`.github/workflows/example-docker-target-stage.yml`](../../.github/workflows/example-docker-target-stage.yml).

example/docker-target-stage/locations.yaml

@@ -0,0 +1,6 @@
+locations:
+  github_action_demo_target_stage:
+    build: .
+    target: first-stage
+    registry: 764506304434.dkr.ecr.us-west-2.amazonaws.com/github-action-demo-build-target
+    python_file: repo.py

example/docker-target-stage/repo.py

@@ -0,0 +1,27 @@
+from dagster import RunRequest, pipeline, repository, schedule, sensor, solid
+
+
+@solid()
+def foo_solid(_):
+    pass
+
+
+@solid()
+def foo_solid_2(_):
+    pass
+
+
+@pipeline
+def foo_pipeline():
+    foo_solid()
+    foo_solid_2()
+
+
+@pipeline
+def other_foo_pipeline():
+    foo_solid()
+
+
+@repository
+def foo_repo():
+    return [foo_pipeline]

src/index.js

@@ -81,13 +81,19 @@ async function run() {
 
         const imageName = `${location['registry']}:${imageTag}`;
 
+        let dockerArguments = [
+          'build', '.',
+          '--label', `sha=${github.context.sha}`,
+          '-f', dockerfile,
+          '-t', imageName
+        ];
+
+        if (location['target']) {
+          dockerArguments = dockerArguments.concat(['--target', location['target']]);
+        }
+
         await exec.exec('docker',
-          [
-            'build', '.',
-            '--label', `sha=${github.context.sha}`,
-            '-f', dockerfile,
-            '-t', imageName
-          ],
+          dockerArguments,
           options = {'cwd': buildPath}
         );
       });