Merge pull request #16 from dagster-io/funwithcontainercontext

Add container_context to github action

.github/workflows/example-container-context.yml

@@ -0,0 +1,33 @@
+name: Example with container context
+on: push
+
+jobs:
+  container-context-example:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v1
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Login to ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Cache Docker builds
+        uses: satackey/[email protected]
+        continue-on-error: true
+
+      - name: Run Dagster Cloud CI/CD action
+        # Here, use `uses: dagster-io/dagster-cloud-cicd-action/deploy@{version}`
+        # e.g. `uses: dagster-io/dagster-cloud-cicd-action/[email protected]`
+        uses: ./deploy
+        with:
+          location-file: ./example/container-context/locations.yaml
+          dagit-url: https://elementl.dogfood.dagster.cloud/dev
+          api-token: ${{ secrets.DAGSTER_AGENT_TOKEN }}

README.md

@@ -114,6 +114,12 @@ Each location specified in the `locations.yaml` can have the following propertie
 
 ## Developing the CI/CD Action
 
+The first time developing against the action, run:
+
+```sh
+npm install
+```
+
 The CI/CD action is run from the packaged files in the `dist/*` folder. When making a change, be sure to repackage the files:
 
 ```sh

example/container-context/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.8-slim
+
+RUN pip install \
+    dagster \
+    dagster-cloud
+
+WORKDIR /opt/dagster/app
+
+COPY repo.py /opt/dagster/app

example/container-context/README.md

@@ -0,0 +1,8 @@
+# Minimal Example
+
+This is a minimal example of a repo set up to use the CI/CD Action. The [`locations.yaml`](./locations.yaml) file
+specifies to build the `github_action_demo_minimal` code location using the Dockerfile in the current
+directory, using `repo.py` as an entry point.
+
+The GitHub actions workflow for this example can be found at
+[`.github/workflows/example-minimal.yml`](../../.github/workflows/example-minimal.yml).

example/container-context/locations.yaml

@@ -0,0 +1,9 @@
+locations:
+  github_action_demo_container_context:
+    build: .
+    registry: 764506304434.dkr.ecr.us-west-2.amazonaws.com/github-action-demo-container-context
+    python_file: repo.py
+    container_context:
+      docker:
+        env_vars:
+          - BAR_ENV_VAR=bar

example/container-context/repo.py

@@ -0,0 +1,35 @@
+from dagster import Field, RunRequest, StringSource, job, op, repository
+
+
+@op(
+    config_schema={
+        "must_be_bar": Field(
+            StringSource,
+        )
+    },
+)
+def foo_op(context):
+    must_be_bar = context.op_config["must_be_bar"]
+    if must_be_bar != "bar":
+        raise Exception(f"Job fails if must_be_bar is not bar, instead was {must_be_bar}")
+
+
+@op()
+def foo_op_2(_):
+    pass
+
+
+@job
+def foo_job():
+    foo_op()
+    foo_op_2()
+
+
+@job
+def other_foo_job():
+    foo_op()
+
+
+@repository
+def foo_repo():
+    return [foo_job, other_foo_job]

example/requirements-txt/requirements.txt

@@ -1,2 +1,2 @@
-dagster==0.12.14
-dagster-cloud==0.12.14
+dagster
+dagster-cloud

src/dagsterCloud.js

@@ -12,12 +12,15 @@ const LIST_LOCATION_QUERY = gql`
 `;
 
 const ADD_LOCATION_MUTATION = gql`
-  mutation ($location: LocationSelector!) {
-    addLocation(location: $location) {
+  mutation ($document: GenericScalar!) {
+    addLocationFromDocument(document: $document) {
       __typename
       ... on WorkspaceEntry {
         locationName
       }
+      ... on InvalidLocationError {
+        errors
+      }
       ... on PythonError {
         message
         stack
@@ -27,12 +30,15 @@ const ADD_LOCATION_MUTATION = gql`
 `;
 
 const UPDATE_LOCATION_MUTATION = gql`
-  mutation ($location: LocationSelector!) {
-    updateLocation(location: $location) {
+  mutation ($document: GenericScalar!) {
+    updateLocationFromDocument(document: $document) {
       __typename
       ... on WorkspaceEntry {
         locationName
       }
+      ... on InvalidLocationError {
+        errors
+      }
       ... on PythonError {
         message
         stack
@@ -68,27 +74,32 @@ export class DagsterCloudClient {
     });
   }
 
-  async updateLocation(location) {
+  async updateLocation(document) {
     const locationList = await this.gqlClient.request(LIST_LOCATION_QUERY);
     const locationNames = locationList.workspace.workspaceEntries.map(
       (entry) => entry.locationName
     );
 
     let result;
-    if (!locationNames.includes(location.name)) {
-      result = (
+    if (!locationNames.includes(document.location_name)) {
+      const gql_result = (
         await this.gqlClient.request(ADD_LOCATION_MUTATION, {
-          location: location,
+          document: document,
         })
-      ).addLocation;
+      );
+      result = gql_result.addLocationFromDocument;
     } else {
-      result = (
+      const gql_result = (
         await this.gqlClient.request(UPDATE_LOCATION_MUTATION, {
-          location: location,
+          document: document,
         })
-      ).updateLocation;
+      );
+      result = gql_result.updateLocationFromDocument;
     }
 
+    if (result.__typename == "InvalidLocationError") {
+      throw new Error("Invalid location:\n" + result.errors.join("\n"))
+    }
     if (result.__typename === "PythonError") {
       throw new Error(result.message);
     }

src/utils.js

@@ -128,6 +128,7 @@ export async function updateLocations(process, client, locations, imageTag) {
       const workingDirectory = location["working_directory"];
       const executablePath = location["executable_path"];
       const attribute = location["attribute"];
+      const containerContext = location["container_context"]
 
       if (
         [pythonFile, packageName, moduleName].filter((x) => !!x).length != 1
@@ -145,20 +146,25 @@ export async function updateLocations(process, client, locations, imageTag) {
         `${github.context.repo.repo}/tree/${shortSha}`;
 
       const locationData = {
-        name: locationName,
+        location_name: locationName,
+        code_source: {
+          python_file: pythonFile,
+          package_name: packageName,
+          module_name: moduleName,
+        },
         image: `${location["registry"]}:${imageTag}`,
-        pythonFile: pythonFile,
-        packageName: packageName,
-        moduleName: moduleName,
-        workingDirectory: workingDirectory,
-        executablePath: executablePath,
+        working_directory: workingDirectory,
+        executable_path: executablePath,
         attribute: attribute,
-        commitHash: sha,
-        url: url,
+        git: {
+          commit_hash: sha,
+          url: url
+        },
+        container_context: containerContext,
       };
 
       const result = await client.updateLocation(locationData);
-      core.info(`Successfully updated location ${result}`);
+      core.info(`Updated location ${result}.`);
     });
   });
 }