Merge pull request #25 from vincentclaes/21-gh-actions

add dockerfile for running github actions

.devcontainer/devcontainer.Dockerfile

@@ -0,0 +1,112 @@
+#-------------------------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
+#-------------------------------------------------------------------------------------------------------------
+
+# Update the VARIANT arg in devcontainer.json to pick a Python version: 3, 3.8, 3.7, 3.6
+ARG VARIANT=3.6
+FROM python:${VARIANT}
+
+# If you would prefer to have multiple Python versions in your container,
+# replace the FROM statement above with the following:
+#
+# FROM ubuntu:bionic
+# ARG PYTHON_PACKAGES="python3.5 python3.6 python3.7 python3.8 python3 python3-pip python3-venv"
+# RUN apt-get update && apt-get install --no-install-recommends -yq software-properties-common \
+#     && add-apt-repository ppa:deadsnakes/ppa && apt-get update \
+#     && apt-get install -yq --no-install-recommends ${PYTHON_PACKAGES} \
+#     && pip3 install --no-cache-dir --upgrade pip setuptools wheel
+
+# This Dockerfile adds a non-root user with sudo access. Use the "remoteUser"
+# property in devcontainer.json to use it. On Linux, the container user's GID/UIDs
+# will be updated to match your local UID/GID (when using the dockerFile property).
+# See https://aka.ms/vscode-remote/containers/non-root-user for details.
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+# Uncomment the following COPY line and the corresponding lines in the `RUN` command if you wish to
+# include your requirements in the image itself. Only do this if your requirements rarely change.
+# COPY requirements.txt /tmp/pip-tmp/
+
+# Default set of utilities to install in a side virtual env
+ARG DEFAULT_UTILS="\
+  pylint \
+  flake8 \
+  autopep8 \
+  black \
+  pytest \
+  yapf \
+  mypy \
+  pydocstyle \
+  pycodestyle \
+  bandit \
+  virtualenv \
+  pipenv \
+  poetry"
+
+ENV PIPX_HOME=/usr/local/py-utils
+ENV PIPX_BIN_DIR=${PIPX_HOME}/bin
+ENV PATH=${PATH}:${PIPX_BIN_DIR}
+
+# Options for common package install script
+ARG INSTALL_ZSH="true"
+ARG UPGRADE_PACKAGES="true"
+ARG COMMON_SCRIPT_SOURCE="https://raw.githubusercontent.com/microsoft/vscode-dev-containers/master/script-library/common-debian.sh"
+ARG COMMON_SCRIPT_SHA="dev-mode"
+
+# Configure apt and install packages
+RUN apt-get update \
+  && export DEBIAN_FRONTEND=noninteractive \
+  #
+  # Verify git, common tools / libs installed, add/modify non-root user, optionally install zsh
+  && apt-get -y install --no-install-recommends curl ca-certificates 2>&1 \
+  && curl -sSL  ${COMMON_SCRIPT_SOURCE} -o /tmp/common-setup.sh \
+  && ([ "${COMMON_SCRIPT_SHA}" = "dev-mode" ] || (echo "${COMMON_SCRIPT_SHA} */tmp/common-setup.sh" | sha256sum -c -)) \
+  && /bin/bash /tmp/common-setup.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" \
+  && rm /tmp/common-setup.sh \
+  #
+  # Setup default python tools in a venv via pipx to avoid conflicts
+  && mkdir -p ${PIPX_BIN_DIR} \
+  && export PYTHONUSERBASE=/tmp/pip-tmp \
+  && pip3 install --disable-pip-version-check --no-warn-script-location --no-cache-dir --user pipx \
+  && /tmp/pip-tmp/bin/pipx install --pip-args=--no-cache-dir pipx \
+  && echo "${DEFAULT_UTILS}" | xargs -n 1 /tmp/pip-tmp/bin/pipx install --system-site-packages --pip-args=--no-cache-dir \
+  && chown -R ${USER_UID}:${USER_GID} ${PIPX_HOME} \
+  && rm -rf /tmp/pip-tmp \
+  #
+  # Tactically remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
+  # Can leave in Dockerfile once upstream base image moves to > 7.0.7-28.
+  && apt-get purge -y imagemagick imagemagick-6-common \
+  #
+  # Clean up
+  && apt-get autoremove -y \
+  && apt-get clean -y \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install github cli
+RUN apt-get update && apt-get -y install software-properties-common \
+  && apt-key adv --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 \
+  && apt-add-repository https://cli.github.com/packages \
+  && apt-get update \
+  && apt-get -y install gh
+
+# Install AWS CLI
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
+  && unzip awscliv2.zip \
+  && ./aws/install \
+  && rm awscliv2.zip
+
+# install AWS CDK
+# this worked to install a specific node version
+# https://stackoverflow.com/a/62838796/1771155
+ENV NVM_DIR /usr/local/nvm
+RUN mkdir -p /usr/local/nvm
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
+ENV NODE_VERSION v14.13.1
+RUN /bin/bash -c "source $NVM_DIR/nvm.sh && nvm install $NODE_VERSION && nvm use --delete-prefix $NODE_VERSION"
+
+ENV NODE_PATH $NVM_DIR/versions/node/$NODE_VERSION/lib/node_modules
+ENV PATH      $NVM_DIR/versions/node/$NODE_VERSION/bin:$PATH
+ENV AWS_DEFAULT_REGION=eu-west-1
+RUN npm install -g aws-cdk@latest

.devcontainer/devcontainer.json

@@ -0,0 +1,52 @@
+{
+	"name": "Python 3",
+	"build": {
+		"dockerfile": "./devcontainer.Dockerfile",
+		"context": "..",
+		// Update 'VARIANT' to pick a Python version. Rebuild the container
+		// if it already exists to update. Available variants: 3, 3.6, 3.7, 3.8
+		"args": { "VARIANT": "3.6" }
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {
+		"terminal.integrated.shell.linux": "/bin/bash",
+		"python.pythonPath": "/usr/local/bin/python",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
+		"python.testing.pytestPath": "/usr/local/py-utils/bin/pytest"
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python",
+		"eamodio.gitlens",
+		"ms-python.vscode-pylance",
+		"grapecity.gc-excelviewer",
+		"github.vscode-pull-request-github",
+		"cschleiden.vscode-github-actions"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [9000],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "",
+
+	// Comment out to connect as root instead.
+	"remoteUser": "vscode",
+
+	"mounts": [
+		 "source=${localEnv:HOME}/.aws,target=/home/vscode/.aws,type=bind,consistency=cached",
+		 "source=${localEnv:HOME}/.config,target=/home/vscode/.config,type=bind,consistency=cached",
+		 "source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,consistency=cached"
+	  ]
+}

.github/workflows/pr.yml

@@ -0,0 +1,22 @@
+name: Build-Test-RunExamples
+on: [pull_request]
+
+env:
+  IMAGE_TAG: local
+
+jobs:
+  Build-Test-RunExamples:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checking out
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.ref }}
+
+    - name: Build Container
+      run: |
+         cat .devcontainer/devcontainer.Dockerfile | docker build -t $IMAGE_TAG -
+
+    - name: Test
+      run: docker run -v $PWD:/workspace/vscode -w /workspace/vscode $IMAGE_TAG make gh-actions

Makefile

@@ -1,7 +1,23 @@
 install:
-	pip install poetry
+	pip3 install poetry
+	poetry config virtualenvs.create true
 	poetry install
-	poetry shell
+
+
+install-dev:
+	make install
 	# install pre commit hooks to check the code
 	# before committing.
-	pre-commit install
+	poetry run pre-commit install
+
+tests:
+	poetry run pytest
+
+run-examples:
+	cd "${CURDIR}/examples/data_pipeline_simple" && poetry run datajob synthesize --config datajob_stack.py --stage dev
+	cd "${CURDIR}/examples/data_pipeline_with_packaged_project" && poetry run datajob synthesize --config datajob_stack.py --stage dev --package
+
+gh-actions:
+	make install
+	make tests
+	make run-examples

datajob/datajob_stack.py

@@ -17,7 +17,7 @@ def __init__(
         include_folder: str = None,
         account: str = None,
         region: str = None,
-        scope: core.Construct = None,
+        scope: core.Construct = core.App(),
         **kwargs,
     ) -> None:
         """
@@ -36,7 +36,7 @@ def __init__(
         )
         region = region if region is not None else os.environ.get("AWS_DEFAULT_REGION")
         env = {"region": region, "account": account}
-        self.scope = scope if scope else core.App()
+        self.scope = scope
         self.stage = self.get_stage(stage)
         self.unique_stack_name = self._create_unique_stack_name(stack_name, self.stage)
         super().__init__(scope=scope, id=self.unique_stack_name, env=env, **kwargs)

datajob/stepfunctions/stepfunctions_workflow.py

@@ -1,6 +1,8 @@
 import contextvars
 import tempfile
 import uuid
+import os
+import boto3
 from pathlib import Path
 
 from aws_cdk import aws_iam as iam
@@ -31,7 +33,12 @@ class StepfunctionsWorkflow(DataJobBase):
     """
 
     def __init__(
-        self, datajob_stack: core.Construct, name: str, role: iam.Role = None, **kwargs
+        self,
+        datajob_stack: core.Construct,
+        name: str,
+        role: iam.Role = None,
+        region: str = None,
+        **kwargs,
     ):
         super().__init__(datajob_stack, name, **kwargs)
         self.chain_of_tasks = []
@@ -43,6 +50,7 @@ def __init__(
             if role is None
             else role
         )
+        self.region = region if region else os.environ["AWS_DEFAULT_REGION"]
 
     def add_task(self, task_other):
         """add a task to the workflow we would like to orchestrate."""
@@ -68,9 +76,7 @@ def add_parallel_tasks(self, task_others):
     def _create_glue_start_job_run_step(job_name):
         logger.debug("creating a step for a glue job.")
         return GlueStartJobRunStep(
-            job_name,
-            wait_for_completion=True,
-            parameters={"JobName": job_name},
+            job_name, wait_for_completion=True, parameters={"JobName": job_name}
         )
 
     def _build_workflow(self):
@@ -80,10 +86,12 @@ def _build_workflow(self):
         )
         workflow_definition = steps.Chain(self.chain_of_tasks)
         logger.debug(f"creating a workflow with name {self.unique_name}")
+        self.client = boto3.client("stepfunctions", region_name=self.region)
         self.workflow = Workflow(
             name=self.unique_name,
             definition=workflow_definition,
             role=self.role.role_arn,
+            client=self.client,
         )
 
     def create(self):

datajob_tests/datajob_stack_test.py

@@ -6,13 +6,13 @@ class DatajobStackTest(unittest.TestCase):
     def test_datajob_stack_initiates_without_error(self):
         exception_ = None
         try:
-            with DataJobStack(stack_name="some-stack-name") as djs:
+            with DataJobStack(stack_name="datajob-stack-no-error") as djs:
                 pass
         except Exception as e:
             exception_ = e
         self.assertIsNone(exception_)
 
     def test_datajob_stack_with_no_stage(self):
-        with DataJobStack(stack_name="some-stack-name") as djs:
+        with DataJobStack(stack_name="datajob-stack-no-stage") as djs:
             pass
         self.assertEqual(djs.stage, DataJobStack.DEFAULT_STAGE)

datajob_tests/datajob_test.py

@@ -1,6 +1,6 @@
 import unittest
 
-from mock import patch
+from unittest.mock import patch
 from typer.testing import CliRunner
 
 from datajob import datajob

datajob_tests/stepfunctions/stepfunctions_workflow_test.py

@@ -1,4 +1,5 @@
 import unittest
+import os
 
 from moto import mock_stepfunctions
 from stepfunctions.steps.compute import GlueStartJobRunStep
@@ -16,6 +17,15 @@ def __init__(self, unique_name):
 
 
 class StepfunctionsWorkflowTestTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        # we need a AWS region else these tests will fail with boto3 stepfunctions.
+        # (even when using moto to mock stepfunctions!)
+        try:
+            os.environ["AWS_DEFAULT_REGION"]
+        except KeyError:
+            os.environ["AWS_DEFAULT_REGION"] = "eu-west-1"
+
     @mock_stepfunctions
     def test_create_tasks_for_orchestration_simple_flow_successfully(self):
         task1 = stepfunctions_workflow.task(SomeMockedClass("task1"))