Merge pull request #29 from vincentclaes/20-secure-command

20 secure command

README.md

@@ -67,8 +67,8 @@ The definition of our pipeline can be found in `examples/data_pipeline_simple/da
 
 Set the aws account number and the profile that contains your aws credentials (`~/.aws/credentials`) as environment variables:
 
-    export AWS_DEFAULT_ACCOUNT=my-account-number
     export AWS_PROFILE=my-profile
+    export AWS_DEFAULT_REGION=your-region # e.g. eu-west-1
 
 Point to the configuration of the pipeline using `--config` and deploy
 

datajob/__init__.py

@@ -1,6 +1,8 @@
 import logging
 import os
 import pathlib
+import shlex
+import subprocess
 from pathlib import Path
 
 ROOT_DIR = pathlib.Path(__file__).parent.absolute()
@@ -16,3 +18,14 @@
 logging.basicConfig(level=logging.getLevelName(log_level))
 project_name = Path(__file__).parent.stem
 logger = logging.getLogger(project_name)
+
+
+def call_subprocess(cmd: str) -> None:
+    """
+    call a command as a subprocess in a secure way.
+    https://stackoverflow.com/a/59090212/1771155
+    :param cmd: the command to execute
+    :return: None
+    """
+    print(f"datajob subprocess command: " f"{cmd}")
+    subprocess.check_call(shlex.split(cmd))

datajob/datajob.py

@@ -1,10 +1,10 @@
 import os
 import pathlib
+import shlex
 import subprocess
 from pathlib import Path
 
 import typer
-import shlex
 
 from datajob.package import wheel
 

datajob/glue/glue_job.py

@@ -1,5 +1,5 @@
-from pathlib import Path
 from enum import Enum
+from pathlib import Path
 
 from aws_cdk import aws_glue as glue, core, aws_s3_deployment
 from aws_cdk import aws_iam as iam

datajob/package/wheel.py

@@ -1,8 +1,6 @@
-import subprocess
 from pathlib import Path
-import shlex
 
-from datajob import logger
+from datajob import logger, call_subprocess
 
 
 class DatajobPackageWheelError(Exception):
@@ -27,16 +25,11 @@ def _setuppy_wheel(project_root: str) -> None:
     :param project_root: the path to the root of your project.
     :return: None
     """
-    setup_py_file = Path(project_root, "setup.py")
-    if setup_py_file.is_file():
-        logger.debug(f"found a setup.py file in {project_root}")
-        cmd = f"cd {project_root}; python setup.py bdist_wheel"
-        _call_create_wheel_command(cmd=cmd)
-    else:
-        raise DatajobPackageWheelError(
-            f"no setup.py file detected in project root {project_root}. "
-            f"Hence we cannot create a python wheel for this project"
-        )
+    _execute_packaging_logic(
+        project_root=project_root,
+        config_file="setup.py",
+        cmd="python setup.py bdist_wheel",
+    )
 
 
 def _poetry_wheel(project_root: str) -> None:
@@ -45,25 +38,27 @@ def _poetry_wheel(project_root: str) -> None:
     :param project_root: the path to the root of your project.
     :return: None
     """
-    poetry_file = Path(project_root, "pyproject.toml")
-    if poetry_file.is_file():
-        logger.debug(f"found a pyproject.toml file in {project_root}")
-        cmd = f"cd {project_root}; poetry build"
-        _call_create_wheel_command(cmd=cmd)
-    else:
-        raise DatajobPackageWheelError(
-            f"no pyproject.toml file detected in project root {project_root}. "
-            f"Hence we cannot create a python wheel for this project"
-        )
+    _execute_packaging_logic(
+        project_root=project_root, config_file="pyproject.toml", cmd="poetry build"
+    )
 
 
-def _call_create_wheel_command(cmd: str) -> None:
+def _execute_packaging_logic(project_root: str, config_file: str, cmd: str) -> None:
     """
-    shell out and call the command to create the wheel.
-    :param cmd: the command to create a wheel
+    check if the config file exists in the project root and execute the command to
+    create a wheel.
+    :param project_root: the path to the root of your project.
+    :param config_file: the confgi file to package the project as a wheel (setup.py or pyproject.toml)
+    :param cmd: the command to execute to create a wheel.
     :return: None
     """
-    logger.debug("creating wheel")
-    print(f"wheel command: {cmd}")
-    # todo - shell=True is not secure
-    subprocess.call(cmd, shell=True)
+    config_file_full_path = Path(project_root, config_file)
+    logger.info(f"expecting {config_file_full_path}")
+    if config_file_full_path.is_file():
+        logger.debug(f"found a {config_file} file in {project_root}")
+        call_subprocess(cmd=cmd)
+    else:
+        raise DatajobPackageWheelError(
+            f"no {config_file} file detected in project root {project_root}. "
+            f"Hence we cannot create a python wheel for this project"
+        )

datajob/stepfunctions/stepfunctions_workflow.py

@@ -1,10 +1,10 @@
-import contextvars
+import os
 import tempfile
 import uuid
-import os
-import boto3
 from pathlib import Path
 
+import boto3
+import contextvars
 from aws_cdk import aws_iam as iam
 from aws_cdk import cloudformation_include as cfn_inc
 from aws_cdk import core

datajob_tests/datajob_context_test.py

@@ -1,4 +1,5 @@
 import unittest
+
 from datajob.datajob_stack import DataJobStack, DatajobContext
 
 

datajob_tests/datajob_stack_test.py

@@ -1,4 +1,5 @@
 import unittest
+
 from datajob.datajob_stack import DataJobStack
 
 

datajob_tests/datajob_test.py

@@ -1,6 +1,6 @@
 import unittest
-
 from unittest.mock import patch
+
 from typer.testing import CliRunner
 
 from datajob import datajob

datajob_tests/glue/glue_job_test.py

@@ -1,4 +1,5 @@
 import unittest
+
 from datajob.datajob_stack import DataJobStack
 from datajob.glue.glue_job import GlueJob, GlueJobType
 

datajob_tests/stepfunctions/stepfunctions_workflow_test.py

@@ -1,5 +1,5 @@
-import unittest
 import os
+import unittest
 
 from moto import mock_stepfunctions
 from stepfunctions.steps.compute import GlueStartJobRunStep

examples/data_pipeline_simple/README.md

@@ -8,7 +8,7 @@ The definition of the datajob can be found in `datajob_stack.py`
 
 # Deployment
 
-    export AWS_DEFAULT_ACCOUNT=my-account-number
     export AWS_PROFILE=my-profile
+    export AWS_DEFAULT_REGION=eu-west-1
     cd examples/data_pipeline_simple
     datajob deploy --config datajob_stack.py

examples/data_pipeline_with_packaged_project/README.md

@@ -17,8 +17,8 @@ Make sure you have configured a `setup.py` in the root of your poject.
 
 ## Deployment
 
-    export AWS_DEFAULT_ACCOUNT=my-account-number
     export AWS_PROFILE=my-profile
+    export AWS_DEFAULT_REGION=eu-west-1
     cd examples/data_pipeline_with_packaged_project
     # if you want to use poetry to create a wheel
     datajob deploy --config datajob_stack.py --package poetry