From 1f51df9eb2b683b0e364b208f888f45094a4016e Mon Sep 17 00:00:00 2001 From: vincent Date: Fri, 29 Jan 2021 17:01:00 +0100 Subject: [PATCH 01/10] bump to 0.5.0 --- README.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e824cd4..bc2544a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Beware that we depend on [aws cdk cli](https://github.com/aws/aws-cdk)! pip install datajob - npm install -g aws-cdk + npm install -g aws-cdk@1.87.1 # latest version of datajob depends this version # Quickstart diff --git a/pyproject.toml b/pyproject.toml index 9f9ed74..feb1316 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datajob" -version = "0.4.0" +version = "0.5.0" description = "Build and deploy a serverless data pipeline with no effort on AWS." authors = ["Vincent Claes "] license = "Apache Software License (Apache 2.0)" From 932be52e1972fd3264ae0495b2ec06bac934f3d9 Mon Sep 17 00:00:00 2001 From: vincent Date: Fri, 29 Jan 2021 18:44:50 +0100 Subject: [PATCH 02/10] expand the package possibilities --- datajob/datajob.py | 4 +-- datajob/package/wheel.py | 41 ++++++++++++++++++++++---- datajob_tests/datajob_test.py | 55 +++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/datajob/datajob.py b/datajob/datajob.py index 8139f4a..04e3712 100644 --- a/datajob/datajob.py +++ b/datajob/datajob.py @@ -22,13 +22,13 @@ def run(): def deploy( stage: str = typer.Option(None), config: str = typer.Option(Path, callback=os.path.abspath), - package: bool = typer.Option(False, "--package"), + package: str = typer.Option(None, "--package"), ctx: typer.Context = typer.Option(list), ): if package: # todo - check if we are building in the right directory project_root = str(Path(config).parent) - wheel.create(project_root=project_root) + wheel.create(project_root=project_root, package=package) # create stepfunctions if requested # make sure you have quotes around the app arguments args = ["--app", f""" "python {config}" """, "-c", f"stage={stage}"] diff --git a/datajob/package/wheel.py b/datajob/package/wheel.py index c146847..76a3ea1 100644 --- a/datajob/package/wheel.py +++ b/datajob/package/wheel.py @@ -8,7 +8,17 @@ class DatajobPackageWheelError(Exception): """any exception occuring when constructing a wheel in data job context.""" -def create(project_root): +def create(project_root: str, package: str) -> None: + + wheel_functions = { + "pip": _setuppy_wheel, + "pipenv": _setuppy_wheel, + "poetry": _poetry_wheel, + } + wheel_functions[package](project_root) + + +def _setuppy_wheel(project_root): """launch a subprocess to built a wheel. todo - use the setuptools/disttools api to create a setup.py. relying on a subprocess feels dangerous. @@ -16,13 +26,34 @@ def create(project_root): setup_py_file = Path(project_root, "setup.py") if setup_py_file.is_file(): logger.debug(f"found a setup.py file in {project_root}") - logger.debug("creating wheel for glue job") cmd = f"cd {project_root}; python setup.py bdist_wheel" - print(f"wheel command: {cmd}") - # todo - shell=True is not secure - subprocess.call(cmd, shell=True) + _call_create_wheel_command(cmd=cmd) else: raise DatajobPackageWheelError( f"no setup.py file detected in project root {project_root}. " f"Hence we cannot create a python wheel for this project" ) + + +def _poetry_wheel(project_root): + """launch a subprocess to built a wheel. + todo - use the setuptools/disttools api to create a setup.py. + relying on a subprocess feels dangerous. + """ + poetry_file = Path(project_root, "pyproject.toml") + if poetry_file.is_file(): + logger.debug(f"found a pyproject.toml file in {project_root}") + cmd = f"cd {project_root}; poetry build" + _call_create_wheel_command(cmd=cmd) + else: + raise DatajobPackageWheelError( + f"no pyproject.toml file detected in project root {project_root}. " + f"Hence we cannot create a python wheel for this project" + ) + + +def _call_create_wheel_command(cmd: str) -> None: + logger.debug("creating wheel") + print(f"wheel command: {cmd}") + # todo - shell=True is not secure + subprocess.call(cmd, shell=True) diff --git a/datajob_tests/datajob_test.py b/datajob_tests/datajob_test.py index f35e6d4..d5f2589 100644 --- a/datajob_tests/datajob_test.py +++ b/datajob_tests/datajob_test.py @@ -49,6 +49,61 @@ def test_datajob_deploy_cli_runs_with_project_root_successfully( "--stage", "some-stage", "--package", + "poetry", + ], + ) + self.assertEqual(result.exit_code, 0) + self.assertEqual(m_create_wheel.call_count, 1) + + @patch("datajob.package.wheel._poetry_wheel") + @patch("datajob.datajob.call_cdk") + def test_datajob_deploy_with_package_poetry(self, m_call_cdk, m_create_wheel): + result = self.runner.invoke( + datajob.app, + [ + "deploy", + "--config", + "some_config.py", + "--stage", + "some-stage", + "--package", + "poetry", + ], + ) + self.assertEqual(result.exit_code, 0) + self.assertEqual(m_create_wheel.call_count, 1) + + @patch("datajob.package.wheel._setuppy_wheel") + @patch("datajob.datajob.call_cdk") + def test_datajob_deploy_with_package_pip(self, m_call_cdk, m_create_wheel): + result = self.runner.invoke( + datajob.app, + [ + "deploy", + "--config", + "some_config.py", + "--stage", + "some-stage", + "--package", + "pip", + ], + ) + self.assertEqual(result.exit_code, 0) + self.assertEqual(m_create_wheel.call_count, 1) + + @patch("datajob.package.wheel._setuppy_wheel") + @patch("datajob.datajob.call_cdk") + def test_datajob_deploy_with_package_pipenv(self, m_call_cdk, m_create_wheel): + result = self.runner.invoke( + datajob.app, + [ + "deploy", + "--config", + "some_config.py", + "--stage", + "some-stage", + "--package", + "pipenv", ], ) self.assertEqual(result.exit_code, 0) From 41bb8a843b924a725cff2cbd700c9fe5cbfc0b84 Mon Sep 17 00:00:00 2001 From: vincent Date: Fri, 29 Jan 2021 19:11:27 +0100 Subject: [PATCH 03/10] refine packaging --- datajob/datajob.py | 4 ++-- datajob/datajob_stack.py | 14 ++++++-------- datajob/package/wheel.py | 6 +----- .../data_pipeline_with_packaged_project/README.md | 5 ++++- .../pyproject.toml | 15 +++++++++++++++ 5 files changed, 28 insertions(+), 16 deletions(-) create mode 100644 examples/data_pipeline_with_packaged_project/pyproject.toml diff --git a/datajob/datajob.py b/datajob/datajob.py index 04e3712..f9231a8 100644 --- a/datajob/datajob.py +++ b/datajob/datajob.py @@ -40,7 +40,7 @@ def deploy( context_settings={"allow_extra_args": True, "ignore_unknown_options": True} ) def synthesize( - stage: str = typer.Option(...), + stage: str = typer.Option(None), config: str = typer.Option(Path, callback=os.path.abspath), ctx: typer.Context = typer.Option(list), ): @@ -53,7 +53,7 @@ def synthesize( context_settings={"allow_extra_args": True, "ignore_unknown_options": True} ) def destroy( - stage: str = typer.Option(...), + stage: str = typer.Option(None), config: str = typer.Option(Path, callback=os.path.abspath), ctx: typer.Context = typer.Option(list), ): diff --git a/datajob/datajob_stack.py b/datajob/datajob_stack.py index e63655d..b1d831f 100644 --- a/datajob/datajob_stack.py +++ b/datajob/datajob_stack.py @@ -93,19 +93,17 @@ def create_resources(self): def get_stage(self, stage): """get the stage parameter and return a default if not found.""" try: - if stage == "None": - logger.debug("no stage parameter is passed via the cli.") + if stage == "None" or not stage: + logger.debug( + "No stage is passed to datajob stack, taking the default one." + ) return DataJobStack.DEFAULT_STAGE - if stage: + elif stage: logger.debug( "a stage parameter is passed via the cli or via the datajob stack configuration file." ) return stage - if stage is None: - logger.debug( - "No stage is passed to datajob stack, taking the default one." - ) - return self.get_context_parameter("stage") + except ValueError: return DataJobStack.DEFAULT_STAGE diff --git a/datajob/package/wheel.py b/datajob/package/wheel.py index 76a3ea1..9a45c14 100644 --- a/datajob/package/wheel.py +++ b/datajob/package/wheel.py @@ -10,11 +10,7 @@ class DatajobPackageWheelError(Exception): def create(project_root: str, package: str) -> None: - wheel_functions = { - "pip": _setuppy_wheel, - "pipenv": _setuppy_wheel, - "poetry": _poetry_wheel, - } + wheel_functions = {"setuppy": _setuppy_wheel, "poetry": _poetry_wheel} wheel_functions[package](project_root) diff --git a/examples/data_pipeline_with_packaged_project/README.md b/examples/data_pipeline_with_packaged_project/README.md index e490d6c..808fcde 100644 --- a/examples/data_pipeline_with_packaged_project/README.md +++ b/examples/data_pipeline_with_packaged_project/README.md @@ -20,4 +20,7 @@ Make sure you have configured a `setup.py` in the root of your poject. export AWS_DEFAULT_ACCOUNT=my-account-number export AWS_PROFILE=my-profile cd examples/data_pipeline_with_packaged_project - datajob deploy --stage dev --config datajob_stack.py --package + # if you want to use poetry to create a wheel + datajob deploy --config datajob_stack.py --package poetry + # if you want to create a wheel from setup.py + datajob deploy --config datajob_stack.py --package setuppy diff --git a/examples/data_pipeline_with_packaged_project/pyproject.toml b/examples/data_pipeline_with_packaged_project/pyproject.toml new file mode 100644 index 0000000..40fb40c --- /dev/null +++ b/examples/data_pipeline_with_packaged_project/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "data_pipeline_with_packaged_project" +version = "0.1.0" +description = "" +authors = ["Vincent Claes "] +license = "" + +readme = 'README.md' # Markdown files are supported + +[tool.poetry.dependencies] +python = "^3.6.1" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" From d3782892da356512344a6be64cdfa92c16612b8a Mon Sep 17 00:00:00 2001 From: vincent Date: Fri, 29 Jan 2021 19:12:22 +0100 Subject: [PATCH 04/10] fix examples --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f86cc11..1f6e4f2 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,8 @@ tests: run-examples: cd "${CURDIR}/examples/data_pipeline_simple" && poetry run datajob synthesize --config datajob_stack.py --stage dev - cd "${CURDIR}/examples/data_pipeline_with_packaged_project" && poetry run datajob synthesize --config datajob_stack.py --stage dev --package + cd "${CURDIR}/examples/data_pipeline_with_packaged_project" && poetry run datajob synthesize --config datajob_stack.py --stage dev --package poetry + cd "${CURDIR}/examples/data_pipeline_with_packaged_project" && poetry run datajob synthesize --config datajob_stack.py --stage dev --package setuppy gh-actions: make install From c22b9400e1b1a802c5fb32ce5892b4f5471f76a1 Mon Sep 17 00:00:00 2001 From: vincent Date: Sat, 30 Jan 2021 14:34:40 +0100 Subject: [PATCH 05/10] fix test setuppy --- datajob_tests/datajob_test.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/datajob_tests/datajob_test.py b/datajob_tests/datajob_test.py index d5f2589..26b7d31 100644 --- a/datajob_tests/datajob_test.py +++ b/datajob_tests/datajob_test.py @@ -75,7 +75,7 @@ def test_datajob_deploy_with_package_poetry(self, m_call_cdk, m_create_wheel): @patch("datajob.package.wheel._setuppy_wheel") @patch("datajob.datajob.call_cdk") - def test_datajob_deploy_with_package_pip(self, m_call_cdk, m_create_wheel): + def test_datajob_deploy_with_package_setuppy(self, m_call_cdk, m_create_wheel): result = self.runner.invoke( datajob.app, [ @@ -85,25 +85,7 @@ def test_datajob_deploy_with_package_pip(self, m_call_cdk, m_create_wheel): "--stage", "some-stage", "--package", - "pip", - ], - ) - self.assertEqual(result.exit_code, 0) - self.assertEqual(m_create_wheel.call_count, 1) - - @patch("datajob.package.wheel._setuppy_wheel") - @patch("datajob.datajob.call_cdk") - def test_datajob_deploy_with_package_pipenv(self, m_call_cdk, m_create_wheel): - result = self.runner.invoke( - datajob.app, - [ - "deploy", - "--config", - "some_config.py", - "--stage", - "some-stage", - "--package", - "pipenv", + "setuppy", ], ) self.assertEqual(result.exit_code, 0) From 527bae26670f8baabbeeb6ca29d9511f51fa2255 Mon Sep 17 00:00:00 2001 From: vincent Date: Sat, 30 Jan 2021 14:58:50 +0100 Subject: [PATCH 06/10] fix test setuppy --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 1f6e4f2..cdda7d7 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,6 @@ tests: run-examples: cd "${CURDIR}/examples/data_pipeline_simple" && poetry run datajob synthesize --config datajob_stack.py --stage dev - cd "${CURDIR}/examples/data_pipeline_with_packaged_project" && poetry run datajob synthesize --config datajob_stack.py --stage dev --package poetry cd "${CURDIR}/examples/data_pipeline_with_packaged_project" && poetry run datajob synthesize --config datajob_stack.py --stage dev --package setuppy gh-actions: From e79711755663ee3fa8b0094ecc070a43fc094cae Mon Sep 17 00:00:00 2001 From: vincent Date: Sat, 30 Jan 2021 15:36:33 +0100 Subject: [PATCH 07/10] remove pyproject.toml from example --- .../pyproject.toml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 examples/data_pipeline_with_packaged_project/pyproject.toml diff --git a/examples/data_pipeline_with_packaged_project/pyproject.toml b/examples/data_pipeline_with_packaged_project/pyproject.toml deleted file mode 100644 index 40fb40c..0000000 --- a/examples/data_pipeline_with_packaged_project/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -[tool.poetry] -name = "data_pipeline_with_packaged_project" -version = "0.1.0" -description = "" -authors = ["Vincent Claes "] -license = "" - -readme = 'README.md' # Markdown files are supported - -[tool.poetry.dependencies] -python = "^3.6.1" - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" From e8f5aca2d39c3c2fc74ddb8b809a1b1969dbf492 Mon Sep 17 00:00:00 2001 From: vincent Date: Sat, 30 Jan 2021 15:46:27 +0100 Subject: [PATCH 08/10] update documentation --- datajob/datajob.py | 2 +- datajob/datajob_stack.py | 2 +- datajob/package/wheel.py | 27 ++++++++++++++++++--------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/datajob/datajob.py b/datajob/datajob.py index f9231a8..9fabd2c 100644 --- a/datajob/datajob.py +++ b/datajob/datajob.py @@ -28,7 +28,7 @@ def deploy( if package: # todo - check if we are building in the right directory project_root = str(Path(config).parent) - wheel.create(project_root=project_root, package=package) + wheel.create_wheel(project_root=project_root, package=package) # create stepfunctions if requested # make sure you have quotes around the app arguments args = ["--app", f""" "python {config}" """, "-c", f"stage={stage}"] diff --git a/datajob/datajob_stack.py b/datajob/datajob_stack.py index b1d831f..5e3c1fb 100644 --- a/datajob/datajob_stack.py +++ b/datajob/datajob_stack.py @@ -88,7 +88,7 @@ def _create_unique_stack_name(stack_name: str, stage: str) -> str: def create_resources(self): """create each of the resources of this stack""" - [resource.create() for resource in self.resources] + [resource.create_wheel() for resource in self.resources] def get_stage(self, stage): """get the stage parameter and return a default if not found.""" diff --git a/datajob/package/wheel.py b/datajob/package/wheel.py index 9a45c14..be29e6d 100644 --- a/datajob/package/wheel.py +++ b/datajob/package/wheel.py @@ -8,16 +8,23 @@ class DatajobPackageWheelError(Exception): """any exception occuring when constructing a wheel in data job context.""" -def create(project_root: str, package: str) -> None: - +def create_wheel(project_root: str, package: str) -> None: + """ + Select the function to build a wheel based on the argument provided with --package. + At the time of writing this can be (setuppy or poetry) + :param project_root: the path to the root of your project. + :param package: the tool you want to use to build your wheel using (setuppy, poetry, ...) + :return: None + """ wheel_functions = {"setuppy": _setuppy_wheel, "poetry": _poetry_wheel} wheel_functions[package](project_root) -def _setuppy_wheel(project_root): - """launch a subprocess to built a wheel. - todo - use the setuptools/disttools api to create a setup.py. - relying on a subprocess feels dangerous. +def _setuppy_wheel(project_root: str) -> None: + """ + build a wheel for your project using poetry. + :param project_root: the path to the root of your project. + :return: None """ setup_py_file = Path(project_root, "setup.py") if setup_py_file.is_file(): @@ -32,9 +39,10 @@ def _setuppy_wheel(project_root): def _poetry_wheel(project_root): - """launch a subprocess to built a wheel. - todo - use the setuptools/disttools api to create a setup.py. - relying on a subprocess feels dangerous. + """ + build a wheel for your project using poetry. + :param project_root: the path to the root of your project. + :return: None """ poetry_file = Path(project_root, "pyproject.toml") if poetry_file.is_file(): @@ -49,6 +57,7 @@ def _poetry_wheel(project_root): def _call_create_wheel_command(cmd: str) -> None: + """shell out and call the command to create the wheel.""" logger.debug("creating wheel") print(f"wheel command: {cmd}") # todo - shell=True is not secure From 175394a4c7a155230ffc076791443edac7c1a0b7 Mon Sep 17 00:00:00 2001 From: vincent Date: Sat, 30 Jan 2021 15:55:48 +0100 Subject: [PATCH 09/10] update documentation --- datajob/datajob_stack.py | 2 +- datajob/package/wheel.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/datajob/datajob_stack.py b/datajob/datajob_stack.py index 5e3c1fb..b1d831f 100644 --- a/datajob/datajob_stack.py +++ b/datajob/datajob_stack.py @@ -88,7 +88,7 @@ def _create_unique_stack_name(stack_name: str, stage: str) -> str: def create_resources(self): """create each of the resources of this stack""" - [resource.create_wheel() for resource in self.resources] + [resource.create() for resource in self.resources] def get_stage(self, stage): """get the stage parameter and return a default if not found.""" diff --git a/datajob/package/wheel.py b/datajob/package/wheel.py index be29e6d..10dd8df 100644 --- a/datajob/package/wheel.py +++ b/datajob/package/wheel.py @@ -11,7 +11,7 @@ class DatajobPackageWheelError(Exception): def create_wheel(project_root: str, package: str) -> None: """ Select the function to build a wheel based on the argument provided with --package. - At the time of writing this can be (setuppy or poetry) + At the time of writing the arguments can be setuppy or poetry. :param project_root: the path to the root of your project. :param package: the tool you want to use to build your wheel using (setuppy, poetry, ...) :return: None @@ -38,7 +38,7 @@ def _setuppy_wheel(project_root: str) -> None: ) -def _poetry_wheel(project_root): +def _poetry_wheel(project_root: str) -> None: """ build a wheel for your project using poetry. :param project_root: the path to the root of your project. @@ -57,7 +57,11 @@ def _poetry_wheel(project_root): def _call_create_wheel_command(cmd: str) -> None: - """shell out and call the command to create the wheel.""" + """ + shell out and call the command to create the wheel. + :param cmd: the command to create a wheel + :return: None + """ logger.debug("creating wheel") print(f"wheel command: {cmd}") # todo - shell=True is not secure From 3f2b375451b329c2ac478d9e3d160f3bd747b5f4 Mon Sep 17 00:00:00 2001 From: vincent Date: Sun, 31 Jan 2021 10:52:50 +0100 Subject: [PATCH 10/10] fix test --- datajob_tests/datajob_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajob_tests/datajob_test.py b/datajob_tests/datajob_test.py index 26b7d31..328a3af 100644 --- a/datajob_tests/datajob_test.py +++ b/datajob_tests/datajob_test.py @@ -35,7 +35,7 @@ def test_datajob_deploy_cli_runs_with_unknown_args_successfully(self, m_call_cdk ) self.assertEqual(result.exit_code, 0) - @patch("datajob.package.wheel.create") + @patch("datajob.package.wheel.create_wheel") @patch("datajob.datajob.call_cdk") def test_datajob_deploy_cli_runs_with_project_root_successfully( self, m_call_cdk, m_create_wheel