diff --git a/Makefile b/Makefile index f86cc11..cdda7d7 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ tests: run-examples: cd "${CURDIR}/examples/data_pipeline_simple" && poetry run datajob synthesize --config datajob_stack.py --stage dev - cd "${CURDIR}/examples/data_pipeline_with_packaged_project" && poetry run datajob synthesize --config datajob_stack.py --stage dev --package + cd "${CURDIR}/examples/data_pipeline_with_packaged_project" && poetry run datajob synthesize --config datajob_stack.py --stage dev --package setuppy gh-actions: make install diff --git a/README.md b/README.md index e824cd4..bc2544a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Beware that we depend on [aws cdk cli](https://github.com/aws/aws-cdk)! pip install datajob - npm install -g aws-cdk + npm install -g aws-cdk@1.87.1 # latest version of datajob depends this version # Quickstart diff --git a/datajob/datajob.py b/datajob/datajob.py index a0ed530..381b85b 100644 --- a/datajob/datajob.py +++ b/datajob/datajob.py @@ -23,13 +23,13 @@ def run(): def deploy( stage: str = typer.Option(None), config: str = typer.Option(Path, callback=os.path.abspath), - package: bool = typer.Option(False, "--package"), + package: str = typer.Option(None, "--package"), ctx: typer.Context = typer.Option(list), ): if package: # todo - check if we are building in the right directory project_root = str(Path(config).parent) - wheel.create(project_root=project_root) + wheel.create_wheel(project_root=project_root, package=package) # create stepfunctions if requested # make sure you have quotes around the app arguments args = ["--app", f""" "python {config}" """, "-c", f"stage={stage}"] @@ -41,7 +41,7 @@ def deploy( context_settings={"allow_extra_args": True, "ignore_unknown_options": True} ) def synthesize( - stage: str = typer.Option(...), + stage: str = typer.Option(None), config: str = typer.Option(Path, callback=os.path.abspath), ctx: typer.Context = typer.Option(list), ): @@ -54,7 +54,7 @@ def synthesize( context_settings={"allow_extra_args": True, "ignore_unknown_options": True} ) def destroy( - stage: str = typer.Option(...), + stage: str = typer.Option(None), config: str = typer.Option(Path, callback=os.path.abspath), ctx: typer.Context = typer.Option(list), ): diff --git a/datajob/datajob_stack.py b/datajob/datajob_stack.py index e63655d..b1d831f 100644 --- a/datajob/datajob_stack.py +++ b/datajob/datajob_stack.py @@ -93,19 +93,17 @@ def create_resources(self): def get_stage(self, stage): """get the stage parameter and return a default if not found.""" try: - if stage == "None": - logger.debug("no stage parameter is passed via the cli.") + if stage == "None" or not stage: + logger.debug( + "No stage is passed to datajob stack, taking the default one." + ) return DataJobStack.DEFAULT_STAGE - if stage: + elif stage: logger.debug( "a stage parameter is passed via the cli or via the datajob stack configuration file." ) return stage - if stage is None: - logger.debug( - "No stage is passed to datajob stack, taking the default one." - ) - return self.get_context_parameter("stage") + except ValueError: return DataJobStack.DEFAULT_STAGE diff --git a/datajob/package/wheel.py b/datajob/package/wheel.py index a3e8264..e326a2a 100644 --- a/datajob/package/wheel.py +++ b/datajob/package/wheel.py @@ -9,21 +9,61 @@ class DatajobPackageWheelError(Exception): """any exception occuring when constructing a wheel in data job context.""" -def create(project_root): - """launch a subprocess to built a wheel. - todo - use the setuptools/disttools api to create a setup.py. - relying on a subprocess feels dangerous. +def create_wheel(project_root: str, package: str) -> None: + """ + Select the function to build a wheel based on the argument provided with --package. + At the time of writing the arguments can be setuppy or poetry. + :param project_root: the path to the root of your project. + :param package: the tool you want to use to build your wheel using (setuppy, poetry, ...) + :return: None + """ + wheel_functions = {"setuppy": _setuppy_wheel, "poetry": _poetry_wheel} + wheel_functions[package](project_root) + + +def _setuppy_wheel(project_root: str) -> None: + """ + build a wheel for your project using poetry. + :param project_root: the path to the root of your project. + :return: None """ setup_py_file = Path(project_root, "setup.py") if setup_py_file.is_file(): logger.debug(f"found a setup.py file in {project_root}") - logger.debug("creating wheel for glue job") cmd = f"cd {project_root}; python setup.py bdist_wheel" - print(f"wheel command: {cmd}") - # todo - shell=True is not secure - subprocess.check_call(shlex.split(cmd)) + _call_create_wheel_command(cmd=cmd) else: raise DatajobPackageWheelError( f"no setup.py file detected in project root {project_root}. " f"Hence we cannot create a python wheel for this project" ) + + +def _poetry_wheel(project_root: str) -> None: + """ + build a wheel for your project using poetry. + :param project_root: the path to the root of your project. + :return: None + """ + poetry_file = Path(project_root, "pyproject.toml") + if poetry_file.is_file(): + logger.debug(f"found a pyproject.toml file in {project_root}") + cmd = f"cd {project_root}; poetry build" + _call_create_wheel_command(cmd=cmd) + else: + raise DatajobPackageWheelError( + f"no pyproject.toml file detected in project root {project_root}. " + f"Hence we cannot create a python wheel for this project" + ) + + +def _call_create_wheel_command(cmd: str) -> None: + """ + shell out and call the command to create the wheel. + :param cmd: the command to create a wheel + :return: None + """ + logger.debug("creating wheel") + print(f"wheel command: {cmd}") + # todo - shell=True is not secure + subprocess.call(cmd, shell=True) diff --git a/datajob_tests/datajob_test.py b/datajob_tests/datajob_test.py index f35e6d4..328a3af 100644 --- a/datajob_tests/datajob_test.py +++ b/datajob_tests/datajob_test.py @@ -35,7 +35,7 @@ def test_datajob_deploy_cli_runs_with_unknown_args_successfully(self, m_call_cdk ) self.assertEqual(result.exit_code, 0) - @patch("datajob.package.wheel.create") + @patch("datajob.package.wheel.create_wheel") @patch("datajob.datajob.call_cdk") def test_datajob_deploy_cli_runs_with_project_root_successfully( self, m_call_cdk, m_create_wheel @@ -49,6 +49,43 @@ def test_datajob_deploy_cli_runs_with_project_root_successfully( "--stage", "some-stage", "--package", + "poetry", + ], + ) + self.assertEqual(result.exit_code, 0) + self.assertEqual(m_create_wheel.call_count, 1) + + @patch("datajob.package.wheel._poetry_wheel") + @patch("datajob.datajob.call_cdk") + def test_datajob_deploy_with_package_poetry(self, m_call_cdk, m_create_wheel): + result = self.runner.invoke( + datajob.app, + [ + "deploy", + "--config", + "some_config.py", + "--stage", + "some-stage", + "--package", + "poetry", + ], + ) + self.assertEqual(result.exit_code, 0) + self.assertEqual(m_create_wheel.call_count, 1) + + @patch("datajob.package.wheel._setuppy_wheel") + @patch("datajob.datajob.call_cdk") + def test_datajob_deploy_with_package_setuppy(self, m_call_cdk, m_create_wheel): + result = self.runner.invoke( + datajob.app, + [ + "deploy", + "--config", + "some_config.py", + "--stage", + "some-stage", + "--package", + "setuppy", ], ) self.assertEqual(result.exit_code, 0) diff --git a/examples/data_pipeline_with_packaged_project/README.md b/examples/data_pipeline_with_packaged_project/README.md index e490d6c..808fcde 100644 --- a/examples/data_pipeline_with_packaged_project/README.md +++ b/examples/data_pipeline_with_packaged_project/README.md @@ -20,4 +20,7 @@ Make sure you have configured a `setup.py` in the root of your poject. export AWS_DEFAULT_ACCOUNT=my-account-number export AWS_PROFILE=my-profile cd examples/data_pipeline_with_packaged_project - datajob deploy --stage dev --config datajob_stack.py --package + # if you want to use poetry to create a wheel + datajob deploy --config datajob_stack.py --package poetry + # if you want to create a wheel from setup.py + datajob deploy --config datajob_stack.py --package setuppy diff --git a/pyproject.toml b/pyproject.toml index 9f9ed74..feb1316 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datajob" -version = "0.4.0" +version = "0.5.0" description = "Build and deploy a serverless data pipeline with no effort on AWS." authors = ["Vincent Claes "] license = "Apache Software License (Apache 2.0)"