diff --git a/.github/workflows/deploy-dagster-cloud.yml b/.github/workflows/deploy-dagster-cloud.yml index 8a707de3..6a44f1db 100644 --- a/.github/workflows/deploy-dagster-cloud.yml +++ b/.github/workflows/deploy-dagster-cloud.yml @@ -76,7 +76,11 @@ jobs: # Build 'data-eng-pipeline' code location - name: Build dbt manifest for data-eng-pipeline if: steps.prerun.outputs.result != 'skip' - run: make manifest + run: | + pip install pyopenssl --upgrade; + pip install click --upgrade; + pip install dbt-core dbt-duckdb dbt-snowflake; + make manifest # Copy production manifest.json to S3 on merge - name: Upload dbt manifest to S3 diff --git a/Dockerfile b/Dockerfile index 6cc775a7..a6dde204 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.8-slim +FROM python:3.11-slim WORKDIR /opt/dagster/app -RUN apt-get update && apt-get install -y git +RUN apt-get update && apt-get install -y git gcc RUN apt install -y default-jre @@ -10,8 +10,8 @@ RUN python -m pip install -U pip # libcrypto fix oct 2023; should be able to remove sometime after that RUN python -m pip uninstall oscrypto -y RUN python -m pip install git+https://github.com/wbond/oscrypto.git@d5f3437ed24257895ae1edd9e503cfb352e635a8 - +RUN python -m pip install -U uv ADD . . -RUN pip install -e . +RUN uv pip install --system -e . diff --git a/Makefile b/Makefile index c3af9fbd..b78683f5 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,6 @@ clean: rm -rf ~/.dagster_home; mkdir ~/.dagster_home; cp dagster.yaml ~/.dagster_home/dagster.yaml manifest: - pip install pyopenssl --upgrade; - pip install click --upgrade; - pip install dbt-core dbt-duckdb dbt-snowflake; dbt parse --project-dir=dbt_project --profiles-dir=dbt_project/config --target BRANCH stateful_dev: clean manifest @@ -18,4 +15,4 @@ stateful_dev_prod: clean manifest export DAGSTER_HOME="~/.dagster_home"; export DAGSTER_CLOUD_DEPLOYMENT_NAME="data-eng-prod"; dagster dev dependencies: - pip install -e ".[dev]" + uv pip install -e ".[dev]" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 12f8bd8d..00000000 --- a/requirements.txt +++ /dev/null @@ -1,48 +0,0 @@ --e ./dagster/python_modules/dagit --e ./dagster/python_modules/dagster --e ./dagster/python_modules/libraries/dagster-airbyte --e ./dagster/python_modules/libraries/dagster-airflow --e ./dagster/python_modules/libraries/dagster-aws --e ./dagster/python_modules/libraries/dagster-azure --e ./dagster/python_modules/libraries/dagster-celery --e ./dagster/python_modules/libraries/dagster-celery-docker --e ./dagster/python_modules/libraries/dagster-celery-k8s --e ./dagster/python_modules/libraries/dagster-dask --e ./dagster/python_modules/libraries/dagster-databricks --e ./dagster/python_modules/libraries/dagster-datadog --e ./dagster/python_modules/libraries/dagster-datahub --e ./dagster/python_modules/libraries/dagster-dbt --e ./dagster/python_modules/libraries/dagster-docker --e ./dagster/python_modules/libraries/dagster-duckdb --e ./dagster/python_modules/libraries/dagster-duckdb-pandas --e ./dagster/python_modules/libraries/dagster-duckdb-pyspark --e ./dagster/python_modules/libraries/dagster-fivetran --e ./dagster/python_modules/libraries/dagster-gcp --e ./dagster/python_modules/libraries/dagster-github --e ./dagster/python_modules/dagster-graphql --e ./dagster/python_modules/libraries/dagster-k8s --e ./dagster/python_modules/libraries/dagster-mlflow --e ./dagster/python_modules/libraries/dagster-msteams --e ./dagster/python_modules/libraries/dagster-mysql --e ./dagster/python_modules/libraries/dagster-pagerduty --e ./dagster/python_modules/libraries/dagster-pandas --e ./dagster/python_modules/libraries/dagster-pandera --e ./dagster/python_modules/libraries/dagster-papertrail --e ./dagster/python_modules/libraries/dagster-postgres --e ./dagster/python_modules/libraries/dagster-prometheus --e ./dagster/python_modules/libraries/dagster-pyspark --e ./dagster/python_modules/libraries/dagster-shell --e ./dagster/python_modules/libraries/dagster-slack --e ./dagster/python_modules/libraries/dagster-snowflake --e ./dagster/python_modules/libraries/dagster-snowflake-pandas --e ./dagster/python_modules/libraries/dagster-spark --e ./dagster/python_modules/libraries/dagster-ssh --e ./dagster/python_modules/libraries/dagster-twilio --e ./dagster/python_modules/libraries/dagstermill --e ./internal/dagster-cloud/python_modules/dagster-cloud -dbt-duckdb -responses -plotnine -requests -gql -requests_toolbelt \ No newline at end of file diff --git a/setup.py b/setup.py index ef34d3ba..8de693cc 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ if __name__ == "__main__": setup( name="hooli_data_eng", + python_requires='<3.12', packages=find_packages(exclude=["hooli_data_eng_tests"]), package_data={"hooli_data_eng": ["dbt_project/*"]}, install_requires=[ @@ -33,4 +34,4 @@ "scikit-learn", ], extras_require={"dev": ["dagit", "pytest"]}, - ) + ) \ No newline at end of file