Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

example-show-changed-assets #70

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
7 changes: 7 additions & 0 deletions dbt_project/models/ANALYTICS/location_stats.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
select
order_date,
state,
count(*) as n_orders,
sum(order_total) as total_revenue
from {{ ref("orders_augmented") }}
group by 1, 2
1 change: 1 addition & 0 deletions dbt_project/models/CLEANED/orders_cleaned.sql
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ select
sku,
dt,
cast(dt as datetime) as order_date,
date_trunc('month', order_date) as order_month
cnolanminich marked this conversation as resolved.
Show resolved Hide resolved
quantity * purchase_price as order_total
from {{ source("RAW_DATA", "orders") }}
{% if is_incremental() %}
Expand Down
3 changes: 2 additions & 1 deletion hooli-demo-assets/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@ WORKDIR /opt/dagster/app

ADD . .

RUN pip install -e .
RUN python -m pip install -U uv
RUN uv pip install --system -e .
3 changes: 2 additions & 1 deletion hooli_basics/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@ WORKDIR /opt/dagster/app

ADD . .

RUN pip install -r requirements.txt
RUN python -m pip install -U uv
RUN uv pip install --system -r requirements.txt
3 changes: 2 additions & 1 deletion hooli_batch_enrichment/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@ WORKDIR /opt/dagster/app

ADD . .

RUN pip install -e .
RUN python -m pip install -U uv
RUN uv pip install --system -e .
2 changes: 1 addition & 1 deletion hooli_data_eng/assets/dbt_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def _process_partitioned_dbt_assets(context: OpExecutionContext, dbt: DbtCliReso

@dbt_assets(
manifest=DBT_MANIFEST,
select="orders_cleaned users_cleaned orders_augmented",
select="orders_cleaned users_cleaned orders_augmented location_stats",
partitions_def=daily_partitions,
dagster_dbt_translator=CustomDagsterDbtTranslator(settings=DagsterDbtTranslatorSettings(enable_asset_checks=True)),
backfill_policy=BackfillPolicy.single_run(),
Expand Down
3 changes: 2 additions & 1 deletion hooli_data_eng/assets/raw_data/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ def check_users(context, users: pd.DataFrame):
backoff=Backoff.LINEAR,
jitter=Jitter.FULL
),
backfill_policy=BackfillPolicy.single_run()
backfill_policy=BackfillPolicy.single_run(),
code_version="1"
)
def orders(context, api: RawDataAPI) -> pd.DataFrame:
"""A table containing all orders that have been placed"""
Expand Down
2 changes: 1 addition & 1 deletion hooli_data_eng/resources/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def get_orders(self, datetime_to_process):

# random order data returned, see utils.py
json = random_data(
extra_columns={"order_id": str, "quantity": int, "purchase_price": float, "sku": str},
extra_columns={"order_id": str, "order_desc": str, "quantity": int, "purchase_price": float, "sku": str},
n = 10,
filter_date=datetime_to_process
).to_json()
Expand Down
3 changes: 2 additions & 1 deletion hooli_snowflake_insights/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@ RUN python -m pip install git+https://github.com/wbond/oscrypto.git@d5f3437ed242

ADD . .

RUN pip install -r requirements.txt
RUN python -m pip install -U uv
RUN uv pip install --system -r requirements.txt
Loading