Dynamic Partitions on daily schedule

[varunp2k]

There are a list of IDs I need to run jobs daily for.
Currently I have a single dimension Dynamic partition setup. Issue here is that it will not run daily

ids_partitions_def = DynamicPartitionsDefinition(name="ids")


def _make_config_for_partition(id: str) -> dict:
    return {"ops": {"init_daily_dag": {"config": {"id": dw_id}}}}


@job(
    partitions_def=ids_partitions_def,
    config=PartitionedConfig(
        partitions_def=ids_partitions_def,
        run_config_for_partition_key_fn=_make_config_for_partition,
    ),
)
def data_pipeline():
    """Job for the entire data pipeline."""
    config = ops.init_daily_dag()
    out = ops.etl1(dp_config=config)
    out = ops.etl2dp_config=config, phantom_dep=out)
    ops.etl3(dp_config=config, phantom_dep=out)

I want to run these partitions everyday, found out about Multi Dimension Partitions. Am not sure if this will help my case. Am open to suggestions. Can I do something like this:

ids_partitions_def= DynamicPartitionsDefinition(name="ids")

daily_dw_def = MultiPartitionsDefinition(
  {
         "date": DailyPartitionsDefinition(start_date="2023-12-12"),
         "dws": dws_partitions_def,
     }
)


def _make_config_for_partition(d: str) -> dict:
 return {"ops": {"init_daily_dag": {"config": {"id": id}}}}


@job(
 partitions_def=daily_dw_def,
 config=PartitionedConfig(
     partitions_def=dws_partitions_def,
     run_config_for_partition_key_fn=_make_config_for_partition,
 ),
)
def data_pipeline():
 """Job for the entire data pipeline."""
 config = ops.init_daily_dag()
 out = ops.etl1(dp_config=config)
 out = ops.etl2dp_config=config, phantom_dep=out)
 ops.etl3dp_config=config, phantom_dep=out)

[sryza]

@varunp2k does this answer your question? #15532