Replace apx with more general name

openstef/feature_engineering/feature_applicator.py

@@ -24,7 +24,9 @@
     remove_non_requested_feature_columns,
 )
 
-LATENCY_CONFIG = {"APX": 24}  # A specific latency is part of a specific feature.
+LATENCY_CONFIG = {
+    "day_ahead_electricity_price": 24
+}  # A specific latency is part of a specific feature.
 
 
 class AbstractFeatureApplicator(ABC):
@@ -94,7 +96,7 @@ def add_features(
                 if not specified a default location is used
             latency_config: (Optional) Invalidate certain features that are not
                 available for a specific horizon due to data latency. Defaults to
-                ``{"APX": 24}``.
+                ``{"day_ahead_electricity_price": 24}``.
 
         Returns:
             Input DataFrame with an extra column for every added feature and sorted on the datetime index.

openstef/model/regressors/linear_quantile.py

@@ -93,7 +93,7 @@ def __init__(
             )
 
         if clipped_features is None:
-            clipped_features = ["APX"]
+            clipped_features = ["day_ahead_electricity_price"]
 
         self.quantiles = quantiles
         self.alpha = alpha

openstef/tasks/create_components_forecast.py

@@ -6,7 +6,7 @@
 This code assumes trained models are available from the persistent storage.
 If these are not available run model_train.py to train all models.
 To provide the prognoses the following steps are carried out:
-  1. Get historic training data (TDCV, Load, Weather and APX price data)
+  1. Get historic training data (TDCV, Load, Weather and day_ahead_electricity_price price data)
   2. Apply features
   3. Load model
   4. Make component prediction

openstef/tasks/create_forecast.py

@@ -6,7 +6,7 @@
 This code assumes trained models are available from the persistent storage. If these
 are not available run model_train.py to train all models.
 To provide the prognoses the folowing steps are carried out:
-  1. Get historic training data (TDCV, Load, Weather and APX price data)
+  1. Get historic training data (TDCV, Load, Weather and day_ahead_electricity_price price data)
   2. Apply features
   3. Load model
   4. Make prediction
@@ -24,7 +24,7 @@
 from pathlib import Path
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import ModelType, PipelineType
+from openstef.enums import BiddingZone, ModelType, PipelineType
 from openstef.exceptions import InputDataOngoingZeroFlatlinerError
 from openstef.pipeline.create_forecast import create_forecast_pipeline
 from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
@@ -82,8 +82,13 @@ def create_forecast_task(
         location=[pj["lat"], pj["lon"]],
         datetime_start=datetime_start,
         datetime_end=datetime_end,
+        market_price=pj.electricity_bidding_zone.value,
     )
 
+    # Add APX price to the input data for backward compatibility,remove this line when all models are retrained
+    if pj.electricity_bidding_zone == BiddingZone.NL:
+        input_data["APX"] = input_data["day_ahead_electricity_price"]
+
     try:
         # Make forecast with the forecast pipeline
         forecast = create_forecast_pipeline(

openstef/tasks/train_model.py

@@ -4,7 +4,7 @@
 """This module contains the CRON job that is periodically executed to retrain the prognosis models.
 
 For this the folowing steps are caried out:
-  1. Get historic training data (TDCV, Load, Weather and APX price data)
+  1. Get historic training data (TDCV, Load, Weather and day_ahead_electricity_price price data)
   2. Apply features
   3. Train and Test the new model
   4. Check if new model performs better than the old model

openstef/validation/validation.py

@@ -313,9 +313,9 @@ def calc_completeness_dataframe(
                     expected_numbers_timedelayed=value,
                 )
 
-        # Correct for APX being only expected to be available up to 24h
-        if "APX" in non_na_count.index:
-            non_na_count["APX"] += max([len(df) - 96, 0])
+        # Correct for day_ahead_electricity_price being only expected to be available up to 24h
+        if "day_ahead_electricity_price" in non_na_count.index:
+            non_na_count["day_ahead_electricity_price"] += max([len(df) - 96, 0])
 
         completeness_per_column_dataframe = non_na_count / (len(df))
 

test/unit/data/fallback_index.csv

@@ -1,4 +1,4 @@
-index,APX,clouds,radiation,temp,winddeg,windspeed,pressure,humidity,rain,mxlD,snowDepth,clearSky_ulf,clearSky_dlf,windspeed_100m,sjv_E1A,sjv_E1B,sjv_E1C,sjv_E2A,sjv_E2B,sjv_E3A,sjv_E3B,sjv_E3C,sjv_E3D,sjv_E4A,T-900min,T-780min,T-15min,T-660min,T-540min,T-30min,T-1440min,T-420min,T-1320min,T-300min,T-45min,T-1200min,T-180min,T-1080min,T-60min,T-960min,T-2880min,T-840min,T-720min,T-600min,T-480min,T-1380min,T-360min,T-1260min,T-240min,T-2160min,T-1140min,T-120min,T-1020min,T-1d,T-2d,T-3d,T-4d,T-5d,T-6d,T-7d,T-8d,T-9d,T-10d,T-11d,T-12d,T-13d,T-14d,IsWeekendDay,IsWeekDay,IsMonday,IsTuesday,IsWednesday,IsThursday,IsFriday,IsSaturday,IsSunday,Month,Quarter,IsChristmas,Is00Hour,Is01Hour,Is02Hour,Is03Hour,Is04Hour,Is05Hour,Is06Hour,Is07Hour,Is08Hour,Is09Hour,Is10Hour,Is11Hour,Is12Hour,Is13Hour,Is14Hour,Is15Hour,Is16Hour,Is17Hour,Is18Hour,Is19Hour,Is20Hour,Is21Hour,Is22Hour,Is23Hour,windspeed_100mExtrapolated,windPowerFit_extrapolated,windpowerFit_harm_arome,saturation_pressure,vapour_pressure,dewpoint,air_density,dtemp_quarter,dtemp_hour,dtemp_day,dtemp_week,dwindspeed_quarter,dwindspeed_hour,dwindspeed_day,dwindspeed_week,dwindspeed_100m_quarter,dwindspeed_100m_hour,dwindspeed_100m_day,dwindspeed_100m_week,dwinddeg_quarter,dwinddeg_hour,dwinddeg_day,dwinddeg_week,dpressure_quarter,dpressure_hour,dpressure_day,dpressure_week,dhumidity_quarter,dhumidity_hour,dhumidity_day,dhumidity_week,dair_density_quarter,dair_density_hour,dair_density_day,dair_density_week
+index,day_ahead_electricity_price,clouds,radiation,temp,winddeg,windspeed,pressure,humidity,rain,mxlD,snowDepth,clearSky_ulf,clearSky_dlf,windspeed_100m,sjv_E1A,sjv_E1B,sjv_E1C,sjv_E2A,sjv_E2B,sjv_E3A,sjv_E3B,sjv_E3C,sjv_E3D,sjv_E4A,T-900min,T-780min,T-15min,T-660min,T-540min,T-30min,T-1440min,T-420min,T-1320min,T-300min,T-45min,T-1200min,T-180min,T-1080min,T-60min,T-960min,T-2880min,T-840min,T-720min,T-600min,T-480min,T-1380min,T-360min,T-1260min,T-240min,T-2160min,T-1140min,T-120min,T-1020min,T-1d,T-2d,T-3d,T-4d,T-5d,T-6d,T-7d,T-8d,T-9d,T-10d,T-11d,T-12d,T-13d,T-14d,IsWeekendDay,IsWeekDay,IsMonday,IsTuesday,IsWednesday,IsThursday,IsFriday,IsSaturday,IsSunday,Month,Quarter,IsChristmas,Is00Hour,Is01Hour,Is02Hour,Is03Hour,Is04Hour,Is05Hour,Is06Hour,Is07Hour,Is08Hour,Is09Hour,Is10Hour,Is11Hour,Is12Hour,Is13Hour,Is14Hour,Is15Hour,Is16Hour,Is17Hour,Is18Hour,Is19Hour,Is20Hour,Is21Hour,Is22Hour,Is23Hour,windspeed_100mExtrapolated,windPowerFit_extrapolated,windpowerFit_harm_arome,saturation_pressure,vapour_pressure,dewpoint,air_density,dtemp_quarter,dtemp_hour,dtemp_day,dtemp_week,dwindspeed_quarter,dwindspeed_hour,dwindspeed_day,dwindspeed_week,dwindspeed_100m_quarter,dwindspeed_100m_hour,dwindspeed_100m_day,dwindspeed_100m_week,dwinddeg_quarter,dwinddeg_hour,dwinddeg_day,dwinddeg_week,dpressure_quarter,dpressure_hour,dpressure_day,dpressure_week,dhumidity_quarter,dhumidity_hour,dhumidity_day,dhumidity_week,dair_density_quarter,dair_density_hour,dair_density_day,dair_density_week
 2020-01-02 14:15:00+00:00,,100.0,87307.71875,5.135711669921875,330.55521392822266,,102121.5546875,0.9630825370550156,,1217.462875366211,0.0,0.8048269748687744,9.19796633720398,9.129966259002686,3.578e-05,3.855e-05,3.476e-05,2.962e-05,3.054e-05,1.816e-05,1.979e-05,1.979e-05,2.571e-05,6.024e-05,7.873333333333333,31.686666666666667,37.6,32.15,33.92333333333333,38.04666666666667,34.34666666666667,37.843333333333334,31.97,38.446666666666665,38.29,8.839999999999998,38.169999999999995,7.466666666666665,38.016666666666666,8.076666666666668,34.53333333333333,23.87333333333333,31.9,32.36,35.38333333333333,32.876666666666665,37.74333333333333,8.870000000000001,38.413333333333334,31.836666666666666,8.083333333333334,38.14666666666667,8.14,34.34666666666667,34.53333333333333,37.53,38.35666666666667,33.836666666666666,34.85333333333333,34.28333333333333,33.74,37.62,37.730000000000004,33.94333333333333,34.45333333333333,39.166666666666664,38.093333333333334,False,True,False,False,False,True,False,False,False,1,1,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,True,False,False,False,False,False,False,False,False,False,,,0.6690336382064527,8.815313721120072,8.489874703472209,4.597185695949985,1.1554210812714338,0.01153564453125,0.19317626953125,1.1988525390625,0.09820556640625,,,,,0.2862677574157715,0.5278754234313965,4.101582050323486,1.275410771369934,-0.28444671630859375,-6.377281188964844,323.67613220214844,274.4930648803711,-14.078125,-65.763671875,-915.0546875,105.66015625,0.0008316487073898315,-0.006531417369842529,0.09409430623054504,0.12278266251087189,-0.00020725164089374637,-0.0015474556236589887,-0.015403507671178485,0.0007831801747628653
 2020-01-02 14:30:00+00:00,,100.0,65007.375,5.147247314453125,330.27076721191406,,102107.4765625,0.9639141857624054,,1001.8500671386719,0.0,1.0695586204528809,9.464561939239502,9.416234016418457,3.578e-05,3.855e-05,3.476e-05,2.962e-05,3.054e-05,1.816e-05,1.979e-05,1.979e-05,2.571e-05,6.024e-05,7.913333333333334,31.50333333333333,37.550000000000004,31.939999999999998,34.61666666666667,37.6,34.32666666666667,37.97,31.72666666666666,38.46,38.04666666666667,8.88666666666667,38.16666666666667,7.233333333333336,38.29,7.933333333333335,34.88333333333333,31.883333333333333,32.093333333333334,32.379999999999995,35.736666666666665,32.66,37.92,8.803333333333335,38.70333333333333,31.833333333333332,7.806666666666665,38.34,7.936666666666667,34.32666666666667,34.88333333333333,37.736666666666665,38.39,33.913333333333334,35.49333333333333,34.74666666666667,33.876666666666665,37.403333333333336,37.443333333333335,33.05,34.586666666666666,39.263333333333335,37.89666666666666,False,True,False,False,False,True,False,False,False,1,1,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,True,False,False,False,False,False,False,False,False,False,,,0.7096942131928761,8.822403770790025,8.504040147188242,4.621002789735361,1.1552138467408297,0.01153564453125,0.1441650390625,1.455322265625,0.1512451171875,,,,,0.2862677574157715,0.733607292175293,3.968555450439453,1.4573724269866943,-0.28444671630859375,-4.6307830810546875,320.1625213623047,269.04998779296875,-14.078125,-62.61328125,-930.859375,109.53125,0.0008316487073898315,-0.003245413303375244,0.08826178312301636,0.1157674491405487,-0.00020723453060411678,-0.001307760801531499,-0.016666548757323207,0.0006070461433393959
 2020-01-02 14:45:00+00:00,,100.0,42707.03125,5.158782958984375,329.98632049560547,,102093.3984375,0.9647458344697952,,786.2372589111328,0.0,1.3342902660369873,9.731157541275024,9.702501773834229,3.578e-05,3.855e-05,3.476e-05,2.962e-05,3.054e-05,1.816e-05,1.979e-05,1.979e-05,2.571e-05,6.024e-05,7.803333333333331,31.623333333333335,37.550000000000004,31.78333333333333,35.17666666666667,37.550000000000004,34.42666666666666,37.92333333333333,31.923333333333332,38.36,37.6,8.456666666666665,38.236666666666665,7.136666666666663,38.04666666666667,7.909999999999998,35.18666666666667,31.82,31.973333333333336,32.85333333333333,36.52333333333333,33.02333333333333,38.54333333333333,8.626666666666663,38.61333333333334,31.660000000000004,7.739999999999998,38.00333333333333,7.790000000000003,34.42666666666666,35.18666666666667,38.123333333333335,38.309999999999995,33.92,35.52333333333333,35.17,33.78333333333333,37.699999999999996,37.406666666666666,32.68666666666666,34.583333333333336,39.21,38.69333333333333,False,True,False,False,False,True,False,False,False,1,1,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,True,False,False,False,False,False,False,False,False,False,,,0.7472466446725978,8.82949885677055,8.518222242525209,4.644812793077298,1.155006629318322,0.01153564453125,0.09515380859375,1.7117919921875,0.20428466796875,,,,,0.2862677574157715,0.9393391609191895,3.83552885055542,1.6393340826034546,-0.28444671630859375,-2.8842849731445312,316.64891052246094,263.6069107055664,-14.078125,-59.462890625,-946.6640625,113.40234375,0.0008316487073898315,4.0590763092041016e-05,0.08242926001548767,0.10875223577022552,-0.00020721742250762176,-0.001068243604647856,-0.017931442600870984,0.00043093703837282504

test/unit/data/input_data.csv

@@ -1,4 +1,4 @@
-index,load,APX,clearSky_dlf,clearSky_ulf,clouds,humidity,mxlD,pressure,radiation,rain,snowDepth,temp,winddeg,windspeed,windspeed_100m,sjv_E1A,sjv_E1B,sjv_E1C,sjv_E2A,sjv_E2B,sjv_E3A,sjv_E3B,sjv_E3C,sjv_E3D,sjv_E4A
+index,load,day_ahead_electricity_price,clearSky_dlf,clearSky_ulf,clouds,humidity,mxlD,pressure,radiation,rain,snowDepth,temp,winddeg,windspeed,windspeed_100m,sjv_E1A,sjv_E1B,sjv_E1C,sjv_E2A,sjv_E2B,sjv_E3A,sjv_E3B,sjv_E3C,sjv_E3D,sjv_E4A
 2019-11-28 14:45:00+00:00,-16.17,,,,,,,,,,,,,,,,,,,,,,,,
 2019-11-28 15:00:00+00:00,-12.986666666666666,47.7,-5.5954742431640625,9.330299377441406,100.0,0.9209308624267578,470.93084716796875,99348.0,5006.3125,0.48,0.0,9.849761962890625,76.83540344238281,4.125,7.760239601135254,3.101e-05,3.238e-05,3.25e-05,3.584e-05,3.673e-05,5.097e-05,4.572e-05,4.572e-05,3.118e-05,0.0
 2019-11-28 15:15:00+00:00,-11.153333333333334,47.7,-5.878936767578125,8.157358407974243,100.0,0.9186419546604156,459.35560607910156,99377.56640625,3764.171875,0.505,0.0,9.690399169921875,70.78819465637207,4.05625,7.924123287200928,3.223e-05,3.379e-05,3.402e-05,3.707e-05,3.809e-05,4.919e-05,4.565e-05,4.565e-05,3.136e-05,0.0

test/unit/data/input_data_multi_horizon_features.csv

@@ -1,4 +1,4 @@
-index,load,APX,IsSunday,IsWeekDay,IsWeekendDay,Month,Quarter,T-1020min,T-1080min,T-10d,T-1140min,T-11d,T-1200min,T-120min,T-1260min,T-12d,T-1320min,T-1380min,T-13d,T-14d,T-15min,T-180min,T-1d,T-240min,T-2d,T-300min,T-30min,T-360min,T-3d,T-420min,T-45min,T-480min,T-4d,T-540min,T-5d,T-600min,T-60min,T-660min,T-6d,T-720min,T-780min,T-7d,T-840min,T-8d,T-900min,T-960min,T-9d,air_density,clearSky_dlf,clearSky_ulf,clouds,day0fweek_cosine,day0fweek_sine,daylight_continuous,dewpoint,dni,gti,humidity,is_bevrijdingsdag,is_bouwvakmidden,is_bouwvaknoord,is_bouwvakzuid,is_bridgeday,is_eerste_kerstdag,is_eerste_paasdag,is_eerste_pinksterdag,is_goede_vrijdag,is_hemelvaart,is_herfstvakantiemidden,is_herfstvakantienoord,is_herfstvakantiezuid,is_kerstvakantie,is_koningsdag,is_meivakantie,is_national_holiday,is_nieuwjaarsdag,is_schoolholiday,is_tweede_kerstdag,is_tweede_paasdag,is_tweede_pinksterdag,is_voorjaarsvakantiemidden,is_voorjaarsvakantienoord,is_voorjaarsvakantiezuid,is_zomervakantiemidden,is_zomervakantienoord,is_zomervakantiezuid,month_cosine,month_sine,mxlD,pressure,radiation,rain,saturation_pressure,season_cosine,season_sine,sjv_E1A,sjv_E1B,sjv_E1C,sjv_E2A,sjv_E2B,sjv_E3A,sjv_E3B,sjv_E3C,sjv_E3D,sjv_E4A,snowDepth,temp,time0fday_cosine,time0fday_sine,vapour_pressure,windPowerFit_extrapolated,winddeg,windpowerFit_harm_arome,windspeed,windspeed_100m,windspeed_100mExtrapolated,horizon
+index,load,day_ahead_electricity_price,IsSunday,IsWeekDay,IsWeekendDay,Month,Quarter,T-1020min,T-1080min,T-10d,T-1140min,T-11d,T-1200min,T-120min,T-1260min,T-12d,T-1320min,T-1380min,T-13d,T-14d,T-15min,T-180min,T-1d,T-240min,T-2d,T-300min,T-30min,T-360min,T-3d,T-420min,T-45min,T-480min,T-4d,T-540min,T-5d,T-600min,T-60min,T-660min,T-6d,T-720min,T-780min,T-7d,T-840min,T-8d,T-900min,T-960min,T-9d,air_density,clearSky_dlf,clearSky_ulf,clouds,day0fweek_cosine,day0fweek_sine,daylight_continuous,dewpoint,dni,gti,humidity,is_bevrijdingsdag,is_bouwvakmidden,is_bouwvaknoord,is_bouwvakzuid,is_bridgeday,is_eerste_kerstdag,is_eerste_paasdag,is_eerste_pinksterdag,is_goede_vrijdag,is_hemelvaart,is_herfstvakantiemidden,is_herfstvakantienoord,is_herfstvakantiezuid,is_kerstvakantie,is_koningsdag,is_meivakantie,is_national_holiday,is_nieuwjaarsdag,is_schoolholiday,is_tweede_kerstdag,is_tweede_paasdag,is_tweede_pinksterdag,is_voorjaarsvakantiemidden,is_voorjaarsvakantienoord,is_voorjaarsvakantiezuid,is_zomervakantiemidden,is_zomervakantienoord,is_zomervakantiezuid,month_cosine,month_sine,mxlD,pressure,radiation,rain,saturation_pressure,season_cosine,season_sine,sjv_E1A,sjv_E1B,sjv_E1C,sjv_E2A,sjv_E2B,sjv_E3A,sjv_E3B,sjv_E3C,sjv_E3D,sjv_E4A,snowDepth,temp,time0fday_cosine,time0fday_sine,vapour_pressure,windPowerFit_extrapolated,winddeg,windpowerFit_harm_arome,windspeed,windspeed_100m,windspeed_100mExtrapolated,horizon
 2019-11-28 14:45:00+00:00,-16.17,,False,True,False,11,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-0.900968867902419,0.43388373911755823,-0.2847783939631754,,0.0,,,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,0.8660254037844384,-0.5000000000000004,,,,,,0.8408305333122887,-0.5412984521036173,,,,,,,,,,,,,-0.7518398074789777,-0.6593458151000685,,,,,,,,0.25
 2019-11-28 15:00:00+00:00,-12.986666666666666,47.7,False,True,False,11,4,,,,,,,,,,,,,,-16.17,,,,,,,,,,,,,,,,,,,,,,,,,,,1.1053045611739607,-5.595474243164063,9.330299377441406,100.0,-0.900968867902419,0.43388373911755823,-0.37538383477921167,8.62815733471107,0.0,7.517884457614255,0.9209308624267578,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,0.8660254037844384,-0.5000000000000004,470.9308471679688,99348.0,5006.3125,0.48,12.168672929456537,0.8408305333122887,-0.5412984521036173,3.101e-05,3.238e-05,3.25e-05,3.584e-05,3.673e-05,5.097e-05,4.572e-05,4.572e-05,3.118e-05,0.0,0.0,9.849761962890623,-0.7071067811865477,-0.7071067811865475,11.20650645551355,0.17488419370941155,76.83540344238281,0.44876028633199644,4.125,7.760239601135254,5.733554603755082,0.25
 2019-11-28 15:15:00+00:00,-11.153333333333334,47.7,False,True,False,11,4,,,,,,,,,,,,,,-12.986666666666666,,,,,,-16.17,,,,,,,,,,,,,,,,,,,,,1.1062570921140946,-5.878936767578125,8.157358407974243,100.0,-0.900968867902419,0.43388373911755823,-0.46065954378253987,8.433669711208791,0.0,2.4575772586080267,0.9186419546604156,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,0.8660254037844384,-0.5000000000000004,459.3556060791016,99377.56640625,3764.171875,0.505,12.039163178537217,0.8408305333122887,-0.5412984521036173,3.223e-05,3.379e-05,3.402e-05,3.707e-05,3.809e-05,4.919e-05,4.565e-05,4.565e-05,3.136e-05,0.0,0.0,9.690399169921877,-0.6593458151000694,-0.7518398074789769,11.059680394807131,0.1659161937158718,70.78819465637207,0.4758033809928022,4.05625,7.924123287200928,5.637995360359165,0.25