From 32026ec5014e2f34f2b4191904cf787be5574751 Mon Sep 17 00:00:00 2001 From: Martin Date: Wed, 21 Feb 2018 22:23:31 +0100 Subject: [PATCH 01/46] Replace part of timeseries script --- .../ego_dp_powerflow_timeseries_generator.py | 130 +++++++++++ .../ego_dp_powerflow_timeseries_generator.sql | 202 ------------------ 2 files changed, 130 insertions(+), 202 deletions(-) create mode 100644 dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py new file mode 100644 index 00000000..eec9e5f2 --- /dev/null +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -0,0 +1,130 @@ +""" +Assign German timeseries data from hidden renpassG!S schema to high voltage +powerflow. +""" + +__copyright__ = "ZNES Flensburg" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "wolfbunke" + +import pandas as pd +import numpby as np + +from dataprocessing.tools.io import oedb_session +from sqlalchemy.orm import sessionmaker +from sqlalchemy import MetaData, func +from sqlalchemy.ext.automap import automap_base + +from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator,\ + EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus + + +SCENARIOMAP = {'Status Quo': 43, 'NEP 2035': 41, 'eGo 100': 40} + +SOURCE_TO_FUEL = { + 1: 'gas', 2: 'lignite', 3: 'mixed_fuels', 4: 'oil', + 5: 'uranium', 6: 'biomass', 8: 'hard_coal', 9: 'run_of_river', 12: 'solar', + 13: 'wind', 14: 'geothermal', 15: 'other_non_renewable', 94: 'storage', + 95: 'load', 96: 'waste', 97: 'reservoir', 98: 'shortage', 99: 'excess'} + +TEMPID = 1 + +# get database connection +conn = oedb_session(section='test') + +Session = sessionmaker(bind=conn) +session = Session() + +meta = MetaData() +meta.bind = conn +meta.reflect(bind=conn, schema='calc_renpass_gis', + only=['renpass_gis_results']) + +# map to classes +Base = automap_base(metadata=meta) +Base.prepare() + +Results = Base.classes.renpass_gis_results + +############################################################################### + + +def _norm(x): + return x / x.sum() + +# delete all from model_draft.ego_grid_pf_hv_generator_pq_set +session.query(PqSet).delete() +session.commit() + +for scn_name, scn_nr in SCENARIOMAP.items(): + + # dataframe from model_draft.pf_generator_single + # with aggr_id, source, p_nom + filters = (Generator.scn_name == scn_name, Generator.aggr_id != None) + fields = [Generator.aggr_id, Generator.source, + func.sum(Generator.p_nom).label('summed_p_nom')] + grouper = Generator.aggr_id, Generator.source + query = session.query(*fields).filter(*filters).group_by(*grouper) + + generators = pd.read_sql(query.statement, query.session.bind) + + # create fraction of nominal power to total nominal power for each source + generators['fraction_of_total_p_nom'] = \ + generators.groupby('source')['summed_p_nom'].apply(_norm) + + # dataframe from calc_renpass_gis.renpass_gis_results + # optimization results to buses + # with obj_label, datetime, val + filters = (Results.obj_label.like('%DE%'), + ~Results.obj_label.like('%powerline%'), + Results.type == 'to_bus', + Results.scenario_id == scn_nr) + fields = Results.obj_label, Results.datetime, Results.val + query = session.query(*fields).filter(*filters) + + results = pd.read_sql(query.statement, query.session.bind) + + # map obj_label to corresponding source + results['source'] = None + for k, v in SOURCE_TO_FUEL.items(): + idx = results['obj_label'].str.contains(v) + results.loc[idx, 'source'] = k + + # aggregate by source and datetime + results = results.groupby(['source', 'datetime'], as_index=False).sum() + + # power generation timeseries for each source in list format + results_s = results.groupby('source')['val'].apply(np.array) + + # map corresponding timeseries with each generator multiplied by fraction + # of nominal power + generators['p_set'] = generators['source'].map(results_s) * \ + generators['fraction_of_total_p_nom'] + + # generators without p_set + ix = generators['p_set'].isnull() + print('Generators with sources {} have no p_sets assigned!'.format( + generators[ix]['source'].unique())) + generators.loc[ix, 'p_set'] = None + + # add columns + empty = ['q_set', 'p_min_pu', 'p_max_pu'] + + pqsets = pd.concat( + [generators[['aggr_id', 'p_set']], + pd.DataFrame(columns=empty)]) + + pqsets.loc[:, empty] = None + + # add scenario name and temporal id + pqsets['scn_name'] = scn_name + pqsets['temp_id'] = TEMPID + + # rename column aggr_id to generator_id + pqsets.rename(columns={'aggr_id': 'generator_id'}, inplace=True) + + # write to db + for i in pqsets.to_dict(orient='records'): + session.add(PqSet(**i)) + session.commit() diff --git a/dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql b/dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql index bf7d8b5d..9d0267c3 100644 --- a/dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql +++ b/dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql @@ -9,208 +9,6 @@ __author__ = "wolfbunke" TODO: storage in storage_pqset #1069 */ --- Status Quo --- aggregate nominal capacity on aggr_id FROM powerflow generators, keeping the source -DROP materialized view if EXISTS calc_renpass_gis.pf_pp_by_source_aggr_id; -CREATE materialized view calc_renpass_gis.pf_pp_by_source_aggr_id -AS -SELECT -SQ.aggr_id, SQ.source, SQ.p_nom / sum(SQ.p_nom) over (partition by SQ.source) AS fraction_of_installed -FROM - (SELECT - aggr_id, - source, - sum(p_nom) AS p_nom - FROM model_draft.ego_supply_pf_generator_single - WHERE scn_name = 'Status Quo' - AND aggr_id IS NOT NULL - GROUP BY aggr_id, source) SQ; - --- map renpassG!S power sources to pf generators, aggr on fuel types, neglect efficiency classes -DROP materialized view if EXISTS calc_renpass_gis.pp_feedin_by_pf_source; -CREATE materialized view calc_renpass_gis.pp_feedin_by_pf_source -AS -SELECT -SQ.source, SQ.datetime, sum(SQ.val) AS val -FROM - (SELECT - CASE - WHEN obj_label LIKE '%%gas%%' THEN 1 - when obj_label LIKE '%%lignite%%' THEN 2 - when obj_label LIKE '%%mixed_fuels%%' THEN 3 - when obj_label LIKE '%%oil%%' THEN 4 - when obj_label LIKE '%%uranium%%' THEN 5 - when obj_label LIKE '%%biomass%%' THEN 6 - when obj_label LIKE '%%hard_coal%%' THEN 8 - when obj_label LIKE '%%run_of_river%%' THEN 9 --- when obj_label LIKE '%%storage_phs%%' THEN 11 - when obj_label LIKE '%%solar%%' THEN 12 - when obj_label LIKE '%%wind%%' THEN 13 - END AS source, - bus_label, - obj_label, - type, - datetime, - val - FROM calc_renpass_gis.renpass_gis_results - -- conds - WHERE obj_label LIKE '%%DE%%' -- only Germany - AND obj_label not LIKE '%%powerline%%' -- without any powerlines - AND scenario_id = 43 - -- take only one flow (input), storage output flow seems to be the right one (?) - AND ((obj_label LIKE '%%storage%%' AND type = 'from_bus') or (obj_label not LIKE '%%storage%%' AND type = 'to_bus')) -) AS SQ -WHERE SQ.source IS not NULL -GROUP BY SQ.source, SQ.datetime; - --- -DELETE FROM model_draft.ego_grid_pf_hv_generator_pq_set; - --- construct array per aggr_id according to source timeseries -INSERT into model_draft.ego_grid_pf_hv_generator_pq_set (scn_name, generator_id, temp_id, p_set) -SELECT - 'Status Quo' AS scn_name, - A.aggr_id, - 1 AS temp_id, - array_agg(A.fraction_of_installed * B.val ORDER BY B.datetime) AS p_set - FROM calc_renpass_gis.pf_pp_by_source_aggr_id A, - calc_renpass_gis.pp_feedin_by_pf_source B -WHERE A.source = B.source -GROUP BY A.aggr_id; - --- NEP 2035 - --- aggregate nominal capacity on aggr_id FROM powerflow generators, keeping the source -DROP materialized view if EXISTS calc_renpass_gis.pf_pp_by_source_aggr_id; -CREATE materialized view calc_renpass_gis.pf_pp_by_source_aggr_id -AS -SELECT -NEP.aggr_id, NEP.source, NEP.p_nom / sum(NEP.p_nom) over (partition by NEP.source) AS fraction_of_installed -FROM - (SELECT - aggr_id, - source, - sum(p_nom) AS p_nom - FROM model_draft.ego_supply_pf_generator_single - WHERE scn_name = 'NEP 2035' - AND aggr_id IS NOT NULL - GROUP BY aggr_id, source) NEP; - --- map renpassG!S power sources to pf generators, aggr on fuel types, neglect efficiency classes -DROP materialized view if EXISTS calc_renpass_gis.pp_feedin_by_pf_source; -CREATE materialized view calc_renpass_gis.pp_feedin_by_pf_source -AS -SELECT -NEP.source, NEP.datetime, sum(NEP.val) AS val -FROM - (SELECT - CASE - WHEN obj_label LIKE '%%gas%%' THEN 1 - when obj_label LIKE '%%lignite%%' THEN 2 - when obj_label LIKE '%%mixed_fuels%%' THEN 3 - when obj_label LIKE '%%oil%%' THEN 4 - when obj_label LIKE '%%uranium%%' THEN 5 - when obj_label LIKE '%%biomass%%' THEN 6 - when obj_label LIKE '%%hard_coal%%' THEN 8 - when obj_label LIKE '%%run_of_river%%' THEN 9 --- when obj_label LIKE '%%storage_phs%%' THEN 11 - when obj_label LIKE '%%solar%%' THEN 12 - when obj_label LIKE '%%wind%%' THEN 13 - END AS source, - bus_label, - obj_label, - type, - datetime, - val - FROM calc_renpass_gis.renpass_gis_results - -- conds - WHERE obj_label LIKE '%%DE%%' -- only Germany - AND obj_label not LIKE '%%powerline%%' -- without any powerlines - AND scenario_id = 41 - -- take only one flow (input), storage output flow seems to be the right one (?) - AND ((obj_label LIKE '%%storage%%' AND type = 'from_bus') or (obj_label not LIKE '%%storage%%' AND type = 'to_bus')) -) AS NEP -WHERE NEP.source IS not NULL -GROUP BY NEP.source, NEP.datetime; - --- construct array per aggr_id according to source timeseries -INSERT into model_draft.ego_grid_pf_hv_generator_pq_set (scn_name, generator_id, temp_id, p_set) -SELECT - 'NEP 2035' AS scn_name, - A.aggr_id, - 1 AS temp_id, - array_agg(A.fraction_of_installed * B.val ORDER BY B.datetime) AS p_set - FROM calc_renpass_gis.pf_pp_by_source_aggr_id A, - calc_renpass_gis.pp_feedin_by_pf_source B -WHERE A.source = B.source -GROUP BY A.aggr_id; - ---eGo100 --- aggregate nominal capacity on aggr_id FROM powerflow generators, keeping the source -DROP materialized view if EXISTS calc_renpass_gis.pf_pp_by_source_aggr_id; -CREATE materialized view calc_renpass_gis.pf_pp_by_source_aggr_id -AS -SELECT -eGo.aggr_id, eGo.source, eGo.p_nom / sum(eGo.p_nom) over (partition by eGo.source) AS fraction_of_installed -FROM - (SELECT - aggr_id, - source, - sum(p_nom) AS p_nom - FROM model_draft.ego_supply_pf_generator_single - WHERE scn_name = 'eGo 100' - AND aggr_id IS NOT NULL - GROUP BY aggr_id, source) eGo; - --- map renpassG!S power sources to pf generators, aggr on fuel types, neglect efficiency classes -DROP materialized view if EXISTS calc_renpass_gis.pp_feedin_by_pf_source; -CREATE materialized view calc_renpass_gis.pp_feedin_by_pf_source -AS -SELECT -eGo.source, eGo.datetime, sum(eGo.val) AS val -FROM - (SELECT - CASE - WHEN obj_label LIKE '%%gas%%' THEN 1 - when obj_label LIKE '%%lignite%%' THEN 2 - when obj_label LIKE '%%mixed_fuels%%' THEN 3 - when obj_label LIKE '%%oil%%' THEN 4 - when obj_label LIKE '%%uranium%%' THEN 5 - when obj_label LIKE '%%biomass%%' THEN 6 - when obj_label LIKE '%%hard_coal%%' THEN 8 - when obj_label LIKE '%%run_of_river%%' THEN 9 --- when obj_label LIKE '%%storage_phs%%' THEN 11 - when obj_label LIKE '%%solar%%' THEN 12 - when obj_label LIKE '%%wind%%' THEN 13 - END AS source, - bus_label, - obj_label, - type, - datetime, - val - FROM calc_renpass_gis.renpass_gis_results - -- conds - WHERE obj_label LIKE '%%DE%%' -- only Germany - AND obj_label not LIKE '%%powerline%%' -- without any powerlines - AND scenario_id = 41 - -- take only one flow (input), storage output flow seems to be the right one (?) - AND ((obj_label LIKE '%%storage%%' AND type = 'from_bus') or (obj_label not LIKE '%%storage%%' AND type = 'to_bus')) -) AS eGo -WHERE eGo.source IS not NULL -GROUP BY eGo.source, eGo.datetime; - --- construct array per aggr_id according to source timeseries -INSERT into model_draft.ego_grid_pf_hv_generator_pq_set (scn_name, generator_id, temp_id, p_set) -SELECT - 'eGo 100' AS scn_name, - A.aggr_id, - 1 AS temp_id, - array_agg(A.fraction_of_installed * B.val ORDER BY B.datetime) AS p_set - FROM calc_renpass_gis.pf_pp_by_source_aggr_id A, - calc_renpass_gis.pp_feedin_by_pf_source B -WHERE A.source = B.source -GROUP BY A.aggr_id; - ------------------ NEIGHBOURING COUNTRIES -- 1 DELETE FROM model_draft.ego_grid_pf_hv_generator WHERE generator_id > 200000 AND scn_name = 'Status Quo'; From 69172ef2fc103b9b39b9a50bc299acb41d56a18d Mon Sep 17 00:00:00 2001 From: Martin Date: Wed, 21 Feb 2018 23:47:08 +0100 Subject: [PATCH 02/46] Fix import --- .../python_scripts/ego_dp_powerflow_timeseries_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index eec9e5f2..9385fe49 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -9,7 +9,7 @@ __author__ = "wolfbunke" import pandas as pd -import numpby as np +import numpy as np from dataprocessing.tools.io import oedb_session from sqlalchemy.orm import sessionmaker From bb91a0e9376739afa07c433ea7684bbd8f3be591 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 19 Mar 2018 21:29:06 +0100 Subject: [PATCH 03/46] Handle generator data for neighbours Taken from renpass_gis hidden relations. --- .../ego_dp_powerflow_neighbours_generator.py | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 dataprocessing/python_scripts/ego_dp_powerflow_neighbours_generator.py diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_neighbours_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_neighbours_generator.py new file mode 100644 index 00000000..a9eb20c9 --- /dev/null +++ b/dataprocessing/python_scripts/ego_dp_powerflow_neighbours_generator.py @@ -0,0 +1,147 @@ +""" +Assign German timeseries data from hidden renpassG!S schema to high voltage +powerflow. +""" + +__copyright__ = "ZNES Flensburg" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "wolfbunke" + +import pandas as pd +import numpy as np + +from dataprocessing.tools.io import oedb_session +from sqlalchemy.orm import sessionmaker +from sqlalchemy import MetaData, func +from ego_dp_powerflow_renpass_gis_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ + TEMPID, NEIGHBOURSID, _flatten, map_on_partial_string, renpass_gis_orm_classes +from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator, \ + EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour + +conn = oedb_session(section='test') +Session = sessionmaker(bind=conn) +session = Session() + +# obligatory delete statement based on NEIGHBOURSID +session.query(Generator).filter(Generator.generator_id >= NEIGHBOURSID).\ + delete(synchronize_session='fetch') + +session.query(PqSet).filter(PqSet.generator_id >= NEIGHBOURSID).\ + delete(synchronize_session='fetch') + + +############################################################################### + +Transformer, Source, Results = renpass_gis_orm_classes(session) + +# get DataFrame each row representing one electrical neighbour by applying +# filter on id and v_nom, not affected by scenario name +query = session.query(Neighbour) +neighbours = pd.read_sql(query.statement, query.session.bind) + +ix = (neighbours['id'] <= 27) & (neighbours['v_nom'] == 380) +neighbours = neighbours.loc[ix, :] +neighbours.set_index('cntr_id', inplace=True) + + +for scn_name, scn_nr in SCENARIOMAP.items(): + + # get renpass_gis scenario data on linear transformers. Parameters are + # defined on those edges directed from the component to the bus. + filters = [Transformer.scenario_id == scn_nr, + ~Transformer.source.like('%powerline%'), + Transformer.label == Transformer.source] # direction + + query = session.query(Transformer).filter(*filters) + transformers = pd.read_sql(query.statement, query.session.bind) + transformers['type'] = 'linear transformer' + + # get data on sources + filters = [Source.scenario_id == scn_nr, ~Source.label.like('GL%')] + query = session.query(Source).filter(*filters) + sources = pd.read_sql(query.statement, query.session.bind) + sources['type'] = 'source' + + # sources and transformers, distinct in renpass_gis, are both seen as + # generators and can be handled together + generators = pd.concat([sources, transformers], ignore_index=True) + + # parameters in renpass_gis are not necessarily scalars and stored in lists + # lists of len one are flattened + generators = generators.applymap(_flatten) + + # 0 does not equal zero. In case a class with zero nominal value + # should be defined for the purpose of scenario definition very small values + # are used in the scenario files. + ix = generators['nominal_value'] < 1e-7 + generators = generators.loc[~ix, :] + + # source in the context of eGo has a different meaning. The column has to + # be renamed + generators.rename(columns={'source': 'renpass_gis_source'}, inplace=True) + + # map from obj label string -> source + generators['source'] = map_on_partial_string( + generators['label'], SOURCE_TO_FUEL) + + generators['cntr_id'] = generators['label'].str[:2] + + # exclude Germany + generators = generators.loc[generators['cntr_id'] != 'DE', :] + + # assign bus_ids according to neighbours DataFrame + generators['bus'] = generators['cntr_id'].map(neighbours['bus_id']) + + # set control, and dispatch parameter + generators['control'] = 'PV' + generators['dispatch'] = generators['type'].map( + {'linear transformer': 'flexible', 'source': 'variable'}) + + # get corresponding optimization results from renpass_gis + # obj_label, datetime, val + filters = (~Results.obj_label.like('%DE%'), + ~Results.obj_label.like('%GL%'), + ~Results.obj_label.like('%powerline%'), + Results.type == 'to_bus', + Results.scenario_id == scn_nr) + fields = Results.obj_label, Results.datetime, Results.val + query = session.query(*fields).filter(*filters) + results = pd.read_sql(query.statement, query.session.bind) + + # map from obj label string -> source + results['source'] = map_on_partial_string( + results['obj_label'], SOURCE_TO_FUEL).astype(int) + + results['cntr_id'] = results['obj_label'].str[:2] + results['bus'] = results['cntr_id'].map(neighbours['bus_id']) + + # create Series with bus_id, source and power generation / actual value + # in list format + results_s = results.groupby(['bus', 'source'])['val'].apply(np.array) + results_s.name = 'p_set' + + # check all arrays are of the same length + assert all(len(i) == len(results_s[0]) for i in results_s) + + # include timeseries data in generators DataFrame + generators = generators.join(results_s, on=['bus', 'source']) + + # set scenario name, temporal id + generators['scn_name'] = scn_name + generators['temp_id'] = TEMPID + + generators['generator_id'] = generators.index + NEIGHBOURSID + + # prepare DataFrames to be exported + generator_ex = generators[['scn_name', 'generator_id', 'bus', 'dispatch', 'control']] + pqsets = generators[['scn_name', 'generator_id', 'temp_id', 'p_set']] + + # write to db + for i in generator_ex.to_dict(orient='records'): + session.add(Generator(**i)) + + for i in pqsets.to_dict(orient='records'): + session.add(PqSet(**i)) + + session.commit() From b5f54ec08a949340a2138d8cf539da79a9f9173e Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 19 Mar 2018 21:31:36 +0100 Subject: [PATCH 04/46] Add helper file To transfer data from renpass_gis to powerflow relations helper functions and constants are needed. --- .../ego_dp_powerflow_renpass_gis_helper.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 dataprocessing/python_scripts/ego_dp_powerflow_renpass_gis_helper.py diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_renpass_gis_helper.py b/dataprocessing/python_scripts/ego_dp_powerflow_renpass_gis_helper.py new file mode 100644 index 00000000..189f0f5d --- /dev/null +++ b/dataprocessing/python_scripts/ego_dp_powerflow_renpass_gis_helper.py @@ -0,0 +1,70 @@ +""" Helper functions and variables to handle renpass_gis tables. +""" + +__copyright__ = "ZNES Flensburg" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "wolfbunke" + +import pandas as pd + +from dataprocessing.tools.io import oedb_session +from sqlalchemy.orm import sessionmaker +from sqlalchemy import MetaData +from sqlalchemy.ext.automap import automap_base + +NEIGHBOURSID = 200000 + +SCENARIOMAP = {'Status Quo': 43, 'NEP 2035': 41, 'eGo 100': 40} + +SOURCE_TO_FUEL = { + 1: 'gas', 2: 'lignite', 3: 'mixed_fuels', 4: 'oil', + 5: 'uranium', 6: 'biomass', 8: 'hard_coal', 9: 'run_of_river', 10: 'reservoir', + 12: 'solar', 13: 'wind_onshore', 14: 'geothermal', 15: 'other_non_renewable', + 16: 'wind_offshore', 94: 'storage', 95: 'load', 96: 'waste', + 97: 'reservoir', 98: 'shortage', 99: 'excess'} + +TEMPID = 1 + +def renpass_gis_orm_classes(session): + """ + Parameters + ---------- + session : sqlalchemy.orm.session.Session + Handling all conversations with the database + + Notes + ----- + Relations in schema calc_renpass_gis are still not part of egoio. If this is + changed in the future this function becomes obsolete. + """ + + meta = MetaData() + meta.reflect(bind=session.bind, schema='calc_renpass_gis', + only=['renpass_gis_linear_transformer', + 'renpass_gis_source', + 'renpass_gis_results']) + + # map to classes + Base = automap_base(metadata=meta) + Base.prepare() + + Transformer, Source, Results = Base.classes.renpass_gis_linear_transformer, \ + Base.classes.renpass_gis_source, Base.classes.renpass_gis_results + + return Transformer, Source, Results + +def _flatten(x): + if isinstance(x, list): + return x[0] if len(x) == 1 else x + else: + return x + + +def map_on_partial_string(series, mapping): + """ Map mapping values to string series. """ + s = pd.Series(index=series.index) + for k, v in mapping.items(): + ix = series.str.contains(v) + s[ix] = k + return s From 2d56679b11561939f1aacb855e42347fce55874c Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 19 Mar 2018 21:41:33 +0100 Subject: [PATCH 05/46] Remove sql part of timeseries generator script --- .../ego_dp_powerflow_timeseries_generator.sql | 451 ------------------ 1 file changed, 451 deletions(-) diff --git a/dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql b/dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql index 9d0267c3..38e36d9b 100644 --- a/dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql +++ b/dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql @@ -9,457 +9,6 @@ __author__ = "wolfbunke" TODO: storage in storage_pqset #1069 */ ------------------- NEIGHBOURING COUNTRIES --- 1 -DELETE FROM model_draft.ego_grid_pf_hv_generator WHERE generator_id > 200000 AND scn_name = 'Status Quo'; -DELETE FROM model_draft.ego_grid_pf_hv_generator WHERE generator_id > 200000 AND scn_name = 'NEP 2035'; -DELETE FROM model_draft.ego_grid_pf_hv_generator WHERE generator_id > 200000 AND scn_name = 'eGo 100'; - - --- INSERT params of LinearTransformers in model_draft.ego_grid_pf_hv_generator (countries besides Germany) --- starting generator_id at 200000, bus_id for neighbouring countries > 2800000 atm --- Status Quo -INSERT into model_draft.ego_grid_pf_hv_generator - - SELECT - 'Status Quo' AS scn_name, - row_number() over () + 200000 AS generator_id, - B.bus_id AS bus, - 'flexible' AS dispatch, - 'PV' AS control, - nominal_value[1] AS p_nom, - FALSE AS p_nom_extendable, - NULL AS p_nom_max, - 0 AS p_nom_min, - 0 AS p_min_pu_fixed, - 1 AS p_max_pu_fixed, - 1 AS sign, - CASE - WHEN source LIKE '%%gas%%' THEN 1 - when source LIKE '%%lignite%%' THEN 2 - when source LIKE '%%mixed_fuels%%' THEN 3 - when source LIKE '%%oil%%' THEN 4 - when source LIKE '%%uranium%%' THEN 5 - when source LIKE '%%biomass%%' THEN 6 - when source LIKE '%%hard_coal%%' THEN 8 - END AS source - FROM calc_renpass_gis.renpass_gis_linear_transformer A join - ( - SELECT - * - FROM - (SELECT *, - max(v_nom) over (partition by cntr_id) AS max_v_nom - FROM - model_draft.ego_grid_hv_electrical_neighbours_bus - where id <= 27 - ) SQ - WHERE SQ.v_nom = SQ.max_v_nom - ) B - ON (substring(A.source, 1, 2) = B.cntr_id) - WHERE substring(A.source, 1, 2) <> 'DE' - AND A.nominal_value IS not NULL - AND A.nominal_value[1] > 0.001 - AND A.source not LIKE '%%powerline%%' - AND A.scenario_id = 43; - - --- NEP 2035 - -INSERT into model_draft.ego_grid_pf_hv_generator - - SELECT - 'NEP 2035' AS scn_name, - row_number() over () + 200000 AS generator_id, - B.bus_id AS bus, - 'flexible' AS dispatch, - 'PV' AS control, - nominal_value[1] AS p_nom, - FALSE AS p_nom_extendable, - NULL AS p_nom_max, - 0 AS p_nom_min, - 0 AS p_min_pu_fixed, - 1 AS p_max_pu_fixed, - 1 AS sign, - CASE - WHEN source LIKE '%%gas%%' THEN 1 - when source LIKE '%%lignite%%' THEN 2 - when source LIKE '%%mixed_fuels%%' THEN 3 - when source LIKE '%%oil%%' THEN 4 - when source LIKE '%%uranium%%' THEN 5 - when source LIKE '%%biomass%%' THEN 6 - when source LIKE '%%hard_coal%%' THEN 8 - END AS source - FROM calc_renpass_gis.renpass_gis_linear_transformer A join - ( - SELECT - * - FROM - (SELECT *, - max(v_nom) over (partition by cntr_id) AS max_v_nom - FROM - model_draft.ego_grid_hv_electrical_neighbours_bus - where id <= 27 - ) SQ - WHERE SQ.v_nom = SQ.max_v_nom - ) B - ON (substring(A.source, 1, 2) = B.cntr_id) - WHERE substring(A.source, 1, 2) <> 'DE' - AND A.nominal_value IS not NULL - AND A.nominal_value[1] > 0.001 - AND A.source not LIKE '%%powerline%%' - AND A.scenario_id = 41; - --- eGo 100 - -INSERT into model_draft.ego_grid_pf_hv_generator - - SELECT - 'eGo 100' AS scn_name, - row_number() over () + 200000 AS generator_id, - B.bus_id AS bus, - 'flexible' AS dispatch, - 'PV' AS control, - nominal_value[1] AS p_nom, - FALSE AS p_nom_extendable, - NULL AS p_nom_max, - 0 AS p_nom_min, - 0 AS p_min_pu_fixed, - 1 AS p_max_pu_fixed, - 1 AS sign, - CASE - WHEN source LIKE '%%gas%%' THEN 1 - when source LIKE '%%lignite%%' THEN 2 - when source LIKE '%%mixed_fuels%%' THEN 3 - when source LIKE '%%oil%%' THEN 4 - when source LIKE '%%uranium%%' THEN 5 - when source LIKE '%%biomass%%' THEN 6 - when source LIKE '%%hard_coal%%' THEN 8 - END AS source - FROM calc_renpass_gis.renpass_gis_linear_transformer A join - ( - SELECT - * - FROM - (SELECT *, - max(v_nom) over (partition by cntr_id) AS max_v_nom - FROM - model_draft.ego_grid_hv_electrical_neighbours_bus - where id <= 27 - ) SQ - WHERE SQ.v_nom = SQ.max_v_nom - ) B - ON (substring(A.source, 1, 2) = B.cntr_id) - WHERE substring(A.source, 1, 2) <> 'DE' - AND A.nominal_value IS not NULL - AND A.nominal_value[1] > 0.001 - AND A.source not LIKE '%%powerline%%' - AND A.scenario_id = 40; - - - - --- INSERT params of Source in model_draft.ego_grid_pf_hv_generator (countries besides Germany) --- Status Quo -INSERT into model_draft.ego_grid_pf_hv_generator - - SELECT - 'Status Quo' AS scn_name, - row_number() over () + (SELECT max(generator_id) FROM model_draft.ego_grid_pf_hv_generator) AS generator_id, - B.bus_id AS bus, - 'variable' AS dispatch, - 'PV' AS control, - nominal_value[1] AS p_nom, - FALSE AS p_nom_extendable, - NULL AS p_nom_max, - 0 AS p_nom_min, - 0 AS p_min_pu_fixed, - 1 AS p_max_pu_fixed, - 1 AS sign, - CASE - WHEN source LIKE '%%run_of_river%%' THEN 9 - WHEN source LIKE '%%solar%%' THEN 12 - WHEN source LIKE '%%wind%%' THEN 13 - when source LIKE '%%reservoir%%' THEN 10 - when source LIKE '%%geothermal%%' THEN 14 - END AS source - FROM calc_renpass_gis.renpass_gis_source A join - ( - SELECT - * - FROM - (SELECT *, - max(v_nom) over (partition by cntr_id) AS max_v_nom - FROM - model_draft.ego_grid_hv_electrical_neighbours_bus - where id <= 27 - ) SQ - WHERE SQ.v_nom = SQ.max_v_nom - ) B - ON (substring(A.source, 1, 2) = B.cntr_id) - WHERE substring(A.source, 1, 2) <> 'DE' - AND A.nominal_value[1] > 0.001 - AND A.scenario_id = 43; - --- NEP 2035 - -INSERT into model_draft.ego_grid_pf_hv_generator - - SELECT - 'NEP 2035' AS scn_name, - row_number() over () + (SELECT max(generator_id) FROM model_draft.ego_grid_pf_hv_generator) AS generator_id, - B.bus_id AS bus, - 'variable' AS dispatch, - 'PV' AS control, - nominal_value[1] AS p_nom, - FALSE AS p_nom_extendable, - NULL AS p_nom_max, - 0 AS p_nom_min, - 0 AS p_min_pu_fixed, - 1 AS p_max_pu_fixed, - 1 AS sign, - CASE - WHEN source LIKE '%%run_of_river%%' THEN 9 - WHEN source LIKE '%%solar%%' THEN 12 - WHEN source LIKE '%%wind%%' THEN 13 - when source LIKE '%%reservoir%%' THEN 10 - when source LIKE '%%geothermal%%' THEN 14 - END AS source - FROM calc_renpass_gis.renpass_gis_source A join - ( - SELECT - * - FROM - (SELECT *, - max(v_nom) over (partition by cntr_id) AS max_v_nom - FROM - model_draft.ego_grid_hv_electrical_neighbours_bus - where id <= 27 - ) SQ - WHERE SQ.v_nom = SQ.max_v_nom - ) B - ON (substring(A.source, 1, 2) = B.cntr_id) - WHERE substring(A.source, 1, 2) <> 'DE' - AND A.nominal_value[1] > 0.001 - AND A.scenario_id = 41; - --- eGo 100 - -INSERT into model_draft.ego_grid_pf_hv_generator - - SELECT - 'eGo 100' AS scn_name, - row_number() over () + (SELECT max(generator_id) FROM model_draft.ego_grid_pf_hv_generator) AS generator_id, - B.bus_id AS bus, - 'variable' AS dispatch, - 'PV' AS control, - nominal_value[1] AS p_nom, - FALSE AS p_nom_extendable, - NULL AS p_nom_max, - 0 AS p_nom_min, - 0 AS p_min_pu_fixed, - 1 AS p_max_pu_fixed, - 1 AS sign, - CASE - WHEN source LIKE '%%run_of_river%%' THEN 9 - WHEN source LIKE '%%solar%%' THEN 12 - WHEN source LIKE '%%wind%%' THEN 13 - when source LIKE '%%reservoir%%' THEN 10 - when source LIKE '%%geothermal%%' THEN 14 - END AS source - FROM calc_renpass_gis.renpass_gis_source A join - ( - SELECT - * - FROM - (SELECT *, - max(v_nom) over (partition by cntr_id) AS max_v_nom - FROM - model_draft.ego_grid_hv_electrical_neighbours_bus - where id <= 27 - ) SQ - WHERE SQ.v_nom = SQ.max_v_nom - ) B - ON (substring(A.source, 1, 2) = B.cntr_id) - WHERE substring(A.source, 1, 2) <> 'DE' - AND A.nominal_value[1] > 0.001 - AND A.scenario_id = 40; - - --- Copy timeseries data ---DELETE FROM model_draft.ego_grid_pf_hv_generator_pq_set WHERE generator_id > 200000 AND scn_name = 'Status Quo'; -DELETE FROM model_draft.ego_grid_pf_hv_generator_pq_set WHERE generator_id > 200000 AND scn_name = 'NEP 2035'; -DELETE FROM model_draft.ego_grid_pf_hv_generator_pq_set WHERE generator_id > 200000 AND scn_name = 'eGo 100'; - --- CREATE a view containing data for generator_id's > 200000 for each timestep --- SELECT * FROM calc_renpass_gis.translate_to_pf limit 1000; --- Status Quo -Drop MATERIALIZED VIEW IF EXISTS calc_renpass_gis.translate_to_pf; - -CREATE MATERIALIZED VIEW calc_renpass_gis.translate_to_pf AS - SELECT - SQ.generator_id, - C.datetime, - C.val - FROM - (SELECT *, - CASE - WHEN A.source = 1 THEN 'gas' - WHEN A.source = 2 THEN 'lignite' - WHEN A.source = 3 THEN 'mixed_fuels' - WHEN A.source = 4 THEN 'oil' - WHEN A.source = 5 THEN 'uranium' - WHEN A.source = 6 THEN 'biomass' - WHEN A.source = 8 THEN 'hard_coal' - WHEN A.source = 9 THEN 'run_of_river' - WHEN A.source = 10 THEN 'reservoir' - WHEN A.source = 12 THEN 'solar' - WHEN A.source = 13 THEN 'wind' - WHEN A.source = 14 THEN 'geothermal' - END AS renpass_gis_source - FROM model_draft.ego_grid_pf_hv_generator A join - model_draft.ego_grid_hv_electrical_neighbours_bus B - ON (A.bus = B.bus_id) - WHERE A.generator_id > 200000 - AND A.scn_name = 'Status Quo' - ) SQ, - calc_renpass_gis.renpass_gis_results C - WHERE - (C.obj_label LIKE '%%' || SQ.cntr_id || '%%' || SQ.renpass_gis_source || '%%') - AND C.scenario_id = 43 - AND C.type = 'to_bus'; - --- Make an array, INSERT into generator_pq_set -INSERT into model_draft.ego_grid_pf_hv_generator_pq_set (scn_name, generator_id, temp_id, p_set) - - SELECT 'Status Quo' AS scn_name, - SQ.generator_id, - 1 AS temp_id, - array_agg(SQ.val ORDER BY SQ.datetime) AS p_set - FROM - ( - SELECT - A.generator_id, - A.datetime, - A.val AS val - FROM calc_renpass_gis.translate_to_pf A join - model_draft.ego_grid_pf_hv_generator B - USING (generator_id) - ) SQ - GROUP BY generator_id; - --- NEP 2035 - -Drop MATERIALIZED VIEW IF EXISTS calc_renpass_gis.translate_to_pf; - -CREATE MATERIALIZED VIEW calc_renpass_gis.translate_to_pf AS - SELECT - NEP.generator_id, - C.datetime, - C.val - FROM - (SELECT *, - CASE - WHEN A.source = 1 THEN 'gas' - WHEN A.source = 2 THEN 'lignite' - WHEN A.source = 3 THEN 'mixed_fuels' - WHEN A.source = 4 THEN 'oil' - WHEN A.source = 5 THEN 'uranium' - WHEN A.source = 6 THEN 'biomass' - WHEN A.source = 8 THEN 'hard_coal' - WHEN A.source = 9 THEN 'run_of_river' - WHEN A.source = 10 THEN 'reservoir' - WHEN A.source = 12 THEN 'solar' - WHEN A.source = 13 THEN 'wind' - WHEN A.source = 14 THEN 'geothermal' - END AS renpass_gis_source - FROM model_draft.ego_grid_pf_hv_generator A join - model_draft.ego_grid_hv_electrical_neighbours_bus B - ON (A.bus = B.bus_id) - WHERE A.generator_id > 200000 - AND A.scn_name = 'NEP 2035' - ) NEP, - calc_renpass_gis.renpass_gis_results C - WHERE - (C.obj_label LIKE '%%' || NEP.cntr_id || '%%' || NEP.renpass_gis_source || '%%') - AND C.scenario_id = 41 - AND C.type = 'to_bus'; - --- Make an array, INSERT into generator_pq_set -INSERT into model_draft.ego_grid_pf_hv_generator_pq_set (scn_name, generator_id, temp_id, p_set) - - SELECT 'NEP 2035' AS scn_name, - NEP.generator_id, - 1 AS temp_id, - array_agg(NEP.val ORDER BY NEP.datetime) AS p_set - FROM - ( - SELECT - A.generator_id, - A.datetime, - A.val AS val - FROM calc_renpass_gis.translate_to_pf A join - model_draft.ego_grid_pf_hv_generator B - USING (generator_id) - ) NEP - GROUP BY generator_id; - --- eGo 100 - -Drop MATERIALIZED VIEW IF EXISTS calc_renpass_gis.translate_to_pf; - -CREATE MATERIALIZED VIEW calc_renpass_gis.translate_to_pf AS - SELECT - EGO.generator_id, - C.datetime, - C.val - FROM - (SELECT *, - CASE - WHEN A.source = 1 THEN 'gas' - WHEN A.source = 2 THEN 'lignite' - WHEN A.source = 3 THEN 'mixed_fuels' - WHEN A.source = 4 THEN 'oil' - WHEN A.source = 5 THEN 'uranium' - WHEN A.source = 6 THEN 'biomass' - WHEN A.source = 8 THEN 'hard_coal' - WHEN A.source = 9 THEN 'run_of_river' - WHEN A.source = 10 THEN 'reservoir' - WHEN A.source = 12 THEN 'solar' - WHEN A.source = 13 THEN 'wind' - WHEN A.source = 14 THEN 'geothermal' - END AS renpass_gis_source - FROM model_draft.ego_grid_pf_hv_generator A join - model_draft.ego_grid_hv_electrical_neighbours_bus B - ON (A.bus = B.bus_id) - WHERE A.generator_id > 200000 - AND A.scn_name = 'eGo 100' - ) EGO, - calc_renpass_gis.renpass_gis_results C - WHERE - (C.obj_label LIKE '%%' || EGO.cntr_id || '%%' || EGO.renpass_gis_source || '%%') - AND C.scenario_id = 41 - AND C.type = 'to_bus'; - --- Make an array, INSERT into generator_pq_set -INSERT into model_draft.ego_grid_pf_hv_generator_pq_set (scn_name, generator_id, temp_id, p_set) - - SELECT 'eGo 100' AS scn_name, - EGO.generator_id, - 1 AS temp_id, - array_agg(EGO.val ORDER BY EGO.datetime) AS p_set - FROM - ( - SELECT - A.generator_id, - A.datetime, - A.val AS val - FROM calc_renpass_gis.translate_to_pf A join - model_draft.ego_grid_pf_hv_generator B - USING (generator_id) - ) EGO - GROUP BY generator_id; - -- DELETE DELETE FROM model_draft.ego_grid_pf_hv_load WHERE bus IN ( From e2bc798719594a0077c8d2b80dbce09892bfc196 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 19 Mar 2018 21:47:10 +0100 Subject: [PATCH 06/46] Update import statement --- .../ego_dp_powerflow_timeseries_generator.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index 9385fe49..4a71fa67 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -18,17 +18,8 @@ from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator,\ EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus - - -SCENARIOMAP = {'Status Quo': 43, 'NEP 2035': 41, 'eGo 100': 40} - -SOURCE_TO_FUEL = { - 1: 'gas', 2: 'lignite', 3: 'mixed_fuels', 4: 'oil', - 5: 'uranium', 6: 'biomass', 8: 'hard_coal', 9: 'run_of_river', 12: 'solar', - 13: 'wind', 14: 'geothermal', 15: 'other_non_renewable', 94: 'storage', - 95: 'load', 96: 'waste', 97: 'reservoir', 98: 'shortage', 99: 'excess'} - -TEMPID = 1 +from ego_dp_powerflow_renpass_gis_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ + TEMPID # get database connection conn = oedb_session(section='test') From 5f5104924ee38f177d69ffbc218ef38b10149af4 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 19 Mar 2018 21:51:44 +0100 Subject: [PATCH 07/46] Rename files --- ..._helper.py => ego_dp_powerflow_timeseries_generator_helper.py} | 0 ...tor.py => ego_dp_powerflow_timeseries_generator_neighbours.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename dataprocessing/python_scripts/{ego_dp_powerflow_renpass_gis_helper.py => ego_dp_powerflow_timeseries_generator_helper.py} (100%) rename dataprocessing/python_scripts/{ego_dp_powerflow_neighbours_generator.py => ego_dp_powerflow_timeseries_generator_neighbours.py} (100%) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_renpass_gis_helper.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py similarity index 100% rename from dataprocessing/python_scripts/ego_dp_powerflow_renpass_gis_helper.py rename to dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_neighbours_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py similarity index 100% rename from dataprocessing/python_scripts/ego_dp_powerflow_neighbours_generator.py rename to dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py From 3ea0e4dde546ba668a4e75c5fd4ef6841ddb7ad3 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 19 Mar 2018 21:53:48 +0100 Subject: [PATCH 08/46] Update eGo_data_processing with new script files --- dataprocessing/eGo_data_processing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dataprocessing/eGo_data_processing.py b/dataprocessing/eGo_data_processing.py index 610bf553..6e235993 100644 --- a/dataprocessing/eGo_data_processing.py +++ b/dataprocessing/eGo_data_processing.py @@ -104,6 +104,8 @@ def data_processing(): 'ego_dp_powerflow_assignment_generator.sql', # Assign generators to corresponding substation (SQ, NEP2035, eGo100) 'ego_dp_powerflow_assignment_load.sql', # Assign loads to their corresponding substation (SQ, NEP2035, eGo100) 'ego_dp_powerflow_assignment_storage.sql', # Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100) + 'ego_dp_powerflow_timeseries_generator.py', # Transfer renpassG!S results into the corresponding powerflow table + 'ego_dp_powerflow_timeseries_generator_neighbours.py', # Tranfer renpassG!S results for neighbours into the corresponding powerflow tables 'ego_dp_powerflow_timeseries_generator.sql', # Transfer renpassG!S results into the corresponding powerflow table 'ego_dp_powerflow_griddistrict_demand.py', # Demand per MV Griddistrict 'ego_dp_powerflow_timeseries_demand.sql', # Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100) From 68e7b4f543c3a462c4331de89356c9dc3aa229d4 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 19 Mar 2018 21:55:20 +0100 Subject: [PATCH 09/46] Address name changes in import statements --- .../python_scripts/ego_dp_powerflow_timeseries_generator.py | 2 +- .../ego_dp_powerflow_timeseries_generator_neighbours.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index 4a71fa67..3989a2af 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -18,7 +18,7 @@ from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator,\ EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus -from ego_dp_powerflow_renpass_gis_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ +from ego_dp_powerflow_timeseries_generator_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ TEMPID # get database connection diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py index a9eb20c9..5f0128c7 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py @@ -14,7 +14,7 @@ from dataprocessing.tools.io import oedb_session from sqlalchemy.orm import sessionmaker from sqlalchemy import MetaData, func -from ego_dp_powerflow_renpass_gis_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ +from ego_dp_powerflow_timeseries_generator_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ TEMPID, NEIGHBOURSID, _flatten, map_on_partial_string, renpass_gis_orm_classes from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator, \ EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour From f70d08a159a318fd85c0a71a41ea4a4c4061cce4 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 1 Apr 2018 21:25:21 +0200 Subject: [PATCH 10/46] Add missing orm classes These are not part of egoio yet and cannot be imported. --- ...p_powerflow_timeseries_generator_helper.py | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py index 189f0f5d..d18f78ec 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py @@ -10,8 +10,10 @@ from dataprocessing.tools.io import oedb_session from sqlalchemy.orm import sessionmaker -from sqlalchemy import MetaData +from sqlalchemy import MetaData, Column, Integer, Float, Text, text from sqlalchemy.ext.automap import automap_base +from sqlalchemy.dialects.postgresql import ARRAY, DOUBLE_PRECISION + NEIGHBOURSID = 200000 @@ -26,8 +28,9 @@ TEMPID = 1 -def renpass_gis_orm_classes(session): - """ +def missing_orm_classes(session): + """ Not yet implemented in ego.io + Parameters ---------- session : sqlalchemy.orm.session.Session @@ -47,12 +50,37 @@ def renpass_gis_orm_classes(session): # map to classes Base = automap_base(metadata=meta) + + # ormclasses not part of egoio yet + class EgoPowerClass(Base): + __tablename__ = 'ego_power_class' + __table_args__ = {'schema': 'model_draft'} + + power_class_id = Column(Integer, primary_key=True, server_default=text("nextval('model_draft.ego_power_class_power_class_id_seq'::regclass)")) + lower_limit = Column(Float(53)) + upper_limit = Column(Float(53)) + wea = Column(Text) + h_hub = Column(Float(53)) + d_rotor = Column(Float(53)) + + + class EgoRenewableFeedin(Base): + __tablename__ = 'ego_renewable_feedin' + __table_args__ = {'schema': 'model_draft'} + + weather_scenario_id = Column(Integer, primary_key=True, nullable=False) + w_id = Column(Integer, primary_key=True, nullable=False) + source = Column(Text, primary_key=True, nullable=False) + weather_year = Column(Integer, primary_key=True, nullable=False) + power_class = Column(Integer, primary_key=True, nullable=False) + feedin = Column(ARRAY(DOUBLE_PRECISION(precision=53))) + Base.prepare() Transformer, Source, Results = Base.classes.renpass_gis_linear_transformer, \ Base.classes.renpass_gis_source, Base.classes.renpass_gis_results - return Transformer, Source, Results + return EgoPowerClass, EgoRenewableFeedin, Transformer, Source, Results def _flatten(x): if isinstance(x, list): From 9bf70be7a01d9af993d52e39b6d01833388402ec Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 1 Apr 2018 21:28:50 +0200 Subject: [PATCH 11/46] Begin handling p_max_pu To simplify this process p_max_pu will be handled in an additional loop over the eGo scenarios. --- .../ego_dp_powerflow_timeseries_generator.py | 49 +++++++++++++------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index 3989a2af..e4c119e6 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -12,31 +12,23 @@ import numpy as np from dataprocessing.tools.io import oedb_session -from sqlalchemy.orm import sessionmaker -from sqlalchemy import MetaData, func +from sqlalchemy.orm import sessionmaker, relationship +from sqlalchemy import MetaData, func, and_, case from sqlalchemy.ext.automap import automap_base from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator,\ EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus + from ego_dp_powerflow_timeseries_generator_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ - TEMPID + TEMPID, missing_orm_classes # get database connection conn = oedb_session(section='test') - Session = sessionmaker(bind=conn) session = Session() -meta = MetaData() -meta.bind = conn -meta.reflect(bind=conn, schema='calc_renpass_gis', - only=['renpass_gis_results']) - -# map to classes -Base = automap_base(metadata=meta) -Base.prepare() +PowerClass, Feedin, *_, Results = missing_orm_classes(session) -Results = Base.classes.renpass_gis_results ############################################################################### @@ -48,14 +40,15 @@ def _norm(x): session.query(PqSet).delete() session.commit() +# p_set for scn_name, scn_nr in SCENARIOMAP.items(): # dataframe from model_draft.pf_generator_single # with aggr_id, source, p_nom - filters = (Generator.scn_name == scn_name, Generator.aggr_id != None) + filters = (Generator.scn_name == scn_name, Generator.aggr_id != None) # comma seperated in fiter() are internally joined using and_ fields = [Generator.aggr_id, Generator.source, func.sum(Generator.p_nom).label('summed_p_nom')] - grouper = Generator.aggr_id, Generator.source + grouper = Generator.aggr_id, Generator.source, query = session.query(*fields).filter(*filters).group_by(*grouper) generators = pd.read_sql(query.statement, query.session.bind) @@ -65,7 +58,7 @@ def _norm(x): generators.groupby('source')['summed_p_nom'].apply(_norm) # dataframe from calc_renpass_gis.renpass_gis_results - # optimization results to buses + # optimal dispacth results directed towards buses # with obj_label, datetime, val filters = (Results.obj_label.like('%DE%'), ~Results.obj_label.like('%powerline%'), @@ -119,3 +112,27 @@ def _norm(x): for i in pqsets.to_dict(orient='records'): session.add(PqSet(**i)) session.commit() + + +# p_max_pu +for scn_name, scn_nr in SCENARIOMAP.items(): + + + sources = ['wind_onshore', 'wind_offshore', 'solar'] + sources_dict = {v:k for k, v in SOURCE_TO_FUEL.items() if v in sources} + casestr = case(sources_dict, value=Feedin.source, else_=None) + + filters = (Generator.scn_name == scn_name, Generator.aggr_id != None) + fields = [Generator.aggr_id, Generator.source, Generator.w_id, Generator.power_class, + func.sum(Generator.p_nom).label('summed_p_nom')] + grouper = Generator.aggr_id, Generator.source, Generator.w_id, Generator.power_class + + t = session.query(*fields).join(Generator.feedin).group_by(*grouper).subquery() + + query = session.query(t, Feedin.feedin).filter(t.c.w_id == Feedin.w_id, + t.c.power_class == Feedin.power_class, t.c.source == casestr) + + generators = pd.read_sql(query.statement, query.session.bind) + + #func = lambda x: sum(np.asarray....... + generators.groupby(['aggr_id', 'source'], as_index=False).apply(func) From f4765af5710d8931106ee02e1987a1ccda6389d6 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 2 Apr 2018 13:25:04 +0200 Subject: [PATCH 12/46] Calculate average feedin per aggr_id, source --- .../ego_dp_powerflow_timeseries_generator.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index e4c119e6..33b4a1fc 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -127,12 +127,26 @@ def _norm(x): func.sum(Generator.p_nom).label('summed_p_nom')] grouper = Generator.aggr_id, Generator.source, Generator.w_id, Generator.power_class - t = session.query(*fields).join(Generator.feedin).group_by(*grouper).subquery() + t = session.query(*fields).group_by(*grouper).filter(*filters).subquery() query = session.query(t, Feedin.feedin).filter(t.c.w_id == Feedin.w_id, t.c.power_class == Feedin.power_class, t.c.source == casestr) generators = pd.read_sql(query.statement, query.session.bind) - #func = lambda x: sum(np.asarray....... - generators.groupby(['aggr_id', 'source'], as_index=False).apply(func) + def weighted_average_feedin(x): + + # 1darray weights for number of feedins + weights = np.array(_norm(x['summed_p_nom'])) + + # ndarray of shape (timesteps, number of feedins) + feedins = np.array(x['feedin'].tolist()).T + + # ndarray of shape (timesteps, number of feedins) + weighted_feedins = np.multiply(weights, feedins) + + # return averaged feedin + return np.mean(weighted_feedins, axis=1) + + p_max_pu = generators.groupby(['aggr_id', 'source'], as_index=False).\ + apply(weighted_average_feedin) From afccca9718b712e3e5baa975d78e79f523da9b7e Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 2 Apr 2018 19:55:51 +0200 Subject: [PATCH 13/46] Fix wrong return value in weighted_average_feedin --- .../python_scripts/ego_dp_powerflow_timeseries_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index 33b4a1fc..54a8fdbb 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -146,7 +146,7 @@ def weighted_average_feedin(x): weighted_feedins = np.multiply(weights, feedins) # return averaged feedin - return np.mean(weighted_feedins, axis=1) + return np.sum(weighted_feedins, axis=1) p_max_pu = generators.groupby(['aggr_id', 'source'], as_index=False).\ apply(weighted_average_feedin) From 6cb76714a31eca9bd92eec194a8081b1935108ca Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 2 Apr 2018 19:57:20 +0200 Subject: [PATCH 14/46] Retrieve PQ-Sets for update --- .../python_scripts/ego_dp_powerflow_timeseries_generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index 54a8fdbb..e69ba8d2 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -150,3 +150,5 @@ def weighted_average_feedin(x): p_max_pu = generators.groupby(['aggr_id', 'source'], as_index=False).\ apply(weighted_average_feedin) + + pqsets = session.query(PqSet).filter(PqSet.scn_name == scn_name).all() From 750486ede6f893e0b8314fa6b8cc43f602a10ec7 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 2 Apr 2018 22:38:30 +0200 Subject: [PATCH 15/46] Update comment --- .../python_scripts/ego_dp_powerflow_timeseries_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index e69ba8d2..96e5c902 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -44,7 +44,7 @@ def _norm(x): for scn_name, scn_nr in SCENARIOMAP.items(): # dataframe from model_draft.pf_generator_single - # with aggr_id, source, p_nom + # with aggr_id (unique combination of bus, scn_name, source), source, p_nom filters = (Generator.scn_name == scn_name, Generator.aggr_id != None) # comma seperated in fiter() are internally joined using and_ fields = [Generator.aggr_id, Generator.source, func.sum(Generator.p_nom).label('summed_p_nom')] From a11f01dfbbcfacaa5029d762619b6fdf8522bb38 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 2 Apr 2018 23:15:45 +0200 Subject: [PATCH 16/46] Update p_max_pu on existing PQ-Sets --- .../ego_dp_powerflow_timeseries_generator.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index 96e5c902..f47672b7 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -1,6 +1,6 @@ """ -Assign German timeseries data from hidden renpassG!S schema to high voltage -powerflow. +This script writes optimized dispatch timeseries data based on renpassG!S results +and renewable feedin timeseries based on simple feedin to the corresponding hv powerflow. """ __copyright__ = "ZNES Flensburg" @@ -17,7 +17,7 @@ from sqlalchemy.ext.automap import automap_base from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator,\ - EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus + EgoGridPfHvGeneratorPqSet as PqSet from ego_dp_powerflow_timeseries_generator_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ TEMPID, missing_orm_classes @@ -117,9 +117,8 @@ def _norm(x): # p_max_pu for scn_name, scn_nr in SCENARIOMAP.items(): - sources = ['wind_onshore', 'wind_offshore', 'solar'] - sources_dict = {v:k for k, v in SOURCE_TO_FUEL.items() if v in sources} + sources_dict = {v: k for k, v in SOURCE_TO_FUEL.items() if v in sources} casestr = case(sources_dict, value=Feedin.source, else_=None) filters = (Generator.scn_name == scn_name, Generator.aggr_id != None) @@ -148,7 +147,11 @@ def weighted_average_feedin(x): # return averaged feedin return np.sum(weighted_feedins, axis=1) - p_max_pu = generators.groupby(['aggr_id', 'source'], as_index=False).\ + p_max_pu = generators.groupby(['aggr_id'], as_index=False).\ apply(weighted_average_feedin) - pqsets = session.query(PqSet).filter(PqSet.scn_name == scn_name).all() + for i in session.query(PqSet).filter(PqSet.scn_name == scn_name).all(): + if i.generator_id in p_max_pu: + i.p_max_pu = p_max_pu[i.generator_id] + + session.commit() From d5cb1e3e1c0fd79b6437ded1305c0d6aa4e37a0b Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 9 Apr 2018 22:10:58 +0200 Subject: [PATCH 17/46] Separate assignment of p_max_pu --- .../ego_dp_powerflow_timeseries_feedin.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py new file mode 100644 index 00000000..e6350ae0 --- /dev/null +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py @@ -0,0 +1,61 @@ +import pandas as pd + +from dataprocessing.tools.io import oedb_session +from sqlalchemy.orm import sessionmaker +from sqlalchemy import case +from dataprocessing.python_scripts.functions.ego_scenario_log \ + import write_ego_scenario_log + +from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle \ + as GeneratorSingle, EgoGridPfHvGeneratorPqSet as PqSet +from ego_dp_powerflow_timeseries_generator_helper import FUEL_TO_SOURCE, SCENARIOMAP, \ + TEMPID, missing_orm_classes + +# get database connection +conn = oedb_session(section='test') +Session = sessionmaker(bind=conn) +session = Session() + +PowerClass, Feedin, *_, Results = missing_orm_classes(session) + +# p_max_pu +for scn_name, scn_nr in SCENARIOMAP.items(): + + sources = ['wind_onshore', 'wind_offshore', 'solar'] + sources_dict = {k: v for k, v in FUEL_TO_SOURCE.items() if k in sources} + casestr = case(sources_dict, value=Feedin.source, else_=None) + + # construct subquery with unique aggr_id, source, w_id, power_class + filters = (GeneratorSingle.scn_name == scn_name, GeneratorSingle.aggr_id != None) + fields = [GeneratorSingle.aggr_id, GeneratorSingle.source, + GeneratorSingle.w_id, GeneratorSingle.power_class] + grouper = GeneratorSingle.aggr_id, GeneratorSingle.source,\ + GeneratorSingle.w_id, GeneratorSingle.power_class + + t = session.query(*fields).group_by(*grouper).filter(*filters).subquery() + + # use subquery to select the corresponding feedin + query = session.query(t, Feedin.feedin.label('p_max_pu')).filter(t.c.w_id == Feedin.w_id, + t.c.power_class == Feedin.power_class, t.c.source == casestr) + + generators = pd.read_sql(query.statement, query.session.bind) + + # rename column aggr_id to generator_id + generators.rename(columns={'aggr_id': 'generator_id'}, inplace=True) + + generators['temp_id'] = TEMPID + generators['scn_name'] = scn_name + + fields = ['generator_id', 'p_max_pu', 'scn_name', 'temp_id'] + for i in generators[fields].to_dict(orient='records'): + session.add(PqSet(**i)) + + session.commit() + + write_ego_scenario_log(conn=conn, + version='v0.3.0', + io='input', + schema='model_draft', + table=PqSet.__tablename__, + script=__file__, + entries=len(generators)) From 2aafccddc1fce0767d58f272daa6f71753e560ce Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 9 Apr 2018 22:12:22 +0200 Subject: [PATCH 18/46] Update renpass_gis helper container --- ...p_powerflow_timeseries_generator_helper.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py index d18f78ec..cf63b559 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py @@ -19,12 +19,32 @@ SCENARIOMAP = {'Status Quo': 43, 'NEP 2035': 41, 'eGo 100': 40} -SOURCE_TO_FUEL = { - 1: 'gas', 2: 'lignite', 3: 'mixed_fuels', 4: 'oil', - 5: 'uranium', 6: 'biomass', 8: 'hard_coal', 9: 'run_of_river', 10: 'reservoir', - 12: 'solar', 13: 'wind_onshore', 14: 'geothermal', 15: 'other_non_renewable', - 16: 'wind_offshore', 94: 'storage', 95: 'load', 96: 'waste', - 97: 'reservoir', 98: 'shortage', 99: 'excess'} +# list of available fuel types in eGo scenarios +FUEL_TO_SOURCE = { + 'gas': 1, + 'lignite': 2, + 'waste': 3, + 'mixed_fuels': 3, + 'oil': 4, + 'uranium': 5, + 'biomass': 6, + 'chp': 7, # is this correct? + 'hard_coal': 8, + 'run_of_river': 9, + 'reservoir': 10, + 'solar': 12, + 'wind_onshore': 13, + 'geothermal': 14, + 'wind_offshore': 17, + 'storage_redox_flow': 99, + 'storage_pumped_hydro': 99, + 'storage_lithium_ion': 99, + 'storage_hydrogen': 99, + 'storage_phs': 99, + 'storage_a_caes': 99, + 'load': 99, + 'shortage': 99, + 'excess': 99} TEMPID = 1 @@ -82,6 +102,7 @@ class EgoRenewableFeedin(Base): return EgoPowerClass, EgoRenewableFeedin, Transformer, Source, Results + def _flatten(x): if isinstance(x, list): return x[0] if len(x) == 1 else x From cf5d3ff073f3955d4f456ecf272a9161ade549fe Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 9 Apr 2018 22:13:29 +0200 Subject: [PATCH 19/46] Use log function with stored variable --- .../ego_dp_powerflow_timeseries_feedin.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py index e6350ae0..903deb3c 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py @@ -19,13 +19,15 @@ PowerClass, Feedin, *_, Results = missing_orm_classes(session) # p_max_pu +logged = 0 for scn_name, scn_nr in SCENARIOMAP.items(): sources = ['wind_onshore', 'wind_offshore', 'solar'] sources_dict = {k: v for k, v in FUEL_TO_SOURCE.items() if k in sources} casestr = case(sources_dict, value=Feedin.source, else_=None) - # construct subquery with unique aggr_id, source, w_id, power_class + # construct subquery from PfGeneratorSingle with unique aggr_id, source, + # w_id, power_class filters = (GeneratorSingle.scn_name == scn_name, GeneratorSingle.aggr_id != None) fields = [GeneratorSingle.aggr_id, GeneratorSingle.source, GeneratorSingle.w_id, GeneratorSingle.power_class] @@ -52,10 +54,12 @@ session.commit() - write_ego_scenario_log(conn=conn, - version='v0.3.0', - io='input', - schema='model_draft', - table=PqSet.__tablename__, - script=__file__, - entries=len(generators)) + logged += len(generators) + +write_ego_scenario_log(conn=conn, + version='v0.3.0', + io='input', + schema='model_draft', + table=PqSet.__tablename__, + script=__file__, + entries=logged) From 4f741aec624a46595a424c2f6ca3e9fe36d9d015 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 9 Apr 2018 22:15:09 +0200 Subject: [PATCH 20/46] Refactor timeseries generator Atm timeseries generator handles discrepancies within the scenario definition of renpass_gis and powerflow. --- .../ego_dp_powerflow_timeseries_generator.py | 142 ++++++------------ 1 file changed, 49 insertions(+), 93 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index f47672b7..92ff0717 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -1,25 +1,23 @@ -""" -This script writes optimized dispatch timeseries data based on renpassG!S results -and renewable feedin timeseries based on simple feedin to the corresponding hv powerflow. +""" This script assign optimized dispatch timeseries data based on renpassG!S +results to high-voltage powerflow generators. """ -__copyright__ = "ZNES Flensburg" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "wolfbunke" +__copyright__ = "ZNES Flensburg" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "wolfbunke" import pandas as pd import numpy as np +import logging from dataprocessing.tools.io import oedb_session -from sqlalchemy.orm import sessionmaker, relationship -from sqlalchemy import MetaData, func, and_, case -from sqlalchemy.ext.automap import automap_base +from dataprocessing.python_scripts.functions.ego_scenario_log import write_ego_scenario_log +from sqlalchemy.orm import sessionmaker -from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator,\ +from egoio.db_tables.model_draft import EgoGridPfHvGenerator as Generator, \ EgoGridPfHvGeneratorPqSet as PqSet - -from ego_dp_powerflow_timeseries_generator_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ +from ego_dp_powerflow_timeseries_generator_helper import FUEL_TO_SOURCE, SCENARIOMAP, \ TEMPID, missing_orm_classes # get database connection @@ -29,10 +27,8 @@ PowerClass, Feedin, *_, Results = missing_orm_classes(session) - ############################################################################### - def _norm(x): return x / x.sum() @@ -43,23 +39,18 @@ def _norm(x): # p_set for scn_name, scn_nr in SCENARIOMAP.items(): - # dataframe from model_draft.pf_generator_single - # with aggr_id (unique combination of bus, scn_name, source), source, p_nom - filters = (Generator.scn_name == scn_name, Generator.aggr_id != None) # comma seperated in fiter() are internally joined using and_ - fields = [Generator.aggr_id, Generator.source, - func.sum(Generator.p_nom).label('summed_p_nom')] - grouper = Generator.aggr_id, Generator.source, - query = session.query(*fields).filter(*filters).group_by(*grouper) - + # model_draft.ego_grid_pf_hv_generator -> pd.DataFrame + query = session.query(Generator).filter(Generator.scn_name == scn_name) generators = pd.read_sql(query.statement, query.session.bind) + assert set(generators['source'].isnull()) == {False}, "Source field empty in generators table." + # create fraction of nominal power to total nominal power for each source - generators['fraction_of_total_p_nom'] = \ - generators.groupby('source')['summed_p_nom'].apply(_norm) + generators['p_nom_fraction'] = generators.groupby('source')['p_nom'].\ + apply(_norm) - # dataframe from calc_renpass_gis.renpass_gis_results - # optimal dispacth results directed towards buses - # with obj_label, datetime, val + # calc_renpass_gis.renpass_gis_results -> pd.DataFrame + # contains inflows to buses in Germany excluding powerlines filters = (Results.obj_label.like('%DE%'), ~Results.obj_label.like('%powerline%'), Results.type == 'to_bus', @@ -71,87 +62,52 @@ def _norm(x): # map obj_label to corresponding source results['source'] = None - for k, v in SOURCE_TO_FUEL.items(): - idx = results['obj_label'].str.contains(v) - results.loc[idx, 'source'] = k + for k, v in FUEL_TO_SOURCE.items(): + idx = results['obj_label'].str.contains(k) + results.loc[idx, 'source'] = v # aggregate by source and datetime + # in this step generation from mixed fuels and waste is summed up + # assuming these sources have identical input parameters results = results.groupby(['source', 'datetime'], as_index=False).sum() - # power generation timeseries for each source in list format + # timeseries for each component in list format + # contains excess, shortage, load, mixed fuels results_s = results.groupby('source')['val'].apply(np.array) # map corresponding timeseries with each generator multiplied by fraction # of nominal power generators['p_set'] = generators['source'].map(results_s) * \ - generators['fraction_of_total_p_nom'] + generators['p_nom_fraction'] - # generators without p_set - ix = generators['p_set'].isnull() - print('Generators with sources {} have no p_sets assigned!'.format( - generators[ix]['source'].unique())) - generators.loc[ix, 'p_set'] = None + # inform the user about possible discrepancies + # dataprocessing has no logging + for i, k in FUEL_TO_SOURCE.items(): + if k not in generators['source'].values: + logging.info("%s: %s: Object label %s is not matched in powerflow." % ( + __file__, scn_name, i)) - # add columns - empty = ['q_set', 'p_min_pu', 'p_max_pu'] + for k in generators.source.unique(): + if k not in results_s.index: + logging.info("%s: %s: Source %s is not matched in renpass_gis." % ( + __file__, scn_name, int(k))) - pqsets = pd.concat( - [generators[['aggr_id', 'p_set']], - pd.DataFrame(columns=empty)]) + # test for missing values for debugging + assert set(generators['p_set'].isnull()) == {False}, "P_set field empty in generators table." - pqsets.loc[:, empty] = None + # OR just get rid of generators with missing p_set + generators = generators[~generators['p_set'].isnull()].copy() - # add scenario name and temporal id - pqsets['scn_name'] = scn_name - pqsets['temp_id'] = TEMPID + # remove solar and wind + sources = [v for k, v in FUEL_TO_SOURCE.items() if k in + ['wind_offshore', 'wind_onshore', 'solar']] - # rename column aggr_id to generator_id - pqsets.rename(columns={'aggr_id': 'generator_id'}, inplace=True) + ix = generators['source'].isin(sources) + generators = generators[~ix].copy() - # write to db - for i in pqsets.to_dict(orient='records'): + generators['temp_id'] = TEMPID + fields = ['generator_id', 'p_set', 'scn_name', 'temp_id'] + for i in generators[fields].to_dict(orient='records'): session.add(PqSet(**i)) - session.commit() - - -# p_max_pu -for scn_name, scn_nr in SCENARIOMAP.items(): - - sources = ['wind_onshore', 'wind_offshore', 'solar'] - sources_dict = {v: k for k, v in SOURCE_TO_FUEL.items() if v in sources} - casestr = case(sources_dict, value=Feedin.source, else_=None) - - filters = (Generator.scn_name == scn_name, Generator.aggr_id != None) - fields = [Generator.aggr_id, Generator.source, Generator.w_id, Generator.power_class, - func.sum(Generator.p_nom).label('summed_p_nom')] - grouper = Generator.aggr_id, Generator.source, Generator.w_id, Generator.power_class - - t = session.query(*fields).group_by(*grouper).filter(*filters).subquery() - - query = session.query(t, Feedin.feedin).filter(t.c.w_id == Feedin.w_id, - t.c.power_class == Feedin.power_class, t.c.source == casestr) - - generators = pd.read_sql(query.statement, query.session.bind) - - def weighted_average_feedin(x): - - # 1darray weights for number of feedins - weights = np.array(_norm(x['summed_p_nom'])) - - # ndarray of shape (timesteps, number of feedins) - feedins = np.array(x['feedin'].tolist()).T - - # ndarray of shape (timesteps, number of feedins) - weighted_feedins = np.multiply(weights, feedins) - - # return averaged feedin - return np.sum(weighted_feedins, axis=1) - - p_max_pu = generators.groupby(['aggr_id'], as_index=False).\ - apply(weighted_average_feedin) - - for i in session.query(PqSet).filter(PqSet.scn_name == scn_name).all(): - if i.generator_id in p_max_pu: - i.p_max_pu = p_max_pu[i.generator_id] session.commit() From 5b90b30d0f840d506c91ee27cc9fe7abcd7a8227 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 9 Apr 2018 22:30:19 +0200 Subject: [PATCH 21/46] Rename constant FUEL_TO_SOURCE --- .../python_scripts/ego_dp_powerflow_timeseries_feedin.py | 4 ++-- .../ego_dp_powerflow_timeseries_generator.py | 8 ++++---- .../ego_dp_powerflow_timeseries_generator_helper.py | 2 +- .../ego_dp_powerflow_timeseries_generator_neighbours.py | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py index 903deb3c..378df318 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py @@ -8,7 +8,7 @@ from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle \ as GeneratorSingle, EgoGridPfHvGeneratorPqSet as PqSet -from ego_dp_powerflow_timeseries_generator_helper import FUEL_TO_SOURCE, SCENARIOMAP, \ +from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ TEMPID, missing_orm_classes # get database connection @@ -23,7 +23,7 @@ for scn_name, scn_nr in SCENARIOMAP.items(): sources = ['wind_onshore', 'wind_offshore', 'solar'] - sources_dict = {k: v for k, v in FUEL_TO_SOURCE.items() if k in sources} + sources_dict = {k: v for k, v in OBJ_LABEL_TO_SOURCE.items() if k in sources} casestr = case(sources_dict, value=Feedin.source, else_=None) # construct subquery from PfGeneratorSingle with unique aggr_id, source, diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index 92ff0717..ad1843bd 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -17,7 +17,7 @@ from egoio.db_tables.model_draft import EgoGridPfHvGenerator as Generator, \ EgoGridPfHvGeneratorPqSet as PqSet -from ego_dp_powerflow_timeseries_generator_helper import FUEL_TO_SOURCE, SCENARIOMAP, \ +from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ TEMPID, missing_orm_classes # get database connection @@ -62,7 +62,7 @@ def _norm(x): # map obj_label to corresponding source results['source'] = None - for k, v in FUEL_TO_SOURCE.items(): + for k, v in OBJ_LABEL_TO_SOURCE.items(): idx = results['obj_label'].str.contains(k) results.loc[idx, 'source'] = v @@ -82,7 +82,7 @@ def _norm(x): # inform the user about possible discrepancies # dataprocessing has no logging - for i, k in FUEL_TO_SOURCE.items(): + for i, k in OBJ_LABEL_TO_SOURCE.items(): if k not in generators['source'].values: logging.info("%s: %s: Object label %s is not matched in powerflow." % ( __file__, scn_name, i)) @@ -99,7 +99,7 @@ def _norm(x): generators = generators[~generators['p_set'].isnull()].copy() # remove solar and wind - sources = [v for k, v in FUEL_TO_SOURCE.items() if k in + sources = [v for k, v in OBJ_LABEL_TO_SOURCE.items() if k in ['wind_offshore', 'wind_onshore', 'solar']] ix = generators['source'].isin(sources) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py index cf63b559..b4c6b3e2 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py @@ -20,7 +20,7 @@ SCENARIOMAP = {'Status Quo': 43, 'NEP 2035': 41, 'eGo 100': 40} # list of available fuel types in eGo scenarios -FUEL_TO_SOURCE = { +OBJ_LABEL_TO_SOURCE = { 'gas': 1, 'lignite': 2, 'waste': 3, diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py index 5f0128c7..29770bff 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py @@ -14,7 +14,7 @@ from dataprocessing.tools.io import oedb_session from sqlalchemy.orm import sessionmaker from sqlalchemy import MetaData, func -from ego_dp_powerflow_timeseries_generator_helper import SOURCE_TO_FUEL, SCENARIOMAP, \ +from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ TEMPID, NEIGHBOURSID, _flatten, map_on_partial_string, renpass_gis_orm_classes from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator, \ EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour @@ -83,7 +83,7 @@ # map from obj label string -> source generators['source'] = map_on_partial_string( - generators['label'], SOURCE_TO_FUEL) + generators['label'], OBJ_LABEL_TO_SOURCE) generators['cntr_id'] = generators['label'].str[:2] @@ -111,7 +111,7 @@ # map from obj label string -> source results['source'] = map_on_partial_string( - results['obj_label'], SOURCE_TO_FUEL).astype(int) + results['obj_label'], OBJ_LABEL_TO_SOURCE).astype(int) results['cntr_id'] = results['obj_label'].str[:2] results['bus'] = results['cntr_id'].map(neighbours['bus_id']) From 5f6447e785e1cd362b95e4b592f8dfecefa10f1f Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 13:19:03 +0200 Subject: [PATCH 22/46] Update header --- .../python_scripts/ego_dp_powerflow_timeseries_feedin.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py index 378df318..784c08d1 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py @@ -1,3 +1,12 @@ +""" This script assigns feedin timeseries data generated with feedinlib +to high-voltage powerflow generators. +""" + +__copyright__ = "ZNES Flensburg" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "wolfbunke" + import pandas as pd from dataprocessing.tools.io import oedb_session From 986ce56b91c684647e7287ded58309118d8247fd Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 13:25:44 +0200 Subject: [PATCH 23/46] Add logging in timeseries generator --- .../ego_dp_powerflow_timeseries_feedin.py | 2 +- .../ego_dp_powerflow_timeseries_generator.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py index 784c08d1..14b1eecc 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py @@ -66,7 +66,7 @@ logged += len(generators) write_ego_scenario_log(conn=conn, - version='v0.3.0', + version='v0.4.0', io='input', schema='model_draft', table=PqSet.__tablename__, diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index ad1843bd..f8328cf8 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -36,7 +36,8 @@ def _norm(x): session.query(PqSet).delete() session.commit() -# p_set +# Assigning p_set for each scenario +logged = 0 for scn_name, scn_nr in SCENARIOMAP.items(): # model_draft.ego_grid_pf_hv_generator -> pd.DataFrame @@ -111,3 +112,13 @@ def _norm(x): session.add(PqSet(**i)) session.commit() + + logged += len(generators) + +write_ego_scenario_log(conn=conn, + version='v0.4.0', + io='input', + schema='model_draft', + table=PqSet.__tablename__, + script=__file__, + entries=logged) From 78a31dc084daeb6ee8f6734f4254f1f3b476a5b8 Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 18:08:06 +0200 Subject: [PATCH 24/46] Move generator assignment for neighbouring countries --- ...werflow_assignment_generator_neighbours.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py b/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py new file mode 100644 index 00000000..bac3beba --- /dev/null +++ b/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py @@ -0,0 +1,110 @@ +""" Transfer renpassG!S scenario definition of LinearTransformers and Sources +to eGo powerflow generator table. +""" + +__copyright__ = "ZNES Flensburg" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "wolfbunke" + +import pandas as pd + +from dataprocessing.tools.io import oedb_session +from sqlalchemy.orm import sessionmaker +from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ + TEMPID, NEIGHBOURSID, _flatten, map_on_partial_string, missing_orm_classes +from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator, \ + EgoGridHvElectricalNeighboursBus as Neighbour + +conn = oedb_session(section='test') +Session = sessionmaker(bind=conn) +session = Session() + +# obligatory delete statement based on NEIGHBOURSID +session.query(Generator).filter(Generator.generator_id >= NEIGHBOURSID).\ + delete(synchronize_session='fetch') + +############################################################################### + +*_, Transformer, Source, Results = missing_orm_classes(session) + +# get DataFrame each row representing one electrical neighbour by applying +# filter on id and v_nom, not affected by scenario name +query = session.query(Neighbour) +neighbours = pd.read_sql(query.statement, query.session.bind) + +ix = (neighbours['id'] <= 27) & (neighbours['v_nom'] == 380) +neighbours = neighbours.loc[ix, :] +neighbours.set_index('cntr_id', inplace=True) + +# for each scenario +for scn_name, scn_nr in SCENARIOMAP.items(): + + # get renpass_gis scenario data on linear transformers. Parameters are + # defined on those edges directed from the component to the bus. + filters = [Transformer.scenario_id == scn_nr, + ~Transformer.source.like('%powerline%'), + Transformer.label == Transformer.source] # direction + + query = session.query(Transformer).filter(*filters) + transformers = pd.read_sql(query.statement, query.session.bind) + transformers['type'] = 'linear transformer' + + # get data on sources + filters = [Source.scenario_id == scn_nr, ~Source.label.like('GL%')] + query = session.query(Source).filter(*filters) + sources = pd.read_sql(query.statement, query.session.bind) + sources['type'] = 'source' + + # sources and transformers, distinct in renpass_gis, are both seen as + # generators and can be handled together + generators = pd.concat([sources, transformers], ignore_index=True) + + # parameters in renpass_gis are not necessarily scalars and stored in lists + # lists of len one are flattened + generators = generators.applymap(_flatten) + + # 0 does not equal zero. In case a class with zero nominal value + # should be defined for the purpose of scenario definition very small + # values are used in the scenario files. + ix = generators['nominal_value'] < 1e-7 + generators = generators.loc[~ix, :] + + # source in the context of eGo has a different meaning. The column has to + # be renamed + generators.rename(columns={'source': 'renpass_gis_source'}, inplace=True) + + # map from obj label string -> source + generators['source'] = map_on_partial_string( + generators['label'], {i: k for k, i in OBJ_LABEL_TO_SOURCE.items()}) + + generators['cntr_id'] = generators['label'].str[:2] + + # exclude Germany + generators = generators.loc[generators['cntr_id'] != 'DE', :] + + # exclude unmatched sources + generators = generators.loc[~generators['source'].isnull(), :] + + # assign bus_ids according to neighbours DataFrame + generators['bus'] = generators['cntr_id'].map(neighbours['bus_id']) + + # set control, and dispatch parameter + generators['control'] = 'PV' + generators['dispatch'] = generators['type'].map( + {'linear transformer': 'flexible', 'source': 'variable'}) + + # set scenario name, temporal id + generators['scn_name'] = scn_name + generators['temp_id'] = TEMPID + generators['generator_id'] = generators.index + NEIGHBOURSID + + # prepare DataFrames to be exported + generator_ex = generators[ + ['scn_name', 'generator_id', 'bus', 'dispatch', 'control', 'source']] + + # write to db + for i in generator_ex.to_dict(orient='records'): + session.add(Generator(**i)) + + session.commit() From 02e34ca559fbb61503df9b5253caca67c4cfd310 Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 18:42:56 +0200 Subject: [PATCH 25/46] Add logging --- ..._powerflow_assignment_generator_neighbours.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py b/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py index bac3beba..9215eb86 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py @@ -1,5 +1,5 @@ -""" Transfer renpassG!S scenario definition of LinearTransformers and Sources -to eGo powerflow generator table. +""" Transfer scenario definition as defined in FlEnS open_eGo scenarios +of LinearTransformers and Sources to eGo powerflow generator table. """ __copyright__ = "ZNES Flensburg" @@ -10,10 +10,11 @@ import pandas as pd from dataprocessing.tools.io import oedb_session +from dataprocessing.python_scripts.functions.ego_scenario_log import write_ego_scenario_log from sqlalchemy.orm import sessionmaker from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ TEMPID, NEIGHBOURSID, _flatten, map_on_partial_string, missing_orm_classes -from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator, \ +from egoio.db_tables.model_draft import EgoGridPfHvGenerator as Generator, \ EgoGridHvElectricalNeighboursBus as Neighbour conn = oedb_session(section='test') @@ -38,6 +39,7 @@ neighbours.set_index('cntr_id', inplace=True) # for each scenario +logged = 0 for scn_name, scn_nr in SCENARIOMAP.items(): # get renpass_gis scenario data on linear transformers. Parameters are @@ -108,3 +110,11 @@ session.add(Generator(**i)) session.commit() + +write_ego_scenario_log(conn=conn, + version='v0.4.0', + io='input', + schema='model_draft', + table=Generator.__tablename__, + script=__file__, + entries=logged) From 6804f86bb9553927e9965b7c920c8f337543bafa Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 18:49:24 +0200 Subject: [PATCH 26/46] Comment out logging info Discrepancies in scenario defintion is handled in a separate issue https://github.com/znes/FlEnS/issues/3 --- .../ego_dp_powerflow_timeseries_generator.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py index f8328cf8..923466a0 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py @@ -81,17 +81,18 @@ def _norm(x): generators['p_set'] = generators['source'].map(results_s) * \ generators['p_nom_fraction'] + # https://github.com/znes/FlEnS/issues/3 # inform the user about possible discrepancies # dataprocessing has no logging - for i, k in OBJ_LABEL_TO_SOURCE.items(): - if k not in generators['source'].values: - logging.info("%s: %s: Object label %s is not matched in powerflow." % ( - __file__, scn_name, i)) - - for k in generators.source.unique(): - if k not in results_s.index: - logging.info("%s: %s: Source %s is not matched in renpass_gis." % ( - __file__, scn_name, int(k))) + #for i, k in OBJ_LABEL_TO_SOURCE.items(): + # if k not in generators['source'].values: + # logging.info("%s: %s: Object label %s is not matched in powerflow." % ( + # __file__, scn_name, i)) + + #for k in generators.source.unique(): + # if k not in results_s.index: + # logging.info("%s: %s: Source %s is not matched in renpass_gis." % ( + # __file__, scn_name, int(k))) # test for missing values for debugging assert set(generators['p_set'].isnull()) == {False}, "P_set field empty in generators table." From 9df8de1af750aa61f3cc1cabc897f5e54a6bbfd5 Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 22:04:13 +0200 Subject: [PATCH 27/46] Rename files --- ...rator.py => ego_dp_powerflow_timeseries_generator_de_p_set.py} | 0 ...eedin.py => ego_dp_powerflow_timeseries_generator_p_max_pu.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename dataprocessing/python_scripts/{ego_dp_powerflow_timeseries_generator.py => ego_dp_powerflow_timeseries_generator_de_p_set.py} (100%) rename dataprocessing/python_scripts/{ego_dp_powerflow_timeseries_feedin.py => ego_dp_powerflow_timeseries_generator_p_max_pu.py} (100%) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py similarity index 100% rename from dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py rename to dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_p_max_pu.py similarity index 100% rename from dataprocessing/python_scripts/ego_dp_powerflow_timeseries_feedin.py rename to dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_p_max_pu.py From c9e0e703cacab4d2680156fb35a5fd6bbb0c440a Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 22:10:58 +0200 Subject: [PATCH 28/46] Rename file --- ...rs.py => ego_dp_powerflow_timeseries_generator_other_p_set.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dataprocessing/python_scripts/{ego_dp_powerflow_timeseries_generator_neighbours.py => ego_dp_powerflow_timeseries_generator_other_p_set.py} (100%) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py similarity index 100% rename from dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py rename to dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py From 1e38ca13099125eaba925514df0b0389d793c270 Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 22:14:25 +0200 Subject: [PATCH 29/46] Filter for neighboursid p_set Germany --- .../ego_dp_powerflow_timeseries_generator_de_p_set.py | 5 +++-- .../ego_dp_powerflow_timeseries_generator_helper.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py index 923466a0..73c30b1e 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py @@ -18,7 +18,7 @@ from egoio.db_tables.model_draft import EgoGridPfHvGenerator as Generator, \ EgoGridPfHvGeneratorPqSet as PqSet from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ - TEMPID, missing_orm_classes + TEMPID, missing_orm_classes, NEIGHBOURSID # get database connection conn = oedb_session(section='test') @@ -41,7 +41,8 @@ def _norm(x): for scn_name, scn_nr in SCENARIOMAP.items(): # model_draft.ego_grid_pf_hv_generator -> pd.DataFrame - query = session.query(Generator).filter(Generator.scn_name == scn_name) + query = session.query(Generator).filter( + Generator.scn_name == scn_name, Generator.generator_id < NEIGHBOURSID) generators = pd.read_sql(query.statement, query.session.bind) assert set(generators['source'].isnull()) == {False}, "Source field empty in generators table." diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py index b4c6b3e2..46a9f5b5 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py @@ -1,4 +1,4 @@ -""" Helper functions and variables to handle renpass_gis tables. +""" Helper functions and constants to handle renpass_gis tables. """ __copyright__ = "ZNES Flensburg" From 6ff96c6f07749779106f408cb0c080d4ba4bb6ae Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 22:32:59 +0200 Subject: [PATCH 30/46] Remove assignment of generators Timeseries_generator_other_p_set does only write timeseries data to the field p_set in case of neighbouring countries. --- ...erflow_timeseries_generator_other_p_set.py | 101 ++++-------------- 1 file changed, 22 insertions(+), 79 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py index 29770bff..d93806ba 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py @@ -13,90 +13,32 @@ from dataprocessing.tools.io import oedb_session from sqlalchemy.orm import sessionmaker -from sqlalchemy import MetaData, func +from dataprocessing.python_scripts.functions.ego_scenario_log import write_ego_scenario_log from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ - TEMPID, NEIGHBOURSID, _flatten, map_on_partial_string, renpass_gis_orm_classes + TEMPID, NEIGHBOURSID, map_on_partial_string, missing_orm_classes from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator, \ - EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour + EgoGridPfHvGeneratorPqSet as PqSet conn = oedb_session(section='test') Session = sessionmaker(bind=conn) session = Session() # obligatory delete statement based on NEIGHBOURSID -session.query(Generator).filter(Generator.generator_id >= NEIGHBOURSID).\ - delete(synchronize_session='fetch') - session.query(PqSet).filter(PqSet.generator_id >= NEIGHBOURSID).\ delete(synchronize_session='fetch') ############################################################################### -Transformer, Source, Results = renpass_gis_orm_classes(session) - -# get DataFrame each row representing one electrical neighbour by applying -# filter on id and v_nom, not affected by scenario name -query = session.query(Neighbour) -neighbours = pd.read_sql(query.statement, query.session.bind) - -ix = (neighbours['id'] <= 27) & (neighbours['v_nom'] == 380) -neighbours = neighbours.loc[ix, :] -neighbours.set_index('cntr_id', inplace=True) - +PowerClass, Feedin, *_, Results = missing_orm_classes(session) +logged = 0 for scn_name, scn_nr in SCENARIOMAP.items(): - # get renpass_gis scenario data on linear transformers. Parameters are - # defined on those edges directed from the component to the bus. - filters = [Transformer.scenario_id == scn_nr, - ~Transformer.source.like('%powerline%'), - Transformer.label == Transformer.source] # direction - - query = session.query(Transformer).filter(*filters) - transformers = pd.read_sql(query.statement, query.session.bind) - transformers['type'] = 'linear transformer' - - # get data on sources - filters = [Source.scenario_id == scn_nr, ~Source.label.like('GL%')] - query = session.query(Source).filter(*filters) - sources = pd.read_sql(query.statement, query.session.bind) - sources['type'] = 'source' - - # sources and transformers, distinct in renpass_gis, are both seen as - # generators and can be handled together - generators = pd.concat([sources, transformers], ignore_index=True) - - # parameters in renpass_gis are not necessarily scalars and stored in lists - # lists of len one are flattened - generators = generators.applymap(_flatten) - - # 0 does not equal zero. In case a class with zero nominal value - # should be defined for the purpose of scenario definition very small values - # are used in the scenario files. - ix = generators['nominal_value'] < 1e-7 - generators = generators.loc[~ix, :] - - # source in the context of eGo has a different meaning. The column has to - # be renamed - generators.rename(columns={'source': 'renpass_gis_source'}, inplace=True) - - # map from obj label string -> source - generators['source'] = map_on_partial_string( - generators['label'], OBJ_LABEL_TO_SOURCE) - - generators['cntr_id'] = generators['label'].str[:2] - - # exclude Germany - generators = generators.loc[generators['cntr_id'] != 'DE', :] - - # assign bus_ids according to neighbours DataFrame - generators['bus'] = generators['cntr_id'].map(neighbours['bus_id']) - - # set control, and dispatch parameter - generators['control'] = 'PV' - generators['dispatch'] = generators['type'].map( - {'linear transformer': 'flexible', 'source': 'variable'}) + # model_draft.ego_grid_pf_hv_generator -> pd.DataFrame + query = session.query(Generator).filter( + Generator.scn_name == scn_name, Generator.generator_id >= NEIGHBOURSID) + generators = pd.read_sql(query.statement, query.session.bind) # get corresponding optimization results from renpass_gis # obj_label, datetime, val @@ -111,10 +53,8 @@ # map from obj label string -> source results['source'] = map_on_partial_string( - results['obj_label'], OBJ_LABEL_TO_SOURCE).astype(int) - - results['cntr_id'] = results['obj_label'].str[:2] - results['bus'] = results['cntr_id'].map(neighbours['bus_id']) + results['obj_label'], + {k: i for i, k in OBJ_LABEL_TO_SOURCE.items()}) # create Series with bus_id, source and power generation / actual value # in list format @@ -131,17 +71,20 @@ generators['scn_name'] = scn_name generators['temp_id'] = TEMPID - generators['generator_id'] = generators.index + NEIGHBOURSID - - # prepare DataFrames to be exported - generator_ex = generators[['scn_name', 'generator_id', 'bus', 'dispatch', 'control']] pqsets = generators[['scn_name', 'generator_id', 'temp_id', 'p_set']] - # write to db - for i in generator_ex.to_dict(orient='records'): - session.add(Generator(**i)) - + # export to db for i in pqsets.to_dict(orient='records'): session.add(PqSet(**i)) session.commit() + + logged += len(pqsets) + +write_ego_scenario_log(conn=conn, + version='v0.4.0', + io='input', + schema='model_draft', + table=PqSet.__tablename__, + script=__file__, + entries=logged) From a8a2138ee686d13791c907aa9b06ec0218457bef Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 22:48:48 +0200 Subject: [PATCH 31/46] Include p_nom in generator assignment --- .../ego_dp_powerflow_assignment_generator_neighbours.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py b/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py index 9215eb86..ff97f8c0 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py @@ -76,6 +76,9 @@ # be renamed generators.rename(columns={'source': 'renpass_gis_source'}, inplace=True) + # rename nominal_value to p_nom + generators.rename(columns={'nominal_value': 'p_nom'}, inplace=True) + # map from obj label string -> source generators['source'] = map_on_partial_string( generators['label'], {i: k for k, i in OBJ_LABEL_TO_SOURCE.items()}) @@ -103,7 +106,8 @@ # prepare DataFrames to be exported generator_ex = generators[ - ['scn_name', 'generator_id', 'bus', 'dispatch', 'control', 'source']] + ['scn_name', 'generator_id', 'bus', 'dispatch', 'control', 'source', + 'p_nom']] # write to db for i in generator_ex.to_dict(orient='records'): From f0132cd32eb09e224d76a6b1a9bda96b1ae2ef74 Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 22:52:20 +0200 Subject: [PATCH 32/46] Replace tab with whitespace in main executable --- dataprocessing/eGo_data_processing.py | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/dataprocessing/eGo_data_processing.py b/dataprocessing/eGo_data_processing.py index 968e901c..50c86656 100644 --- a/dataprocessing/eGo_data_processing.py +++ b/dataprocessing/eGo_data_processing.py @@ -89,23 +89,23 @@ def data_processing(): # 'rea/ego_dp_rea_results.sql', # Results and statistics ## POWERFLOW -# 'ego_dp_powerflow_assignment_otgid.sql', # assign otg_id to pp lists -# 'ego_dp_powerflow_assignment_unid.sql', # create a unified_id over all pp (res and conv) -# 'ego_dp_powerflow_create_pp_mview.sql', # create mviews to display power plants per scenario -# 'ego_dp_powerflow_hv_setup.sql', # Set schema/tables for EHV/HV powerflow calculations up -# 'ego_dp_powerflow_osmtgmod_to_pypsa.sql', # Include data from osmTGmod into EHV/HV powerflow schema -# 'ego_dp_powerflow_electrical_neighbour.sql', # Create border crossing lines and buses in neighbouring countries -# 'ego_dp_powerflow_fix_ehv_subnetworks.sql', # Fix topological errors in eHV grid -# 'ego_dp_powerflow_grid_future_scenarios.sql', # Copy grid to future scenarios -# 'ego_dp_powerflow_assignment_generator.sql', # Assign generators to corresponding substation (SQ, NEP2035, eGo100) -# 'ego_dp_powerflow_assignment_load.sql', # Assign loads to their corresponding substation (SQ, NEP2035, eGo100) -# 'ego_dp_powerflow_assignment_storage.sql', # Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100) -# 'ego_dp_powerflow_timeseries_generator.sql', # Transfer renpassG!S results into the corresponding powerflow table -# 'ego_dp_powerflow_griddistrict_demand.py', # Demand per MV Griddistrict -# 'ego_dp_powerflow_timeseries_demand.sql', # Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100) -# 'ego_dp_powerflow_lopf_data.sql', # Set marginal costs for generators and storages - - +# 'ego_dp_powerflow_assignment_otgid.sql', # assign otg_id to pp lists +# 'ego_dp_powerflow_assignment_unid.sql', # create a unified_id over all pp (res and conv) +# 'ego_dp_powerflow_create_pp_mview.sql', # create mviews to display power plants per scenario +# 'ego_dp_powerflow_hv_setup.sql', # Set schema/tables for EHV/HV powerflow calculations up +# 'ego_dp_powerflow_osmtgmod_to_pypsa.sql', # Include data from osmTGmod into EHV/HV powerflow schema +# 'ego_dp_powerflow_electrical_neighbour.sql', # Create border crossing lines and buses in neighbouring countries +# 'ego_dp_powerflow_fix_ehv_subnetworks.sql', # Fix topological errors in eHV grid +# 'ego_dp_powerflow_grid_future_scenarios.sql', # Copy grid to future scenarios +# 'ego_dp_powerflow_assignment_generator.sql', # Assign generators to corresponding substation (SQ, NEP2035, eGo100) +# 'ego_dp_powerflow_assignment_load.sql', # Assign loads to their corresponding substation (SQ, NEP2035, eGo100) +# 'ego_dp_powerflow_assignment_storage.sql', # Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100) +# 'ego_dp_powerflow_timeseries_generator.sql', # Transfer renpassG!S results into the corresponding powerflow table +# 'ego_dp_powerflow_griddistrict_demand.py', # Demand per MV Griddistrict +# 'ego_dp_powerflow_timeseries_demand.sql', # Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100) +# 'ego_dp_powerflow_lopf_data.sql', # Set marginal costs for generators and storages + + ## VERSIONING # 'ego_dp_versioning.sql', # Versioning # 'ego_dp_versioning_mviews.sql' , # Versioning of mviews From 2948d0cfc471bc7681b92e4949725c6e40d12eaf Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 22:53:46 +0200 Subject: [PATCH 33/46] Update main executable --- dataprocessing/eGo_data_processing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dataprocessing/eGo_data_processing.py b/dataprocessing/eGo_data_processing.py index 50c86656..4c8bc73c 100644 --- a/dataprocessing/eGo_data_processing.py +++ b/dataprocessing/eGo_data_processing.py @@ -98,6 +98,7 @@ def data_processing(): # 'ego_dp_powerflow_fix_ehv_subnetworks.sql', # Fix topological errors in eHV grid # 'ego_dp_powerflow_grid_future_scenarios.sql', # Copy grid to future scenarios # 'ego_dp_powerflow_assignment_generator.sql', # Assign generators to corresponding substation (SQ, NEP2035, eGo100) +# 'ego_dp_powerflow_assignment_generator_neighbours.py', # Create generators for neighbouring countries (SQ, NEP2035, eGo100) # 'ego_dp_powerflow_assignment_load.sql', # Assign loads to their corresponding substation (SQ, NEP2035, eGo100) # 'ego_dp_powerflow_assignment_storage.sql', # Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100) # 'ego_dp_powerflow_timeseries_generator.sql', # Transfer renpassG!S results into the corresponding powerflow table From cb9835404d41ca6ba17abeb50ca3d7165291bdaa Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 23:41:42 +0200 Subject: [PATCH 34/46] Rename file --- ...erator.sql => ego_dp_powerflow_assignment_load_neighbours.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dataprocessing/sql_snippets/{ego_dp_powerflow_timeseries_generator.sql => ego_dp_powerflow_assignment_load_neighbours.sql} (100%) diff --git a/dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql b/dataprocessing/sql_snippets/ego_dp_powerflow_assignment_load_neighbours.sql similarity index 100% rename from dataprocessing/sql_snippets/ego_dp_powerflow_timeseries_generator.sql rename to dataprocessing/sql_snippets/ego_dp_powerflow_assignment_load_neighbours.sql From 71968a2d2f4b53e1f0aee6a192f6806b6515e27e Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 23:43:57 +0200 Subject: [PATCH 35/46] Update main executable --- dataprocessing/eGo_data_processing.py | 6 +-- ...p_powerflow_assignment_load_neighbours.sql | 54 ++----------------- 2 files changed, 7 insertions(+), 53 deletions(-) diff --git a/dataprocessing/eGo_data_processing.py b/dataprocessing/eGo_data_processing.py index 4c8bc73c..37757b40 100644 --- a/dataprocessing/eGo_data_processing.py +++ b/dataprocessing/eGo_data_processing.py @@ -98,10 +98,10 @@ def data_processing(): # 'ego_dp_powerflow_fix_ehv_subnetworks.sql', # Fix topological errors in eHV grid # 'ego_dp_powerflow_grid_future_scenarios.sql', # Copy grid to future scenarios # 'ego_dp_powerflow_assignment_generator.sql', # Assign generators to corresponding substation (SQ, NEP2035, eGo100) -# 'ego_dp_powerflow_assignment_generator_neighbours.py', # Create generators for neighbouring countries (SQ, NEP2035, eGo100) +# 'ego_dp_powerflow_create_generator_neighbours.py', # Create generators for neighbouring countries (SQ, NEP2035, eGo100) # 'ego_dp_powerflow_assignment_load.sql', # Assign loads to their corresponding substation (SQ, NEP2035, eGo100) +# 'ego_dp_powerflow_create_load_neighbours.sql', # Create loads for neighouring countries (SQ, NEP2035, eGo100) # 'ego_dp_powerflow_assignment_storage.sql', # Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100) -# 'ego_dp_powerflow_timeseries_generator.sql', # Transfer renpassG!S results into the corresponding powerflow table # 'ego_dp_powerflow_griddistrict_demand.py', # Demand per MV Griddistrict # 'ego_dp_powerflow_timeseries_demand.sql', # Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100) # 'ego_dp_powerflow_lopf_data.sql', # Set marginal costs for generators and storages @@ -114,7 +114,7 @@ def data_processing(): ## POST-PROCESSING # 'post_processing/ego_pp_nep2035_grid_variations.sql' # Create extension_tables and insert NEP-data - + ## VACUUM FULL # 'ego_dp_vacuum_full.sql' ] diff --git a/dataprocessing/sql_snippets/ego_dp_powerflow_assignment_load_neighbours.sql b/dataprocessing/sql_snippets/ego_dp_powerflow_assignment_load_neighbours.sql index 13d985f1..c9075adc 100644 --- a/dataprocessing/sql_snippets/ego_dp_powerflow_assignment_load_neighbours.sql +++ b/dataprocessing/sql_snippets/ego_dp_powerflow_assignment_load_neighbours.sql @@ -1,25 +1,17 @@ -/* -This is a quick workaround to transfer `renpassG!S results `_ on generator time series -into the corresponding powerflow table. It adds time series (p_set) for generators in Germany and the neighbouring countries to the -`respective table `_ +/* Create entries in hv load for all neighbouring countries and scenarios __copyright__ = "Europa Universitaet Flensburg, Centre for Sustainable Energy Systems" __license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" __url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" __author__ = "wolfbunke, MarlonSchlemminger" - -TODO: storage in storage_pqset #1069 */ - -- DELETE DELETE FROM model_draft.ego_grid_pf_hv_load WHERE bus IN ( SELECT bus_id FROM model_draft.ego_grid_hv_electrical_neighbours_bus WHERE central_bus = TRUE); - --- INSERT neigbouring states in load table --- Status Quo +-- Status quo INSERT into model_draft.ego_grid_pf_hv_load (scn_name, load_id, bus, sign) SELECT @@ -39,8 +31,7 @@ INSERT into model_draft.ego_grid_pf_hv_load (scn_name, load_id, bus, sign) WHERE v_nom = max_v_nom; --- NEP 2035 - +-- NEP2035 INSERT into model_draft.ego_grid_pf_hv_load (scn_name, load_id, bus, sign) SELECT @@ -61,8 +52,7 @@ INSERT into model_draft.ego_grid_pf_hv_load (scn_name, load_id, bus, sign) WHERE v_nom = max_v_nom; --- eGo 100 - +-- eGo100 INSERT into model_draft.ego_grid_pf_hv_load (scn_name, load_id, bus, sign) SELECT @@ -82,41 +72,5 @@ INSERT into model_draft.ego_grid_pf_hv_load (scn_name, load_id, bus, sign) ) EGO WHERE v_nom = max_v_nom; - - - - - - --- Demand timeseries - --- NEP 2035 -/* Handled in ego_dp_powerflow_load_timeseries_NEP2035.sql - -INSERT INTO model_draft.ego_grid_pf_hv_load_pq_set (scn_name, load_id, temp_id, p_set) - - SELECT - 'NEP 2035' AS scn_name, - C.load_id AS load_id, - 1 AS temp_id, - array_agg(SQ.val ORDER BY SQ.datetime) AS p_set - FROM - ( - SELECT *, - max(B.v_nom) over (partition by B.cntr_id) AS max_v_nom - FROM calc_renpass_gis.renpass_gis_results A - join model_draft.ego_grid_hv_electrical_neighbours_bus B - ON (B.cntr_id = substring(A.obj_label, 1, 2)) - WHERE A.obj_label LIKE '%%load%%' - AND B.id <= 27 - AND A.type = 'from_bus' - AND A.scenario_id = 38 - ) SQ - JOIN model_draft.ego_grid_pf_hv_load C on (C.bus = SQ.bus_id) - WHERE SQ.v_nom = SQ.max_v_nom - AND C.scn_name = 'NEP 2035' - GROUP BY C.load_id; -*/ - -- scenario log (project,version,io,schema_name,table_name,script_name,comment) SELECT scenario_log('eGo_DP', 'v0.4.0','output','model_draft','ego_grid_pf_hv_generator_pq_set','ego_dp_powerflow_timeseries_generator.sql',' '); From f74b5f3dd0b633fbef7c7fc1dd0f89676a8c067d Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 23:44:37 +0200 Subject: [PATCH 36/46] Rename file --- ...neighbours.sql => ego_dp_powerflow_create_load_neighbours.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dataprocessing/sql_snippets/{ego_dp_powerflow_assignment_load_neighbours.sql => ego_dp_powerflow_create_load_neighbours.sql} (100%) diff --git a/dataprocessing/sql_snippets/ego_dp_powerflow_assignment_load_neighbours.sql b/dataprocessing/sql_snippets/ego_dp_powerflow_create_load_neighbours.sql similarity index 100% rename from dataprocessing/sql_snippets/ego_dp_powerflow_assignment_load_neighbours.sql rename to dataprocessing/sql_snippets/ego_dp_powerflow_create_load_neighbours.sql From 114dc45262fd1dfb37babeab11b64111ef1a575e Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 17 Apr 2018 23:45:33 +0200 Subject: [PATCH 37/46] Rename file --- ...ghbours.py => ego_dp_powerflow_create_generator_neighbours.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dataprocessing/python_scripts/{ego_dp_powerflow_assignment_generator_neighbours.py => ego_dp_powerflow_create_generator_neighbours.py} (100%) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py b/dataprocessing/python_scripts/ego_dp_powerflow_create_generator_neighbours.py similarity index 100% rename from dataprocessing/python_scripts/ego_dp_powerflow_assignment_generator_neighbours.py rename to dataprocessing/python_scripts/ego_dp_powerflow_create_generator_neighbours.py From 9f885b065258b69a8b32c99c28cbd95a4dcea370 Mon Sep 17 00:00:00 2001 From: Martin Date: Wed, 18 Apr 2018 00:04:24 +0200 Subject: [PATCH 38/46] Update main executable --- dataprocessing/eGo_data_processing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dataprocessing/eGo_data_processing.py b/dataprocessing/eGo_data_processing.py index 37757b40..ec86fc60 100644 --- a/dataprocessing/eGo_data_processing.py +++ b/dataprocessing/eGo_data_processing.py @@ -102,6 +102,9 @@ def data_processing(): # 'ego_dp_powerflow_assignment_load.sql', # Assign loads to their corresponding substation (SQ, NEP2035, eGo100) # 'ego_dp_powerflow_create_load_neighbours.sql', # Create loads for neighouring countries (SQ, NEP2035, eGo100) # 'ego_dp_powerflow_assignment_storage.sql', # Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100) +# 'ego_dp_powerflow_timeseries_generator_de_p_set.py' # Assign p_sets for Germany based on renpassG!S optimization results (SQ, NEP 2035, eGo 100) +# 'ego_dp_powerflow_timeseries_generator_other_p_set.py' # Assign p_sets for neighbouring countries based on renpassG!S optimization results (SQ, NEP 2035, eGo 100) +# 'ego_dp_powerflow_timeseries_generator_p_max_pu.py' # Assign p_max_pu based on feedin timeseries data (SQ, NEP 2035, eGo 100) # 'ego_dp_powerflow_griddistrict_demand.py', # Demand per MV Griddistrict # 'ego_dp_powerflow_timeseries_demand.sql', # Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100) # 'ego_dp_powerflow_lopf_data.sql', # Set marginal costs for generators and storages From fbf450c854a88ded1002212677697a2c2055a03a Mon Sep 17 00:00:00 2001 From: Martin Date: Wed, 18 Apr 2018 00:04:40 +0200 Subject: [PATCH 39/46] Bugfix in other_p_set Neighbours table has to be used to determine the bus_id based on country codes. --- ...erflow_timeseries_generator_other_p_set.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py index d93806ba..a0b97952 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py @@ -1,6 +1,5 @@ -""" -Assign German timeseries data from hidden renpassG!S schema to high voltage -powerflow. +""" Assigns dispatch optimization results from renpassG!S to corresponding +hv powerflow generators for neighbouring countries """ __copyright__ = "ZNES Flensburg" @@ -16,8 +15,8 @@ from dataprocessing.python_scripts.functions.ego_scenario_log import write_ego_scenario_log from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ TEMPID, NEIGHBOURSID, map_on_partial_string, missing_orm_classes -from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator, \ - EgoGridPfHvGeneratorPqSet as PqSet +from egoio.db_tables.model_draft import EgoGridPfHvGenerator as Generator, \ + EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour conn = oedb_session(section='test') Session = sessionmaker(bind=conn) @@ -32,6 +31,15 @@ PowerClass, Feedin, *_, Results = missing_orm_classes(session) +# get DataFrame each row representing one electrical neighbour by applying +# filter on id and v_nom, not affected by scenario name +query = session.query(Neighbour) +neighbours = pd.read_sql(query.statement, query.session.bind) + +ix = (neighbours['id'] <= 27) & (neighbours['v_nom'] == 380) +neighbours = neighbours.loc[ix, :] +neighbours.set_index('cntr_id', inplace=True) + logged = 0 for scn_name, scn_nr in SCENARIOMAP.items(): @@ -56,6 +64,9 @@ results['obj_label'], {k: i for i, k in OBJ_LABEL_TO_SOURCE.items()}) + results['cntr_id'] = results['obj_label'].str[:2] + results['bus'] = results['cntr_id'].map(neighbours['bus_id']) + # create Series with bus_id, source and power generation / actual value # in list format results_s = results.groupby(['bus', 'source'])['val'].apply(np.array) From 77cd70ca697325c41dea7e310bdcebb0975fe3d4 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 22 Apr 2018 15:04:49 +0200 Subject: [PATCH 40/46] Rename file --- ...pu.py => ego_dp_powerflow_timeseries_generator_de_p_max_pu.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dataprocessing/python_scripts/{ego_dp_powerflow_timeseries_generator_p_max_pu.py => ego_dp_powerflow_timeseries_generator_de_p_max_pu.py} (100%) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_p_max_pu.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_max_pu.py similarity index 100% rename from dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_p_max_pu.py rename to dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_max_pu.py From bfa0757993d009cb8a7eb5c160f9880861f9d266 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 22 Apr 2018 15:05:21 +0200 Subject: [PATCH 41/46] Add missing orm class --- ...p_powerflow_timeseries_generator_helper.py | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py index 46a9f5b5..d65318f0 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py @@ -1,16 +1,15 @@ """ Helper functions and constants to handle renpass_gis tables. """ -__copyright__ = "ZNES Flensburg" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "wolfbunke" +__copyright__ = "ZNES Flensburg" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "wolfbunke" import pandas as pd -from dataprocessing.tools.io import oedb_session -from sqlalchemy.orm import sessionmaker -from sqlalchemy import MetaData, Column, Integer, Float, Text, text +from geoalchemy2.types import Geometry +from sqlalchemy import MetaData, Column, Integer, Float, Text, text, BigInteger from sqlalchemy.ext.automap import automap_base from sqlalchemy.dialects.postgresql import ARRAY, DOUBLE_PRECISION @@ -48,6 +47,7 @@ TEMPID = 1 + def missing_orm_classes(session): """ Not yet implemented in ego.io @@ -64,14 +64,14 @@ def missing_orm_classes(session): meta = MetaData() meta.reflect(bind=session.bind, schema='calc_renpass_gis', - only=['renpass_gis_linear_transformer', - 'renpass_gis_source', - 'renpass_gis_results']) + only=['renpass_gis_linear_transformer', + 'renpass_gis_source', + 'renpass_gis_results']) # map to classes Base = automap_base(metadata=meta) - # ormclasses not part of egoio yet + # ormclasses not part of egoio yet class EgoPowerClass(Base): __tablename__ = 'ego_power_class' __table_args__ = {'schema': 'model_draft'} @@ -83,7 +83,6 @@ class EgoPowerClass(Base): h_hub = Column(Float(53)) d_rotor = Column(Float(53)) - class EgoRenewableFeedin(Base): __tablename__ = 'ego_renewable_feedin' __table_args__ = {'schema': 'model_draft'} @@ -95,12 +94,20 @@ class EgoRenewableFeedin(Base): power_class = Column(Integer, primary_key=True, nullable=False) feedin = Column(ARRAY(DOUBLE_PRECISION(precision=53))) + class EgoNeighboursOffshorePoint(Base): + __tablename__ = 'ego_neighbours_offshore_point' + __table_args__ = {'schema': 'model_draft'} + + cntr_id = Column(Text, primary_key=True) + coastdat_id = Column(BigInteger) + geom = Column(Geometry('POINT', 4326)) + Base.prepare() Transformer, Source, Results = Base.classes.renpass_gis_linear_transformer, \ Base.classes.renpass_gis_source, Base.classes.renpass_gis_results - return EgoPowerClass, EgoRenewableFeedin, Transformer, Source, Results + return EgoNeighboursOffshorePoint, EgoPowerClass, EgoRenewableFeedin, Transformer, Source, Results def _flatten(x): From 77284bc32f0e092b35755eb71647107543089636 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 22 Apr 2018 15:05:48 +0200 Subject: [PATCH 42/46] Do not write p_sets for renewables --- .../ego_dp_powerflow_timeseries_generator_other_p_set.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py index a0b97952..cfee259e 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py @@ -29,7 +29,7 @@ ############################################################################### -PowerClass, Feedin, *_, Results = missing_orm_classes(session) +_, PowerClass, Feedin, *_, Results = missing_orm_classes(session) # get DataFrame each row representing one electrical neighbour by applying # filter on id and v_nom, not affected by scenario name @@ -78,6 +78,13 @@ # include timeseries data in generators DataFrame generators = generators.join(results_s, on=['bus', 'source']) + # remove solar and wind + sources = [v for k, v in OBJ_LABEL_TO_SOURCE.items() if k in + ['wind_offshore', 'wind_onshore', 'solar']] + + ix = generators['source'].isin(sources) + generators = generators[~ix].copy() + # set scenario name, temporal id generators['scn_name'] = scn_name generators['temp_id'] = TEMPID From e6fdb88f6615cbdd65c957c4cafeebed2286c0d1 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 22 Apr 2018 15:07:08 +0200 Subject: [PATCH 43/46] Add script handling p_max_pu neighbouring countries --- ...low_timeseries_generator_other_p_max_pu.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_max_pu.py diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_max_pu.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_max_pu.py new file mode 100644 index 00000000..7ec26967 --- /dev/null +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_max_pu.py @@ -0,0 +1,83 @@ +""" This script assigns feedin timeseries data generated with feedinlib +to high-voltage powerflow generators for neighbouring countries. +""" + +__copyright__ = "ZNES Flensburg" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "wolfbunke" + +import pandas as pd + +from dataprocessing.tools.io import oedb_session +from sqlalchemy.orm import sessionmaker +from sqlalchemy import case, func +from dataprocessing.python_scripts.functions.ego_scenario_log \ + import write_ego_scenario_log + +from egoio.db_tables.climate import Cosmoclmgrid +from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle \ + as GeneratorSingle, EgoGridPfHvGeneratorPqSet as PqSet, \ + EgoGridPfHvGenerator as Generator, \ + EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour +from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ + TEMPID, missing_orm_classes + +# get database connection +conn = oedb_session(section='test') +Session = sessionmaker(bind=conn) +session = Session() + +_, PowerClass, Feedin, *_, Results = missing_orm_classes(session) + +# get DataFrame each row representing one electrical neighbour by applying +# filter on id and v_nom, not affected by scenario name +query = session.query(Neighbour) +neighbours = pd.read_sql(query.statement, query.session.bind) + +ix = (neighbours['id'] <= 27) & (neighbours['v_nom'] == 380) +neighbours = neighbours.loc[ix, :] +neighbours.set_index('cntr_id', inplace=True) + +# p_max_pu +logged = 0 + +t = session.query(Cosmoclmgrid.gid.label('w_id'), Neighbour.bus_id).filter( + func.ST_Intersects(Cosmoclmgrid.geom, Neighbour.geom)).subquery() + +sources = ['wind_onshore', 'solar'] +sources_dict = {v: k for k, v in OBJ_LABEL_TO_SOURCE.items() if k in sources} + +casestr = case(sources_dict, value=Generator.source, else_=None).label('source') + +query = session.query(Generator.scn_name, Generator.generator_id, casestr, t.c.w_id).filter( + Generator.bus.in_(neighbours.bus_id)).filter( + t.c.bus_id == Generator.bus, Generator.source.in_(sources_dict)) + +# pd.DataFrame with generator_id, source, w_id +generators = pd.read_sql(query.statement, query.session.bind) + +query = session.query( + Feedin.w_id, Feedin.source, Feedin.feedin.label('p_max_pu')).filter( + Feedin.w_id.in_(generators.w_id), Feedin.power_class.in_([0, 4])) +feedins = pd.read_sql(query.statement, query.session.bind) + +# +generators = generators.merge(feedins, on=['source', 'w_id'], how='inner', validate='many_to_one') +generators['temp_id'] = TEMPID + +fields = ['generator_id', 'p_max_pu', 'scn_name', 'temp_id'] +for i in generators[fields].to_dict(orient='records'): + session.add(PqSet(**i)) + +logged += len(generators) + +session.commit() + +write_ego_scenario_log(conn=conn, + version='v0.4.0', + io='input', + schema='model_draft', + table=PqSet.__tablename__, + script=__file__, + entries=logged) From b66bc1fcad16c18af88d0925e6507a37b5aa3c56 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 22 Apr 2018 15:07:38 +0200 Subject: [PATCH 44/46] Add script handling offshore feedins --- ..._timeseries_generator_offshore_p_max_pu.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py new file mode 100644 index 00000000..d3e0e598 --- /dev/null +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py @@ -0,0 +1,82 @@ +""" This script assigns feedin timeseries data generated with feedinlib +to high-voltage powerflow generators for neighbouring countries. +""" + +__copyright__ = "ZNES Flensburg" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "wolfbunke" + +import pandas as pd + +from dataprocessing.tools.io import oedb_session +from sqlalchemy.orm import sessionmaker +from sqlalchemy import case +from dataprocessing.python_scripts.functions.ego_scenario_log \ + import write_ego_scenario_log + +from egoio.db_tables.model_draft import EgoGridPfHvGenerator as Generator, \ + EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour +from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE,\ + TEMPID, missing_orm_classes + +# get database connection +conn = oedb_session(section='test') +Session = sessionmaker(bind=conn) +session = Session() + +OffshorePoints, PowerClass, Feedin, *_, Results = missing_orm_classes(session) + +# get DataFrame each row representing one electrical neighbour by applying +# filter on id and v_nom, not affected by scenario name +query = session.query(Neighbour) +neighbours = pd.read_sql(query.statement, query.session.bind) + +ix = (neighbours['id'] <= 27) & (neighbours['v_nom'] == 380) +neighbours = neighbours.loc[ix, :] +neighbours.set_index('cntr_id', inplace=True) + +# p_max_pu +logged = 0 + +# assign w_id to neighbours definition +neighbours['w_id'] = pd.Series(dict( + session.query(OffshorePoints.cntr_id, OffshorePoints.coastdat_id).all())) +neighbours = neighbours[~neighbours['w_id'].isna()].copy() +neighbours['w_id'] = neighbours['w_id'].astype(int) + +sources = ['wind_offshore'] +sources_dict = {v: k for k, v in OBJ_LABEL_TO_SOURCE.items() if k in sources} + +bus_to_wid = dict(neighbours[['bus_id', 'w_id']].values.tolist()) + +casestr = case(bus_to_wid, value=Generator.bus, else_=None).label('w_id') + +query = session.query(Generator.scn_name, Generator.bus, Generator.generator_id, casestr).filter( + Generator.bus.in_(neighbours.bus_id)).filter(Generator.source.in_(sources_dict)) + +generators = pd.read_sql(query.statement, query.session.bind) + +query = session.query( + Feedin.w_id, Feedin.feedin.label('p_max_pu')).filter( + Feedin.w_id.in_(generators.w_id), Feedin.power_class.in_([0, 4])) +feedins = pd.read_sql(query.statement, query.session.bind) + +generators = generators.merge(feedins, on=['w_id'], how='inner', validate='many_to_one') +generators['temp_id'] = TEMPID + +fields = ['generator_id', 'p_max_pu', 'scn_name', 'temp_id'] +for i in generators[fields].to_dict(orient='records'): + session.add(PqSet(**i)) + +logged += len(generators) + +session.commit() + +write_ego_scenario_log(conn=conn, + version='v0.4.0', + io='input', + schema='model_draft', + table=PqSet.__tablename__, + script=__file__, + entries=logged) From 6657ae7caa73e63dd9b2038f1885116929b3b97c Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 22 Apr 2018 15:57:09 +0200 Subject: [PATCH 45/46] Update main executable --- dataprocessing/eGo_data_processing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dataprocessing/eGo_data_processing.py b/dataprocessing/eGo_data_processing.py index ec86fc60..f107dceb 100644 --- a/dataprocessing/eGo_data_processing.py +++ b/dataprocessing/eGo_data_processing.py @@ -104,7 +104,9 @@ def data_processing(): # 'ego_dp_powerflow_assignment_storage.sql', # Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100) # 'ego_dp_powerflow_timeseries_generator_de_p_set.py' # Assign p_sets for Germany based on renpassG!S optimization results (SQ, NEP 2035, eGo 100) # 'ego_dp_powerflow_timeseries_generator_other_p_set.py' # Assign p_sets for neighbouring countries based on renpassG!S optimization results (SQ, NEP 2035, eGo 100) -# 'ego_dp_powerflow_timeseries_generator_p_max_pu.py' # Assign p_max_pu based on feedin timeseries data (SQ, NEP 2035, eGo 100) +# 'ego_dp_powerflow_timeseries_generator_de_p_max_pu.py' # Assign p_max_pu based on feedin timeseries data (Germany) (SQ, NEP 2035, eGo 100) +# 'ego_dp_powerflow_timeseries_generator_other_p_max_pu.py' # Assign p_max_pu based on feedin timeseries data (Neighbouring Countries) (SQ, NEP 2035, eGo 100) +# 'ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py' # Assign p_max_pu based on feedin timeseries data (Neighbouring countries Offshore) (SQ, NEP 2035, eGo 100) # 'ego_dp_powerflow_griddistrict_demand.py', # Demand per MV Griddistrict # 'ego_dp_powerflow_timeseries_demand.sql', # Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100) # 'ego_dp_powerflow_lopf_data.sql', # Set marginal costs for generators and storages From ea28b6ab5f3a2120ebbaed699964ec5b904ad558 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 22 Apr 2018 15:57:32 +0200 Subject: [PATCH 46/46] Update comments / pep8 --- ...erflow_timeseries_generator_de_p_max_pu.py | 48 ++++++++++++------- ...powerflow_timeseries_generator_de_p_set.py | 9 ++-- ..._timeseries_generator_offshore_p_max_pu.py | 36 ++++++++++---- ...low_timeseries_generator_other_p_max_pu.py | 42 ++++++++++------ ...erflow_timeseries_generator_other_p_set.py | 10 ++-- 5 files changed, 95 insertions(+), 50 deletions(-) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_max_pu.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_max_pu.py index 14b1eecc..e89b4305 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_max_pu.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_max_pu.py @@ -1,5 +1,5 @@ """ This script assigns feedin timeseries data generated with feedinlib -to high-voltage powerflow generators. +as p_max_pu parameter to high-voltage powerflow generators within Germany. """ __copyright__ = "ZNES Flensburg" @@ -20,43 +20,55 @@ from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ TEMPID, missing_orm_classes -# get database connection +# Get database connection conn = oedb_session(section='test') Session = sessionmaker(bind=conn) session = Session() -PowerClass, Feedin, *_, Results = missing_orm_classes(session) +_, PowerClass, Feedin, *_, Results = missing_orm_classes(session) -# p_max_pu logged = 0 for scn_name, scn_nr in SCENARIOMAP.items(): + # Select the correct sources + # Map source_id to feedin source name sources = ['wind_onshore', 'wind_offshore', 'solar'] - sources_dict = {k: v for k, v in OBJ_LABEL_TO_SOURCE.items() if k in sources} + sources_dict = { + k: v for k, v in OBJ_LABEL_TO_SOURCE.items() if k in sources} + + # Wrap case around feedin source to return source_id instead of name casestr = case(sources_dict, value=Feedin.source, else_=None) - # construct subquery from PfGeneratorSingle with unique aggr_id, source, - # w_id, power_class - filters = (GeneratorSingle.scn_name == scn_name, GeneratorSingle.aggr_id != None) + # Get a unique represenation of PfGeneratorSingle with regard to aggr_id, + # source, w_id, power_class + filters = ( + GeneratorSingle.scn_name == scn_name, GeneratorSingle.aggr_id != None) fields = [GeneratorSingle.aggr_id, GeneratorSingle.source, GeneratorSingle.w_id, GeneratorSingle.power_class] grouper = GeneratorSingle.aggr_id, GeneratorSingle.source,\ GeneratorSingle.w_id, GeneratorSingle.power_class + # Construct a subquery with filters, fields and grouper t = session.query(*fields).group_by(*grouper).filter(*filters).subquery() - # use subquery to select the corresponding feedin - query = session.query(t, Feedin.feedin.label('p_max_pu')).filter(t.c.w_id == Feedin.w_id, - t.c.power_class == Feedin.power_class, t.c.source == casestr) + # Use subquery and Feedin table to get the correct feedin for each + # generator + query = session.query(t, Feedin.feedin.label('p_max_pu')).filter( + t.c.w_id == Feedin.w_id, t.c.power_class == Feedin.power_class, + t.c.source == casestr) generators = pd.read_sql(query.statement, query.session.bind) - # rename column aggr_id to generator_id + # Substation's aggr_id are the basis for generators in high-voltage + # powerflow + # Rename column aggr_id to generator_id generators.rename(columns={'aggr_id': 'generator_id'}, inplace=True) generators['temp_id'] = TEMPID generators['scn_name'] = scn_name + # Select fields for PqSets + # Write PqSets to database fields = ['generator_id', 'p_max_pu', 'scn_name', 'temp_id'] for i in generators[fields].to_dict(orient='records'): session.add(PqSet(**i)) @@ -66,9 +78,9 @@ logged += len(generators) write_ego_scenario_log(conn=conn, - version='v0.4.0', - io='input', - schema='model_draft', - table=PqSet.__tablename__, - script=__file__, - entries=logged) + version='v0.4.0', + io='input', + schema='model_draft', + table=PqSet.__tablename__, + script=__file__, + entries=logged) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py index 73c30b1e..8ab94175 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_set.py @@ -1,5 +1,6 @@ -""" This script assign optimized dispatch timeseries data based on renpassG!S -results to high-voltage powerflow generators. +""" +This script assign optimized dispatch timeseries data based on renpassG!S +results to generators in the high-voltage powerflow tables. """ __copyright__ = "ZNES Flensburg" @@ -25,7 +26,7 @@ Session = sessionmaker(bind=conn) session = Session() -PowerClass, Feedin, *_, Results = missing_orm_classes(session) +_, PowerClass, Feedin, *_, Results = missing_orm_classes(session) ############################################################################### @@ -109,6 +110,8 @@ def _norm(x): generators = generators[~ix].copy() generators['temp_id'] = TEMPID + + # write PqSets to the database fields = ['generator_id', 'p_set', 'scn_name', 'temp_id'] for i in generators[fields].to_dict(orient='records'): session.add(PqSet(**i)) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py index d3e0e598..4e299e6a 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py @@ -1,5 +1,6 @@ -""" This script assigns feedin timeseries data generated with feedinlib -to high-voltage powerflow generators for neighbouring countries. +""" This script assigns offshore timeseries data generated with feedinlib +as parameter p_max_pu to high-voltage powerflow generators +in case of neighbouring countries. """ __copyright__ = "ZNES Flensburg" @@ -15,20 +16,23 @@ from dataprocessing.python_scripts.functions.ego_scenario_log \ import write_ego_scenario_log -from egoio.db_tables.model_draft import EgoGridPfHvGenerator as Generator, \ - EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour +from egoio.db_tables.model_draft import \ + EgoGridPfHvGenerator as Generator, \ + EgoGridPfHvGeneratorPqSet as PqSet, \ + EgoGridHvElectricalNeighboursBus as Neighbour from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE,\ TEMPID, missing_orm_classes -# get database connection +# Get database connection conn = oedb_session(section='test') Session = sessionmaker(bind=conn) session = Session() OffshorePoints, PowerClass, Feedin, *_, Results = missing_orm_classes(session) -# get DataFrame each row representing one electrical neighbour by applying -# filter on id and v_nom, not affected by scenario name +# TODO: This part is used very often and could be outsourced +# Read in a DataFrame with data of neighbouring countries +# Filtered on id, v_nom query = session.query(Neighbour) neighbours = pd.read_sql(query.statement, query.session.bind) @@ -36,35 +40,47 @@ neighbours = neighbours.loc[ix, :] neighbours.set_index('cntr_id', inplace=True) -# p_max_pu +# logging variable logged = 0 -# assign w_id to neighbours definition +# With OffshorePoints, a representation of iso-country-code to w_id, +# assign w_id to data of neighbouring countries +# Delete countries that have no w_id assigned / have no offshore generation neighbours['w_id'] = pd.Series(dict( session.query(OffshorePoints.cntr_id, OffshorePoints.coastdat_id).all())) neighbours = neighbours[~neighbours['w_id'].isna()].copy() neighbours['w_id'] = neighbours['w_id'].astype(int) +# Map source name to source_id sources = ['wind_offshore'] sources_dict = {v: k for k, v in OBJ_LABEL_TO_SOURCE.items() if k in sources} +# Map bus to w_id bus_to_wid = dict(neighbours[['bus_id', 'w_id']].values.tolist()) +# Wrap mapping case around Generator.bus to return w_id instead casestr = case(bus_to_wid, value=Generator.bus, else_=None).label('w_id') +# Retrieve existing offshore Generators query = session.query(Generator.scn_name, Generator.bus, Generator.generator_id, casestr).filter( Generator.bus.in_(neighbours.bus_id)).filter(Generator.source.in_(sources_dict)) generators = pd.read_sql(query.statement, query.session.bind) +# Select wind_offshore feedins that match the weather_id query = session.query( Feedin.w_id, Feedin.feedin.label('p_max_pu')).filter( - Feedin.w_id.in_(generators.w_id), Feedin.power_class.in_([0, 4])) + Feedin.source.in_(sources_dict.values()), + Feedin.w_id.in_(generators.w_id), + Feedin.power_class.in_([0, 4])) feedins = pd.read_sql(query.statement, query.session.bind) +# Merge DataFrames on weather_id +# This is a many_to_one relation because of multiple scenarios generators = generators.merge(feedins, on=['w_id'], how='inner', validate='many_to_one') generators['temp_id'] = TEMPID +# Write PqSets to the database fields = ['generator_id', 'p_max_pu', 'scn_name', 'temp_id'] for i in generators[fields].to_dict(orient='records'): session.add(PqSet(**i)) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_max_pu.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_max_pu.py index 7ec26967..15078c01 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_max_pu.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_max_pu.py @@ -1,7 +1,7 @@ -""" This script assigns feedin timeseries data generated with feedinlib -to high-voltage powerflow generators for neighbouring countries. +""" This script assigns solar and wind_onshore data generated with feedinlib +as p_max_pu parameter to high-voltage powerflow generators for neighbouring +countries. """ - __copyright__ = "ZNES Flensburg" __license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" __url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" @@ -16,22 +16,26 @@ import write_ego_scenario_log from egoio.db_tables.climate import Cosmoclmgrid -from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle \ - as GeneratorSingle, EgoGridPfHvGeneratorPqSet as PqSet, \ +from egoio.db_tables.model_draft import \ + EgoSupplyPfGeneratorSingle as GeneratorSingle, + EgoGridPfHvGeneratorPqSet as PqSet, \ EgoGridPfHvGenerator as Generator, \ - EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour + EgoGridPfHvGeneratorPqSet as PqSet, \ + EgoGridHvElectricalNeighboursBus as Neighbour from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ TEMPID, missing_orm_classes -# get database connection +# There is no need for a scenario loop. This can be done in one operation. + +# Get database connection conn = oedb_session(section='test') Session = sessionmaker(bind=conn) session = Session() _, PowerClass, Feedin, *_, Results = missing_orm_classes(session) -# get DataFrame each row representing one electrical neighbour by applying -# filter on id and v_nom, not affected by scenario name +# Read in a DataFrame with data of neighbouring countries +# Filtered on id, v_nom query = session.query(Neighbour) neighbours = pd.read_sql(query.statement, query.session.bind) @@ -39,33 +43,41 @@ neighbours = neighbours.loc[ix, :] neighbours.set_index('cntr_id', inplace=True) -# p_max_pu +# logging variable logged = 0 +# Construct a subquery with w_id, bus_id based on intersecting geometries t = session.query(Cosmoclmgrid.gid.label('w_id'), Neighbour.bus_id).filter( func.ST_Intersects(Cosmoclmgrid.geom, Neighbour.geom)).subquery() +# Make a mapping from source_id to source name sources = ['wind_onshore', 'solar'] sources_dict = {v: k for k, v in OBJ_LABEL_TO_SOURCE.items() if k in sources} +# Wrap mapping case around Generator.source to retrieve source name casestr = case(sources_dict, value=Generator.source, else_=None).label('source') -query = session.query(Generator.scn_name, Generator.generator_id, casestr, t.c.w_id).filter( - Generator.bus.in_(neighbours.bus_id)).filter( - t.c.bus_id == Generator.bus, Generator.source.in_(sources_dict)) +# Get generators and include w_id based on subquery, source name based on case +query = session.query( + Generator.scn_name, Generator.generator_id, casestr, t.c.w_id).filter( + Generator.bus.in_(neighbours.bus_id), t.c.bus_id == Generator.bus, + Generator.source.in_(sources_dict)) -# pd.DataFrame with generator_id, source, w_id generators = pd.read_sql(query.statement, query.session.bind) +# Get feedins and name timeseries p_max_pu query = session.query( Feedin.w_id, Feedin.source, Feedin.feedin.label('p_max_pu')).filter( Feedin.w_id.in_(generators.w_id), Feedin.power_class.in_([0, 4])) feedins = pd.read_sql(query.statement, query.session.bind) -# +# Merge DataFrames +# Assign p_max_pu to generator based on source, w_id +# This is many to one, because of multiple scenario values generators = generators.merge(feedins, on=['source', 'w_id'], how='inner', validate='many_to_one') generators['temp_id'] = TEMPID +# Construct PqSets and write to database fields = ['generator_id', 'p_max_pu', 'scn_name', 'temp_id'] for i in generators[fields].to_dict(orient='records'): session.add(PqSet(**i)) diff --git a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py index cfee259e..b67f8105 100644 --- a/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py +++ b/dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_other_p_set.py @@ -1,5 +1,6 @@ -""" Assigns dispatch optimization results from renpassG!S to corresponding -hv powerflow generators for neighbouring countries +""" +Assigns dispatch optimization results from renpassG!S to generators connected +to neighbouring countries in high-voltage powerflow tables. """ __copyright__ = "ZNES Flensburg" @@ -13,8 +14,9 @@ from dataprocessing.tools.io import oedb_session from sqlalchemy.orm import sessionmaker from dataprocessing.python_scripts.functions.ego_scenario_log import write_ego_scenario_log -from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \ - TEMPID, NEIGHBOURSID, map_on_partial_string, missing_orm_classes +from ego_dp_powerflow_timeseries_generator_helper import \ + OBJ_LABEL_TO_SOURCE, SCENARIOMAP, TEMPID, NEIGHBOURSID, \ + map_on_partial_string, missing_orm_classes from egoio.db_tables.model_draft import EgoGridPfHvGenerator as Generator, \ EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour