Fixes/#220

dataprocessing/eGo_data_processing.py

@@ -4,7 +4,7 @@
 Also see corresponding BPML diagram.
 """
 
-__copyright__ = "Reiner Lemoine Institut"
+__copyright__ = "Reiner Lemoine Institut gGmbH"
 __license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)"
 __url__ = "https://github.com/openego/data_processing/blob/master/LICENSE"
 __author__ = "gplssm, Ludee"
@@ -89,31 +89,37 @@ def data_processing():
 #    'rea/ego_dp_rea_results.sql',                       # Results and statistics
 
 	## POWERFLOW
-#	'ego_dp_powerflow_assignment_otgid.sql',		# assign otg_id to pp lists
-#	'ego_dp_powerflow_assignment_unid.sql',			# create a unified_id over all pp (res and conv) 
-#	'ego_dp_powerflow_create_pp_mview.sql',			# create mviews to display power plants per scenario
-#	'ego_dp_powerflow_hv_setup.sql',			# Set schema/tables for EHV/HV powerflow calculations up
-#	'ego_dp_powerflow_osmtgmod_to_pypsa.sql',		# Include data from osmTGmod into EHV/HV powerflow schema
-#	'ego_dp_powerflow_electrical_neighbour.sql',		# Create border crossing lines and buses in neighbouring countries
-#	'ego_dp_powerflow_fix_ehv_subnetworks.sql',		# Fix topological errors in eHV grid
-#	'ego_dp_powerflow_grid_future_scenarios.sql',		# Copy grid to future scenarios
-#	'ego_dp_powerflow_assignment_generator.sql',  		# Assign generators to corresponding substation (SQ, NEP2035, eGo100)
-#	'ego_dp_powerflow_assignment_load.sql',        		# Assign loads to their corresponding substation (SQ, NEP2035, eGo100)
-#	'ego_dp_powerflow_assignment_storage.sql',		# Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100)
-#	'ego_dp_powerflow_timeseries_generator.sql',		# Transfer renpassG!S results into the corresponding powerflow table
-#	'ego_dp_powerflow_griddistrict_demand.py',		# Demand per MV Griddistrict
-#	'ego_dp_powerflow_timeseries_demand.sql',		# Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100)
-#	'ego_dp_powerflow_lopf_data.sql',			# Set marginal costs for generators and storages
-
-
+#    'ego_dp_powerflow_assignment_otgid.sql',                       # assign otg_id to pp lists
+#    'ego_dp_powerflow_assignment_unid.sql',                        # create a unified_id over all pp (res and conv)
+#    'ego_dp_powerflow_create_pp_mview.sql',                        # create mviews to display power plants per scenario
+#    'ego_dp_powerflow_hv_setup.sql',                               # Set schema/tables for EHV/HV powerflow calculations up
+#    'ego_dp_powerflow_osmtgmod_to_pypsa.sql',                      # Include data from osmTGmod into EHV/HV powerflow schema
+#    'ego_dp_powerflow_electrical_neighbour.sql',                   # Create border crossing lines and buses in neighbouring countries
+#    'ego_dp_powerflow_fix_ehv_subnetworks.sql',                    # Fix topological errors in eHV grid
+#    'ego_dp_powerflow_grid_future_scenarios.sql',                  # Copy grid to future scenarios
+#    'ego_dp_powerflow_assignment_generator.sql',                   # Assign generators to corresponding substation (SQ, NEP2035, eGo100)
+#    'ego_dp_powerflow_create_generator_neighbours.py',             # Create generators for neighbouring countries (SQ, NEP2035, eGo100)
+#    'ego_dp_powerflow_assignment_load.sql',                        # Assign loads to their corresponding substation (SQ, NEP2035, eGo100)
+#    'ego_dp_powerflow_create_load_neighbours.sql',                 # Create loads for neighouring countries (SQ, NEP2035, eGo100)
+#    'ego_dp_powerflow_assignment_storage.sql',                     # Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100)
+#    'ego_dp_powerflow_timeseries_generator_de_p_set.py'            # Assign p_sets for Germany based on renpassG!S optimization results (SQ, NEP 2035, eGo 100)
+#    'ego_dp_powerflow_timeseries_generator_other_p_set.py'         # Assign p_sets for neighbouring countries based on renpassG!S optimization results (SQ, NEP 2035, eGo 100)
+#    'ego_dp_powerflow_timeseries_generator_de_p_max_pu.py'         # Assign p_max_pu based on feedin timeseries data (Germany) (SQ, NEP 2035, eGo 100)
+#    'ego_dp_powerflow_timeseries_generator_other_p_max_pu.py'      # Assign p_max_pu based on feedin timeseries data (Neighbouring Countries) (SQ, NEP 2035, eGo 100)
+#    'ego_dp_powerflow_timeseries_generator_offshore_p_max_pu.py'   # Assign p_max_pu based on feedin timeseries data (Neighbouring countries Offshore) (SQ, NEP 2035, eGo 100)
+#    'ego_dp_powerflow_griddistrict_demand.py',                     # Demand per MV Griddistrict
+#    'ego_dp_powerflow_timeseries_demand.sql',                      # Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100)
+#    'ego_dp_powerflow_lopf_data.sql',                              # Set marginal costs for generators and storages
+
+
     ## VERSIONING
 #   	'ego_dp_versioning.sql',				# Versioning
 #	'ego_dp_versioning_mviews.sql' ,			# Versioning of mviews
 
 	## POST-PROCESSING
 #	'post_processing/ego_pp_nep2035_grid_variations.sql'	# Create extension_tables and insert NEP-data
 
-    
+
 	## VACUUM FULL
 #	 'ego_dp_vacuum_full.sql'
     ]

dataprocessing/python_scripts/ego_dp_powerflow_create_generator_neighbours.py

@@ -0,0 +1,124 @@
+""" Transfer scenario definition as defined in FlEnS open_eGo scenarios
+of LinearTransformers and Sources to eGo powerflow generator table.
+"""
+
+__copyright__ 	= "ZNES Flensburg"
+__license__ 	= "GNU Affero General Public License Version 3 (AGPL-3.0)"
+__url__ 		= "https://github.com/openego/data_processing/blob/master/LICENSE"
+__author__ 		= "wolfbunke"
+
+import pandas as pd
+
+from dataprocessing.tools.io import oedb_session
+from dataprocessing.python_scripts.functions.ego_scenario_log import write_ego_scenario_log
+from sqlalchemy.orm import sessionmaker
+from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \
+    TEMPID, NEIGHBOURSID, _flatten, map_on_partial_string, missing_orm_classes
+from egoio.db_tables.model_draft import EgoGridPfHvGenerator as Generator, \
+    EgoGridHvElectricalNeighboursBus as Neighbour
+
+conn = oedb_session(section='test')
+Session = sessionmaker(bind=conn)
+session = Session()
+
+# obligatory delete statement based on NEIGHBOURSID
+session.query(Generator).filter(Generator.generator_id >= NEIGHBOURSID).\
+    delete(synchronize_session='fetch')
+
+###############################################################################
+
+*_, Transformer, Source, Results = missing_orm_classes(session)
+
+# get DataFrame each row representing one electrical neighbour by applying
+# filter on id and v_nom, not affected by scenario name
+query = session.query(Neighbour)
+neighbours = pd.read_sql(query.statement, query.session.bind)
+
+ix = (neighbours['id'] <= 27) & (neighbours['v_nom'] == 380)
+neighbours = neighbours.loc[ix, :]
+neighbours.set_index('cntr_id', inplace=True)
+
+# for each scenario
+logged = 0
+for scn_name, scn_nr in SCENARIOMAP.items():
+
+    # get renpass_gis scenario data on linear transformers. Parameters are
+    # defined on those edges directed from the component to the bus.
+    filters = [Transformer.scenario_id == scn_nr,
+               ~Transformer.source.like('%powerline%'),
+               Transformer.label == Transformer.source]  # direction
+
+    query = session.query(Transformer).filter(*filters)
+    transformers = pd.read_sql(query.statement, query.session.bind)
+    transformers['type'] = 'linear transformer'
+
+    # get data on sources
+    filters = [Source.scenario_id == scn_nr, ~Source.label.like('GL%')]
+    query = session.query(Source).filter(*filters)
+    sources = pd.read_sql(query.statement, query.session.bind)
+    sources['type'] = 'source'
+
+    # sources and transformers, distinct in renpass_gis, are both seen as
+    # generators and can be handled together
+    generators = pd.concat([sources, transformers], ignore_index=True)
+
+    # parameters in renpass_gis are not necessarily scalars and stored in lists
+    # lists of len one are flattened
+    generators = generators.applymap(_flatten)
+
+    # 0 does not equal zero. In case a class with zero nominal value
+    # should be defined for the purpose of scenario definition very small
+    # values are used in the scenario files.
+    ix = generators['nominal_value'] < 1e-7
+    generators = generators.loc[~ix, :]
+
+    # source in the context of eGo has a different meaning. The column has to
+    # be renamed
+    generators.rename(columns={'source': 'renpass_gis_source'}, inplace=True)
+
+    # rename nominal_value to p_nom
+    generators.rename(columns={'nominal_value': 'p_nom'}, inplace=True)
+
+    # map from obj label string -> source
+    generators['source'] = map_on_partial_string(
+        generators['label'], {i: k for k, i in OBJ_LABEL_TO_SOURCE.items()})
+
+    generators['cntr_id'] = generators['label'].str[:2]
+
+    # exclude Germany
+    generators = generators.loc[generators['cntr_id'] != 'DE', :]
+
+    # exclude unmatched sources
+    generators = generators.loc[~generators['source'].isnull(), :]
+
+    # assign bus_ids according to neighbours DataFrame
+    generators['bus'] = generators['cntr_id'].map(neighbours['bus_id'])
+
+    # set control, and dispatch parameter
+    generators['control'] = 'PV'
+    generators['dispatch'] = generators['type'].map(
+        {'linear transformer': 'flexible', 'source': 'variable'})
+
+    # set scenario name, temporal id
+    generators['scn_name'] = scn_name
+    generators['temp_id'] = TEMPID
+    generators['generator_id'] = generators.index + NEIGHBOURSID
+
+    # prepare DataFrames to be exported
+    generator_ex = generators[
+        ['scn_name', 'generator_id', 'bus', 'dispatch', 'control', 'source',
+         'p_nom']]
+
+    # write to db
+    for i in generator_ex.to_dict(orient='records'):
+        session.add(Generator(**i))
+
+    session.commit()
+
+write_ego_scenario_log(conn=conn,
+                       version='v0.4.0',
+                       io='input',
+                       schema='model_draft',
+                       table=Generator.__tablename__,
+                       script=__file__,
+                       entries=logged)

dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_de_p_max_pu.py

@@ -0,0 +1,86 @@
+""" This script assigns feedin timeseries data generated with feedinlib
+as p_max_pu parameter to high-voltage powerflow generators within Germany.
+"""
+
+__copyright__ = "ZNES Flensburg"
+__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)"
+__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE"
+__author__ = "wolfbunke"
+
+import pandas as pd
+
+from dataprocessing.tools.io import oedb_session
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import case
+from dataprocessing.python_scripts.functions.ego_scenario_log \
+    import write_ego_scenario_log
+
+from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle \
+    as GeneratorSingle, EgoGridPfHvGeneratorPqSet as PqSet
+from ego_dp_powerflow_timeseries_generator_helper import OBJ_LABEL_TO_SOURCE, SCENARIOMAP, \
+    TEMPID, missing_orm_classes
+
+# Get database connection
+conn = oedb_session(section='test')
+Session = sessionmaker(bind=conn)
+session = Session()
+
+_, PowerClass, Feedin, *_, Results = missing_orm_classes(session)
+
+logged = 0
+for scn_name, scn_nr in SCENARIOMAP.items():
+
+    # Select the correct sources
+    # Map source_id to feedin source name
+    sources = ['wind_onshore', 'wind_offshore', 'solar']
+    sources_dict = {
+        k: v for k, v in OBJ_LABEL_TO_SOURCE.items() if k in sources}
+
+    # Wrap case around feedin source to return source_id instead of name
+    casestr = case(sources_dict, value=Feedin.source, else_=None)
+
+    # Get a unique represenation of PfGeneratorSingle with regard to aggr_id,
+    # source, w_id, power_class
+    filters = (
+        GeneratorSingle.scn_name == scn_name, GeneratorSingle.aggr_id != None)
+    fields = [GeneratorSingle.aggr_id, GeneratorSingle.source,
+              GeneratorSingle.w_id, GeneratorSingle.power_class]
+    grouper = GeneratorSingle.aggr_id, GeneratorSingle.source,\
+        GeneratorSingle.w_id, GeneratorSingle.power_class
+
+    # Construct a subquery with filters, fields and grouper
+    t = session.query(*fields).group_by(*grouper).filter(*filters).subquery()
+
+    # Use subquery and Feedin table to get the correct feedin for each
+    # generator
+    query = session.query(t, Feedin.feedin.label('p_max_pu')).filter(
+        t.c.w_id == Feedin.w_id, t.c.power_class == Feedin.power_class,
+        t.c.source == casestr)
+
+    generators = pd.read_sql(query.statement, query.session.bind)
+
+    # Substation's aggr_id are the basis for generators in high-voltage
+    # powerflow
+    # Rename column aggr_id to generator_id
+    generators.rename(columns={'aggr_id': 'generator_id'}, inplace=True)
+
+    generators['temp_id'] = TEMPID
+    generators['scn_name'] = scn_name
+
+    # Select fields for PqSets
+    # Write PqSets to database
+    fields = ['generator_id', 'p_max_pu', 'scn_name', 'temp_id']
+    for i in generators[fields].to_dict(orient='records'):
+        session.add(PqSet(**i))
+
+    session.commit()
+
+    logged += len(generators)
+
+write_ego_scenario_log(conn=conn,
+                       version='v0.4.0',
+                       io='input',
+                       schema='model_draft',
+                       table=PqSet.__tablename__,
+                       script=__file__,
+                       entries=logged)