Fixes/#220

dataprocessing/eGo_data_processing.py

@@ -104,6 +104,8 @@ def data_processing():
 	'ego_dp_powerflow_assignment_generator.sql',  		# Assign generators to corresponding substation (SQ, NEP2035, eGo100)
 	'ego_dp_powerflow_assignment_load.sql',        		# Assign loads to their corresponding substation (SQ, NEP2035, eGo100)
 	'ego_dp_powerflow_assignment_storage.sql',		# Assign storages to their corresponding substation (SQ, NEP 2035, eGo 100)
+    'ego_dp_powerflow_timeseries_generator.py',     # Transfer renpassG!S results into the corresponding powerflow table
+    'ego_dp_powerflow_timeseries_generator_neighbours.py',      # Tranfer renpassG!S results for neighbours into the corresponding powerflow tables
 	'ego_dp_powerflow_timeseries_generator.sql',		# Transfer renpassG!S results into the corresponding powerflow table
 	'ego_dp_powerflow_griddistrict_demand.py',		# Demand per MV Griddistrict
 	'ego_dp_powerflow_timeseries_demand.sql',		# Insert demand series into corresponding powerflow table (SQ, NEP2035, eGo100)

dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator.py

@@ -0,0 +1,121 @@
+"""
+Assign German timeseries data from hidden renpassG!S schema to high voltage
+powerflow.
+"""
+
+__copyright__ 	= "ZNES Flensburg"
+__license__ 	= "GNU Affero General Public License Version 3 (AGPL-3.0)"
+__url__ 		= "https://github.com/openego/data_processing/blob/master/LICENSE"
+__author__ 		= "wolfbunke"
+
+import pandas as pd
+import numpy as np
+
+from dataprocessing.tools.io import oedb_session
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import MetaData, func
+from sqlalchemy.ext.automap import automap_base
+
+from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator,\
+    EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus
+from ego_dp_powerflow_timeseries_generator_helper import SOURCE_TO_FUEL, SCENARIOMAP, \
+    TEMPID
+
+# get database connection
+conn = oedb_session(section='test')
+
+Session = sessionmaker(bind=conn)
+session = Session()
+
+meta = MetaData()
+meta.bind = conn
+meta.reflect(bind=conn, schema='calc_renpass_gis',
+             only=['renpass_gis_results'])
+
+# map to classes
+Base = automap_base(metadata=meta)
+Base.prepare()
+
+Results = Base.classes.renpass_gis_results
+
+###############################################################################
+
+
+def _norm(x):
+    return x / x.sum()
+
+# delete all from model_draft.ego_grid_pf_hv_generator_pq_set
+session.query(PqSet).delete()
+session.commit()
+
+for scn_name, scn_nr in SCENARIOMAP.items():
+
+    # dataframe from model_draft.pf_generator_single
+    # with aggr_id, source, p_nom
+    filters = (Generator.scn_name == scn_name, Generator.aggr_id != None)
+    fields = [Generator.aggr_id, Generator.source,
+              func.sum(Generator.p_nom).label('summed_p_nom')]
+    grouper = Generator.aggr_id, Generator.source
+    query = session.query(*fields).filter(*filters).group_by(*grouper)
+
+    generators = pd.read_sql(query.statement, query.session.bind)
+
+    # create fraction of nominal power to total nominal power for each source
+    generators['fraction_of_total_p_nom'] = \
+        generators.groupby('source')['summed_p_nom'].apply(_norm)
+
+    # dataframe from calc_renpass_gis.renpass_gis_results
+    # optimization results to buses
+    # with obj_label, datetime, val
+    filters = (Results.obj_label.like('%DE%'),
+               ~Results.obj_label.like('%powerline%'),
+               Results.type == 'to_bus',
+               Results.scenario_id == scn_nr)
+    fields = Results.obj_label, Results.datetime, Results.val
+    query = session.query(*fields).filter(*filters)
+
+    results = pd.read_sql(query.statement, query.session.bind)
+
+    # map obj_label to corresponding source
+    results['source'] = None
+    for k, v in SOURCE_TO_FUEL.items():
+        idx = results['obj_label'].str.contains(v)
+        results.loc[idx, 'source'] = k
+
+    # aggregate by source and datetime
+    results = results.groupby(['source', 'datetime'], as_index=False).sum()
+
+    # power generation timeseries for each source in list format
+    results_s = results.groupby('source')['val'].apply(np.array)
+
+    # map corresponding timeseries with each generator multiplied by fraction
+    # of nominal power
+    generators['p_set'] = generators['source'].map(results_s) * \
+        generators['fraction_of_total_p_nom']
+
+    # generators without p_set
+    ix = generators['p_set'].isnull()
+    print('Generators with sources {} have no p_sets assigned!'.format(
+        generators[ix]['source'].unique()))
+    generators.loc[ix, 'p_set'] = None
+
+    # add columns
+    empty = ['q_set', 'p_min_pu', 'p_max_pu']
+
+    pqsets = pd.concat(
+        [generators[['aggr_id', 'p_set']],
+         pd.DataFrame(columns=empty)])
+
+    pqsets.loc[:, empty] = None
+
+    # add scenario name and temporal id
+    pqsets['scn_name'] = scn_name
+    pqsets['temp_id'] = TEMPID
+
+    # rename column aggr_id to generator_id
+    pqsets.rename(columns={'aggr_id': 'generator_id'}, inplace=True)
+
+    # write to db
+    for i in pqsets.to_dict(orient='records'):
+        session.add(PqSet(**i))
+    session.commit()

dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_helper.py

@@ -0,0 +1,70 @@
+""" Helper functions and variables to handle renpass_gis tables.
+"""
+
+__copyright__ 	= "ZNES Flensburg"
+__license__ 	= "GNU Affero General Public License Version 3 (AGPL-3.0)"
+__url__ 		= "https://github.com/openego/data_processing/blob/master/LICENSE"
+__author__ 		= "wolfbunke"
+
+import pandas as pd
+
+from dataprocessing.tools.io import oedb_session
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import MetaData
+from sqlalchemy.ext.automap import automap_base
+
+NEIGHBOURSID = 200000
+
+SCENARIOMAP = {'Status Quo': 43, 'NEP 2035': 41, 'eGo 100': 40}
+
+SOURCE_TO_FUEL = {
+    1: 'gas', 2: 'lignite', 3: 'mixed_fuels', 4: 'oil',
+    5: 'uranium', 6: 'biomass', 8: 'hard_coal', 9: 'run_of_river', 10: 'reservoir',
+    12: 'solar', 13: 'wind_onshore', 14: 'geothermal', 15: 'other_non_renewable',
+    16: 'wind_offshore', 94: 'storage', 95: 'load', 96: 'waste',
+    97: 'reservoir', 98: 'shortage', 99: 'excess'}
+
+TEMPID = 1
+
+def renpass_gis_orm_classes(session):
+    """
+    Parameters
+    ----------
+    session : sqlalchemy.orm.session.Session
+        Handling all conversations with the database
+
+    Notes
+    -----
+    Relations in schema calc_renpass_gis are still not part of egoio. If this is
+    changed in the future this function becomes obsolete.
+    """
+
+    meta = MetaData()
+    meta.reflect(bind=session.bind, schema='calc_renpass_gis',
+                only=['renpass_gis_linear_transformer',
+                      'renpass_gis_source',
+                      'renpass_gis_results'])
+
+    # map to classes
+    Base = automap_base(metadata=meta)
+    Base.prepare()
+
+    Transformer, Source, Results = Base.classes.renpass_gis_linear_transformer, \
+        Base.classes.renpass_gis_source, Base.classes.renpass_gis_results
+
+    return Transformer, Source, Results
+
+def _flatten(x):
+    if isinstance(x, list):
+        return x[0] if len(x) == 1 else x
+    else:
+        return x
+
+
+def map_on_partial_string(series, mapping):
+    """ Map mapping values to string series.  """
+    s = pd.Series(index=series.index)
+    for k, v in mapping.items():
+        ix = series.str.contains(v)
+        s[ix] = k
+    return s

dataprocessing/python_scripts/ego_dp_powerflow_timeseries_generator_neighbours.py

@@ -0,0 +1,147 @@
+"""
+Assign German timeseries data from hidden renpassG!S schema to high voltage
+powerflow.
+"""
+
+__copyright__ 	= "ZNES Flensburg"
+__license__ 	= "GNU Affero General Public License Version 3 (AGPL-3.0)"
+__url__ 		= "https://github.com/openego/data_processing/blob/master/LICENSE"
+__author__ 		= "wolfbunke"
+
+import pandas as pd
+import numpy as np
+
+from dataprocessing.tools.io import oedb_session
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import MetaData, func
+from ego_dp_powerflow_timeseries_generator_helper import SOURCE_TO_FUEL, SCENARIOMAP, \
+    TEMPID, NEIGHBOURSID, _flatten, map_on_partial_string, renpass_gis_orm_classes
+from egoio.db_tables.model_draft import EgoSupplyPfGeneratorSingle as Generator, \
+    EgoGridPfHvGeneratorPqSet as PqSet, EgoGridHvElectricalNeighboursBus as Neighbour
+
+conn = oedb_session(section='test')
+Session = sessionmaker(bind=conn)
+session = Session()
+
+# obligatory delete statement based on NEIGHBOURSID
+session.query(Generator).filter(Generator.generator_id >= NEIGHBOURSID).\
+    delete(synchronize_session='fetch')
+
+session.query(PqSet).filter(PqSet.generator_id >= NEIGHBOURSID).\
+    delete(synchronize_session='fetch')
+
+
+###############################################################################
+
+Transformer, Source, Results = renpass_gis_orm_classes(session)
+
+# get DataFrame each row representing one electrical neighbour by applying
+# filter on id and v_nom, not affected by scenario name
+query = session.query(Neighbour)
+neighbours = pd.read_sql(query.statement, query.session.bind)
+
+ix = (neighbours['id'] <= 27) & (neighbours['v_nom'] == 380)
+neighbours = neighbours.loc[ix, :]
+neighbours.set_index('cntr_id', inplace=True)
+
+
+for scn_name, scn_nr in SCENARIOMAP.items():
+
+    # get renpass_gis scenario data on linear transformers. Parameters are
+    # defined on those edges directed from the component to the bus.
+    filters = [Transformer.scenario_id == scn_nr,
+               ~Transformer.source.like('%powerline%'),
+               Transformer.label == Transformer.source]  # direction
+
+    query = session.query(Transformer).filter(*filters)
+    transformers = pd.read_sql(query.statement, query.session.bind)
+    transformers['type'] = 'linear transformer'
+
+    # get data on sources
+    filters = [Source.scenario_id == scn_nr, ~Source.label.like('GL%')]
+    query = session.query(Source).filter(*filters)
+    sources = pd.read_sql(query.statement, query.session.bind)
+    sources['type'] = 'source'
+
+    # sources and transformers, distinct in renpass_gis, are both seen as
+    # generators and can be handled together
+    generators = pd.concat([sources, transformers], ignore_index=True)
+
+    # parameters in renpass_gis are not necessarily scalars and stored in lists
+    # lists of len one are flattened
+    generators = generators.applymap(_flatten)
+
+    # 0 does not equal zero. In case a class with zero nominal value
+    # should be defined for the purpose of scenario definition very small values
+    # are used in the scenario files.
+    ix = generators['nominal_value'] < 1e-7
+    generators = generators.loc[~ix, :]
+
+    # source in the context of eGo has a different meaning. The column has to
+    # be renamed
+    generators.rename(columns={'source': 'renpass_gis_source'}, inplace=True)
+
+    # map from obj label string -> source
+    generators['source'] = map_on_partial_string(
+        generators['label'], SOURCE_TO_FUEL)
+
+    generators['cntr_id'] = generators['label'].str[:2]
+
+    # exclude Germany
+    generators = generators.loc[generators['cntr_id'] != 'DE', :]
+
+    # assign bus_ids according to neighbours DataFrame
+    generators['bus'] = generators['cntr_id'].map(neighbours['bus_id'])
+
+    # set control, and dispatch parameter
+    generators['control'] = 'PV'
+    generators['dispatch'] = generators['type'].map(
+        {'linear transformer': 'flexible', 'source': 'variable'})
+
+    # get corresponding optimization results from renpass_gis
+    # obj_label, datetime, val
+    filters = (~Results.obj_label.like('%DE%'),
+               ~Results.obj_label.like('%GL%'),
+               ~Results.obj_label.like('%powerline%'),
+               Results.type == 'to_bus',
+               Results.scenario_id == scn_nr)
+    fields = Results.obj_label, Results.datetime, Results.val
+    query = session.query(*fields).filter(*filters)
+    results = pd.read_sql(query.statement, query.session.bind)
+
+    # map from obj label string -> source
+    results['source'] = map_on_partial_string(
+        results['obj_label'], SOURCE_TO_FUEL).astype(int)
+
+    results['cntr_id'] = results['obj_label'].str[:2]
+    results['bus'] = results['cntr_id'].map(neighbours['bus_id'])
+
+    # create Series with bus_id, source and power generation / actual value
+    # in list format
+    results_s = results.groupby(['bus', 'source'])['val'].apply(np.array)
+    results_s.name = 'p_set'
+
+    # check all arrays are of the same length
+    assert all(len(i) == len(results_s[0]) for i in results_s)
+
+    # include timeseries data in generators DataFrame
+    generators = generators.join(results_s, on=['bus', 'source'])
+
+    # set scenario name, temporal id
+    generators['scn_name'] = scn_name
+    generators['temp_id'] = TEMPID
+
+    generators['generator_id'] = generators.index + NEIGHBOURSID
+
+    # prepare DataFrames to be exported
+    generator_ex = generators[['scn_name', 'generator_id', 'bus', 'dispatch', 'control']]
+    pqsets = generators[['scn_name', 'generator_id', 'temp_id', 'p_set']]
+
+    # write to db
+    for i in generator_ex.to_dict(orient='records'):
+        session.add(Generator(**i))
+
+    for i in pqsets.to_dict(orient='records'):
+        session.add(PqSet(**i))
+
+    session.commit()