Database Imports Not Working & SSH Tunnel Limitations

.github/workflows/tests-coverage.yml

@@ -66,12 +66,16 @@ jobs:
         run: |
           python -m pip install pytest pytest-notebook
           python -m pytest --runslow --runonlinux --disable-warnings --color=yes -v
+        env:
+          TOEP_TOKEN_KH: ${{ secrets.TOEP_TOKEN_KH }}
 
       - name: Run tests Windows
         if: runner.os == 'Windows'
         run: |
           python -m pip install pytest pytest-notebook
           python -m pytest --runslow --disable-warnings --color=yes -v
+        env:
+          TOEP_TOKEN_KH: ${{ secrets.TOEP_TOKEN_KH }}
 
       - name: Run tests, coverage and send to coveralls
         if: runner.os == 'Linux' && matrix.python-version == 3.9 && matrix.name-suffix == 'coverage'
@@ -80,5 +84,6 @@ jobs:
           coverage run --source=edisgo -m pytest --runslow --runonlinux --disable-warnings --color=yes -v
           coveralls
         env:
+          TOEP_TOKEN_KH: ${{ secrets.TOEP_TOKEN_KH }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_SERVICE_NAME: github

.gitignore

@@ -25,3 +25,5 @@ eDisGo.egg-info/
 /edisgo/opf/opf_solutions/*.json
 /edisgo/opf/eDisGo_OPF.jl/.vscode
 .vscode/settings.json
+
+*TOEP_TOKEN.*

doc/whatsnew/v0-3-0.rst

@@ -28,4 +28,6 @@ Changes
 * Move function to assign feeder to Topology class and add methods to the Grid class to get information on the feeders `#360 <https://github.com/openego/eDisGo/pull/360>`_
 * Added a storage operation strategy where the storage is charged when PV feed-in is higher than electricity demand of the household and discharged when electricity demand exceeds PV generation `#386 <https://github.com/openego/eDisGo/pull/386>`_
 * Added an estimation of the voltage deviation over a cable when selecting a suitable cable to connect a new component `#411 <https://github.com/openego/eDisGo/pull/411>`_
-* Added clipping of heat pump electrical power at its maximum value #428 <https://github.com/openego/eDisGo/pull/428>
+* Added clipping of heat pump electrical power at its maximum value `#428 <https://github.com/openego/eDisGo/pull/428>`_
+* Made OEP database call optional in get_database_alias_dictionaries, allowing setup without OEP when using an alternative eGon-data database. `#451 <https://github.com/openego/eDisGo/pull/451>`_
+* Fixed database import issues by addressing table naming assumptions and added support for external SSH tunneling in eGon-data configurations. `#451 <https://github.com/openego/eDisGo/pull/451>`_

edisgo/edisgo.py

@@ -71,6 +71,11 @@ class EDisGo:
     ----------
     ding0_grid : :obj:`str`
         Path to directory containing csv files of network to be loaded.
+    engine : :sqlalchemy:`sqlalchemy.Engine<sqlalchemy.engine.Engine>` or None
+        Database engine for connecting to the `OpenEnergy DataBase OEDB
+        <https://openenergyplatform.org/dataedit/schemas>`_ or other eGon-data
+        databases. Defaults to the OEDB engine. Can be set to None if no scenario is to
+        be loaded.
     generator_scenario : None or :obj:`str`, optional
         If None, the generator park of the imported grid is kept as is.
         Otherwise defines which scenario of future generator park to use
@@ -158,8 +163,10 @@ class EDisGo:
     """
 
     def __init__(self, **kwargs):
+        # Set database engine for future scenarios
+        self.engine: Engine | None = kwargs.pop("engine", toep_engine())
         # load configuration
-        self._config = Config(**kwargs)
+        self._config = Config(engine=self.engine, **kwargs)
 
         # instantiate topology object and load grid data
         self.topology = Topology(config=self.config)
@@ -418,12 +425,9 @@ def set_time_series_active_power_predefined(
                 Technology- and weather cell-specific hourly feed-in time series are
                 obtained from the
                 `OpenEnergy DataBase
-                <https://openenergyplatform.org/dataedit/schemas>`_. See
-                :func:`edisgo.io.timeseries_import.feedin_oedb` for more information.
-
-                This option requires that the parameter `engine` is provided in case
-                new ding0 grids with geo-referenced LV grids are used. For further
-                settings, the parameter `timeindex` can also be provided.
+                <https://openenergyplatform.org/dataedit/schemas>`_ or other eGon-data
+                databases. See :func:`edisgo.io.timeseries_import.feedin_oedb` for more
+                information.
 
             * :pandas:`pandas.DataFrame<DataFrame>`
 
@@ -536,9 +540,6 @@ def set_time_series_active_power_predefined(
 
         Other Parameters
         ------------------
-        engine : :sqlalchemy:`sqlalchemy.Engine<sqlalchemy.engine.Engine>`
-            Database engine. This parameter is only required in case
-            `conventional_loads_ts` or `fluctuating_generators_ts` is 'oedb'.
         scenario : str
             Scenario for which to retrieve demand data. Possible options are 'eGon2035'
             and 'eGon100RE'. This parameter is only required in case
@@ -569,7 +570,7 @@ def set_time_series_active_power_predefined(
                 self,
                 fluctuating_generators_ts,
                 fluctuating_generators_names,
-                engine=kwargs.get("engine", toep_engine()),
+                engine=self.engine,
                 timeindex=kwargs.get("timeindex", None),
             )
         if dispatchable_generators_ts is not None:
@@ -584,7 +585,7 @@ def set_time_series_active_power_predefined(
                 loads_ts_df = timeseries_import.electricity_demand_oedb(
                     edisgo_obj=self,
                     scenario=kwargs.get("scenario"),
-                    engine=kwargs.get("engine", toep_engine()),
+                    engine=self.engine,
                     timeindex=kwargs.get("timeindex", None),
                     load_names=conventional_loads_names,
                 )
@@ -971,9 +972,7 @@ def import_generators(self, generator_scenario=None, **kwargs):
         Other Parameters
         ----------------
         kwargs :
-            In case you are using new ding0 grids, where the LV is geo-referenced, a
-            database engine needs to be provided through keyword argument `engine`.
-            In case you are using old ding0 grids, where the LV is not geo-referenced,
+            If you are using old ding0 grids, where the LV is not geo-referenced,
             you can check :func:`edisgo.io.generators_import.oedb_legacy` for possible
             keyword arguments.
 
@@ -985,7 +984,7 @@ def import_generators(self, generator_scenario=None, **kwargs):
         else:
             generators_import.oedb(
                 edisgo_object=self,
-                engine=kwargs.get("engine", toep_engine()),
+                engine=self.engine,
                 scenario=generator_scenario,
             )
 
@@ -1350,7 +1349,7 @@ def reinforce(
 
         """
         if copy_grid:
-            edisgo_obj = copy.deepcopy(self)
+            edisgo_obj = self.copy()
         else:
             edisgo_obj = self
 
@@ -1846,9 +1845,11 @@ def _aggregate_time_series(attribute, groups, naming):
                 [
                     pd.DataFrame(
                         {
-                            naming.format("_".join(k))
-                            if isinstance(k, tuple)
-                            else naming.format(k): getattr(self.timeseries, attribute)
+                            (
+                                naming.format("_".join(k))
+                                if isinstance(k, tuple)
+                                else naming.format(k)
+                            ): getattr(self.timeseries, attribute)
                             .loc[:, v]
                             .sum(axis=1)
                         }
@@ -1917,9 +1918,8 @@ def _aggregate_time_series(attribute, groups, naming):
 
     def import_electromobility(
         self,
-        data_source: str,
+        data_source: str = "oedb",
         scenario: str = None,
-        engine: Engine = None,
         charging_processes_dir: PurePath | str = None,
         potential_charging_points_dir: PurePath | str = None,
         import_electromobility_data_kwds=None,
@@ -1961,10 +1961,8 @@ def import_electromobility(
             * "oedb"
 
                 Electromobility data is obtained from the `OpenEnergy DataBase
-                <https://openenergyplatform.org/dataedit/schemas>`_.
-
-                This option requires that the parameters `scenario` and `engine` are
-                provided.
+                <https://openenergyplatform.org/dataedit/schemas>`_ or other eGon-data
+                databases depending on the provided Engine.
 
             * "directory"
 
@@ -1974,9 +1972,6 @@ def import_electromobility(
         scenario : str
             Scenario for which to retrieve electromobility data in case `data_source` is
             set to "oedb". Possible options are "eGon2035" and "eGon100RE".
-        engine : :sqlalchemy:`sqlalchemy.Engine<sqlalchemy.engine.Engine>`
-            Database engine. Needs to be provided in case `data_source` is set to
-            "oedb".
         charging_processes_dir : str or pathlib.PurePath
             Directory holding data on charging processes (standing times, charging
             demand, etc. per vehicle), including metadata, from SimBEV.
@@ -2038,7 +2033,7 @@ def import_electromobility(
             import_electromobility_from_oedb(
                 self,
                 scenario=scenario,
-                engine=engine,
+                engine=self.engine,
                 **import_electromobility_data_kwds,
             )
         elif data_source == "directory":
@@ -2131,10 +2126,11 @@ def apply_charging_strategy(self, strategy="dumb", **kwargs):
         """
         charging_strategy(self, strategy=strategy, **kwargs)
 
-    def import_heat_pumps(self, scenario, engine, timeindex=None, import_types=None):
+    def import_heat_pumps(self, scenario, timeindex=None, import_types=None):
         """
-        Gets heat pump data for specified scenario from oedb and integrates the heat
-        pumps into the grid.
+        Gets heat pump data for specified scenario from the OEDB or other eGon-data
+        databases depending on the provided Engine and integrates the heat pumps into
+        the grid.
 
         Besides heat pump capacity the heat pump's COP and heat demand to be served
         are as well retrieved.
@@ -2189,8 +2185,6 @@ def import_heat_pumps(self, scenario, engine, timeindex=None, import_types=None)
         scenario : str
             Scenario for which to retrieve heat pump data. Possible options
             are 'eGon2035' and 'eGon100RE'.
-        engine : :sqlalchemy:`sqlalchemy.Engine<sqlalchemy.engine.Engine>`
-            Database engine.
         timeindex : :pandas:`pandas.DatetimeIndex<DatetimeIndex>` or None
             Specifies time steps for which to set COP and heat demand data. Leap years
             can currently not be handled. In case the given
@@ -2231,31 +2225,31 @@ def import_heat_pumps(self, scenario, engine, timeindex=None, import_types=None)
             year = tools.get_year_based_on_scenario(scenario)
             return self.import_heat_pumps(
                 scenario,
-                engine,
+                self.engine,
                 timeindex=pd.date_range(f"1/1/{year}", periods=8760, freq="H"),
                 import_types=import_types,
             )
 
         integrated_heat_pumps = import_heat_pumps_oedb(
             edisgo_object=self,
             scenario=scenario,
-            engine=engine,
+            engine=self.engine,
             import_types=import_types,
         )
         if len(integrated_heat_pumps) > 0:
             self.heat_pump.set_heat_demand(
                 self,
                 "oedb",
                 heat_pump_names=integrated_heat_pumps,
-                engine=engine,
+                engine=self.engine,
                 scenario=scenario,
                 timeindex=timeindex,
             )
             self.heat_pump.set_cop(
                 self,
                 "oedb",
                 heat_pump_names=integrated_heat_pumps,
-                engine=engine,
+                engine=self.engine,
                 timeindex=timeindex,
             )
 
@@ -2303,7 +2297,7 @@ def apply_heat_pump_operating_strategy(
         """
         hp_operating_strategy(self, strategy=strategy, heat_pump_names=heat_pump_names)
 
-    def import_dsm(self, scenario: str, engine: Engine, timeindex=None):
+    def import_dsm(self, scenario: str, timeindex=None):
         """
         Gets industrial and CTS DSM profiles from the
         `OpenEnergy DataBase <https://openenergyplatform.org/dataedit/schemas>`_.
@@ -2322,8 +2316,6 @@ def import_dsm(self, scenario: str, engine: Engine, timeindex=None):
         scenario : str
             Scenario for which to retrieve DSM data. Possible options
             are 'eGon2035' and 'eGon100RE'.
-        engine : :sqlalchemy:`sqlalchemy.Engine<sqlalchemy.engine.Engine>`
-            Database engine.
         timeindex : :pandas:`pandas.DatetimeIndex<DatetimeIndex>` or None
             Specifies time steps for which to get data. Leap years can currently not be
             handled. In case the given timeindex contains a leap year, the data will be
@@ -2336,7 +2328,7 @@ def import_dsm(self, scenario: str, engine: Engine, timeindex=None):
 
         """
         dsm_profiles = dsm_import.oedb(
-            edisgo_obj=self, scenario=scenario, engine=engine, timeindex=timeindex
+            edisgo_obj=self, scenario=scenario, engine=self.engine, timeindex=timeindex
         )
         self.dsm.p_min = dsm_profiles["p_min"]
         self.dsm.p_max = dsm_profiles["p_max"]
@@ -2346,7 +2338,6 @@ def import_dsm(self, scenario: str, engine: Engine, timeindex=None):
     def import_home_batteries(
         self,
         scenario: str,
-        engine: Engine,
     ):
         """
         Gets home battery data for specified scenario and integrates the batteries into
@@ -2357,7 +2348,8 @@ def import_home_batteries(
         between two scenarios: 'eGon2035' and 'eGon100RE'.
 
         The data is retrieved from the
-        `open energy platform <https://openenergyplatform.org/>`_.
+        `open energy platform <https://openenergyplatform.org/>`_ or other eGon-data
+        databases depending on the given Engine.
 
         The batteries are integrated into the grid (added to
         :attr:`~.network.topology.Topology.storage_units_df`) based on their building
@@ -2374,14 +2366,12 @@ def import_home_batteries(
         scenario : str
             Scenario for which to retrieve home battery data. Possible options
             are 'eGon2035' and 'eGon100RE'.
-        engine : :sqlalchemy:`sqlalchemy.Engine<sqlalchemy.engine.Engine>`
-            Database engine.
 
         """
         home_batteries_oedb(
             edisgo_obj=self,
             scenario=scenario,
-            engine=engine,
+            engine=self.engine,
         )
 
     def plot_mv_grid_topology(self, technologies=False, **kwargs):
@@ -3131,7 +3121,7 @@ def spatial_complexity_reduction(
 
         """
         if copy_edisgo is True:
-            edisgo_obj = copy.deepcopy(self)
+            edisgo_obj = self.copy()
         else:
             edisgo_obj = self
         busmap_df, linemap_df = spatial_complexity_reduction(
@@ -3345,6 +3335,44 @@ def resample_timeseries(
         self.heat_pump.resample_timeseries(method=method, freq=freq)
         self.overlying_grid.resample(method=method, freq=freq)
 
+    def copy(self, deep=True):
+        """
+        Returns a copy of the object, with an option for a deep copy.
+
+        The SQLAlchemy engine is excluded from the copying process and restored
+        afterward.
+
+        Parameters
+        ----------
+        deep : bool
+            If True, performs a deep copy; otherwise, performs a shallow copy.
+
+        Returns
+        ---------
+        :class:`~.EDisGo`
+            Copied EDisGo object.
+
+        """
+        tmp_engine = (
+            getattr(self, "engine", None)
+            if isinstance(getattr(self, "engine", None), Engine)
+            else None
+        )
+
+        if tmp_engine:
+            logging.info("Temporarily removing the SQLAlchemy engine before copying.")
+            self.engine = self.config._engine = None
+
+        cpy = copy.deepcopy(self) if deep else copy.copy(self)
+
+        if tmp_engine:
+            logging.info("Restoring the SQLAlchemy engine after copying.")
+            self.engine = self.config._engine = cpy.engine = cpy.config._engine = (
+                tmp_engine
+            )
+
+        return cpy
+
 
 def import_edisgo_from_pickle(filename, path=""):
     """