Merge branch 'main' into feature/sim-bench

Signed-off-by: Bram Stoeller <[email protected]>

Author: bramstoeller

src/power_grid_model_io/converters/pandapower_converter.py

@@ -29,14 +29,15 @@ class PandaPowerConverter(BaseConverter[PandaPowerData]):
 
     __slots__ = ("pp_input_data", "pgm_input_data", "idx", "idx_lookup", "next_idx", "system_frequency")
 
-    def __init__(self, system_frequency: float = 50.0):
+    def __init__(self, system_frequency: float = 50.0, trafo_loading: str = "current"):
         """
         Prepare some member variables
 
         Args:
             system_frequency: fundamental frequency of the alternating current and voltage in the Network measured in Hz
         """
         super().__init__(source=None, destination=None)
+        self.trafo_loading = trafo_loading
         self.system_frequency: float = system_frequency
         self.pp_input_data: PandaPowerData = {}
         self.pgm_input_data: SingleDataset = {}
@@ -1060,16 +1061,29 @@ def _pp_trafos_output(self):
         # TODO: create unit tests for the function
         assert "res_trafo" not in self.pp_output_data
 
-        if "transformer" not in self.pgm_output_data or self.pgm_output_data["transformer"].size == 0:
+        if ("transformer" not in self.pgm_output_data or self.pgm_output_data["transformer"].size == 0) or (
+            "trafo" not in self.pp_input_data or len(self.pp_input_data["trafo"]) == 0
+        ):
             return
 
         pgm_input_transformers = self.pgm_input_data["transformer"]
-
+        pp_input_transformers = self.pp_input_data["trafo"]
         pgm_output_transformers = self.pgm_output_data["transformer"]
 
         from_nodes = self.pgm_nodes_lookup.loc[pgm_input_transformers["from_node"]]
         to_nodes = self.pgm_nodes_lookup.loc[pgm_input_transformers["to_node"]]
 
+        # Only derating factor used here. Sn is already being multiplied by parallel
+        loading_multiplier = pp_input_transformers["df"]
+        if self.trafo_loading == "current":
+            ui_from = pgm_output_transformers["i_from"] * pgm_input_transformers["u1"]
+            ui_to = pgm_output_transformers["i_to"] * pgm_input_transformers["u2"]
+            loading = np.maximum(ui_from, ui_to) / pgm_input_transformers["sn"] * loading_multiplier * 1e2
+        elif self.trafo_loading == "power":
+            loading = pgm_output_transformers["loading"] * loading_multiplier * 1e2
+        else:
+            raise ValueError(f"Invalid transformer loading type: {str(self.trafo_loading)}")
+
         pp_output_trafos = pd.DataFrame(
             columns=[
                 "p_hv_mw",
@@ -1100,7 +1114,7 @@ def _pp_trafos_output(self):
         pp_output_trafos["vm_lv_pu"] = to_nodes["u_pu"].values
         pp_output_trafos["va_hv_degree"] = from_nodes["u_degree"].values
         pp_output_trafos["va_lv_degree"] = to_nodes["u_degree"].values
-        pp_output_trafos["loading_percent"] = pgm_output_transformers["loading"] * 1e2
+        pp_output_trafos["loading_percent"] = loading
 
         self.pp_output_data["res_trafo"] = pp_output_trafos
 

src/power_grid_model_io/data_stores/csv_dir_store.py

@@ -1,17 +1,17 @@
-# SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model project <[email protected]>
+# SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model IO project <[email protected]>
 #
 # SPDX-License-Identifier: MPL-2.0
 """
 CSV Directory Store
 """
 
 from pathlib import Path
-from typing import Any, Callable, Dict, List
+from typing import Any, Dict, List
 
 import pandas as pd
 
 from power_grid_model_io.data_stores.base_data_store import BaseDataStore
-from power_grid_model_io.data_types import TabularData
+from power_grid_model_io.data_types import LazyDataFrame, TabularData
 
 
 class CsvDirStore(BaseDataStore[TabularData]):
@@ -35,13 +35,13 @@ def load(self) -> TabularData:
         Create a lazy loader for all CSV files in a directory and store them in a TabularData instance.
         """
 
-        def lazy_csv_loader(csv_path: Path) -> Callable[[], pd.DataFrame]:
+        def lazy_csv_loader(csv_path: Path) -> LazyDataFrame:
             def csv_loader():
                 return pd.read_csv(filepath_or_buffer=csv_path, header=self._header_rows, **self._csv_kwargs)
 
             return csv_loader
 
-        data: Dict[str, Callable[[], pd.DataFrame]] = {}
+        data: Dict[str, LazyDataFrame] = {}
         for path in self._dir_path.glob("*.csv"):
             data[path.stem] = lazy_csv_loader(path)
 

src/power_grid_model_io/data_stores/excel_file_store.py

@@ -12,7 +12,7 @@
 import pandas as pd
 
 from power_grid_model_io.data_stores.base_data_store import BaseDataStore
-from power_grid_model_io.data_types import TabularData
+from power_grid_model_io.data_types import LazyDataFrame, TabularData
 
 
 class ExcelFileStore(BaseDataStore[TabularData]):
@@ -24,7 +24,7 @@ class ExcelFileStore(BaseDataStore[TabularData]):
     same values) or renamed.
     """
 
-    __slots__ = ("_file_paths", "_header_rows")
+    __slots__ = ("_file_paths", "_excel_files", "_header_rows")
 
     _unnamed_pattern: re.Pattern = re.compile(r"Unnamed: \d+_level_\d+")
 
@@ -62,19 +62,26 @@ def load(self) -> TabularData:
         have no prefix, while the tables of all the extra files will be prefixed with the name of the key word argument
         as supplied in the constructor.
         """
-        data: Dict[str, pd.DataFrame] = {}
-        for name, path in self._file_paths.items():
-            with path.open(mode="rb") as file_pointer:
-                spreadsheet = pd.read_excel(io=file_pointer, sheet_name=None, header=self._header_rows)
-            for sheet_name, sheet_data in spreadsheet.items():
+
+        def lazy_sheet_loader(xls_file: pd.ExcelFile, xls_sheet_name: str):
+            def sheet_loader():
+                sheet_data = xls_file.parse(xls_sheet_name, header=self._header_rows)
                 sheet_data = self._remove_unnamed_column_placeholders(data=sheet_data)
-                sheet_data = self._handle_duplicate_columns(data=sheet_data, sheet_name=sheet_name)
-                if name:
+                sheet_data = self._handle_duplicate_columns(data=sheet_data, sheet_name=xls_sheet_name)
+                return sheet_data
+
+            return sheet_loader
+
+        data: Dict[str, LazyDataFrame] = {}
+        for name, path in self._file_paths.items():
+            excel_file = pd.ExcelFile(path)
+            for sheet_name in excel_file.sheet_names:
+                loader = lazy_sheet_loader(excel_file, sheet_name)
+                if name != "":  # If the Excel file is not the main file, prefix the sheet name with the file name
                     sheet_name = f"{name}.{sheet_name}"
                 if sheet_name in data:
                     raise ValueError(f"Duplicate sheet name '{sheet_name}'")
-                data[sheet_name] = sheet_data
-
+                data[sheet_name] = loader
         return TabularData(**data)
 
     def save(self, data: TabularData) -> None:

src/power_grid_model_io/data_types/__init__.py

@@ -6,4 +6,4 @@
 """
 
 from power_grid_model_io.data_types._data_types import ExtraInfo, ExtraInfoLookup, StructuredData
-from power_grid_model_io.data_types.tabular_data import TabularData
+from power_grid_model_io.data_types.tabular_data import LazyDataFrame, TabularData

src/power_grid_model_io/data_types/tabular_data.py

@@ -24,7 +24,7 @@ class TabularData:
     which supports unit conversions and value substitutions
     """
 
-    def __init__(self, **tables: Union[pd.DataFrame, LazyDataFrame, np.ndarray]):
+    def __init__(self, **tables: Union[pd.DataFrame, np.ndarray, LazyDataFrame]):
         """
         Tabular data can either be a collection of pandas DataFrames and/or numpy structured arrays.
         The key word arguments will define the keys of the data.
@@ -41,7 +41,7 @@ def __init__(self, **tables: Union[pd.DataFrame, LazyDataFrame, np.ndarray]):
                     f"Invalid data type for table '{table_name}'; "
                     f"expected a pandas DataFrame or NumPy array, got {type(table_data).__name__}."
                 )
-        self._data: Dict[str, Union[pd.DataFrame, LazyDataFrame, np.ndarray]] = tables
+        self._data: Dict[str, Union[pd.DataFrame, np.ndarray, LazyDataFrame]] = tables
         self._units: Optional[UnitMapping] = None
         self._substitution: Optional[ValueMapping] = None
         self._log = structlog.get_logger(type(self).__name__)
@@ -156,6 +156,14 @@ def _apply_unit_conversion(self, table_data: pd.DataFrame, table: str, field: st
 
         return table_data[pd.MultiIndex.from_tuples([(field, si_unit)])[0]]
 
+    def __len__(self) -> int:
+        """
+        Return the number of tables (regardless of if they are already loaded or not)
+
+        Returns: The number of tables
+        """
+        return len(self._data)
+
     def __contains__(self, table_name: str) -> bool:
         """
         Mimic the dictionary 'in' operator
@@ -195,7 +203,10 @@ def items(self) -> Generator[Tuple[str, Union[pd.DataFrame, np.ndarray]], None,
         """
         Mimic the dictionary .items() function
 
-        Returns: An iterator over the table names and the raw table data
+        Returns: A generator of the table names and the raw table data
         """
+
+        # Note: PyCharm complains about the type, but it is correct, as an ItemsView extends from
+        # AbstractSet[Tuple[_KT_co, _VT_co]], which actually is compatible with Iterable[_KT_co, _VT_co]
         for key in self._data:
             yield key, self[key]

tests/unit/converters/test_pandapower_converter_output.py

@@ -11,6 +11,7 @@
 from power_grid_model import initialize_array
 
 from power_grid_model_io.converters.pandapower_converter import PandaPowerConverter
+from tests.utils import MockDf
 
 
 @pytest.fixture
@@ -286,9 +287,11 @@ def test_output_sgen(converter):
         converter.pp_output_data.__setitem__.assert_called_once_with("res_sgen", mock_pp_df.return_value)
 
 
-def test_output_trafos(converter):
+def test_output_trafos__current(converter):
     # Arrange
     mock_pgm_array = MagicMock()
+    converter.trafo_loading = "current"
+    converter.pp_input_data["trafo"] = MockDf(2)
     converter.pgm_input_data["transformer"] = mock_pgm_array
     converter.pgm_output_data["transformer"] = mock_pgm_array
     converter.pgm_nodes_lookup = pd.DataFrame(
@@ -314,9 +317,8 @@ def test_output_trafos(converter):
         mock_pgm_array.__getitem__.assert_any_call("q_to")
         mock_pgm_array.__getitem__.assert_any_call("i_from")
         mock_pgm_array.__getitem__.assert_any_call("i_to")
-        # mock_pgm_array.__getitem__.assert_any_call("u_pu")
-        # mock_pgm_array.__getitem__.assert_any_call("u_degree")
-        mock_pgm_array.__getitem__.assert_any_call("loading")
+        # current loading retrieval
+        mock_pgm_array.__getitem__.assert_any_call("sn")
 
         # assignment
         mock_pp_df.return_value.__setitem__.assert_any_call("p_hv_mw", ANY)
@@ -337,6 +339,42 @@ def test_output_trafos(converter):
         converter.pp_output_data.__setitem__.assert_called_once_with("res_trafo", mock_pp_df.return_value)
 
 
+def test_output_trafos__power(converter):
+    # Arrange
+    mock_pgm_array = MagicMock()
+    converter.trafo_loading = "power"
+    converter.pp_input_data["trafo"] = MockDf(2)
+    converter.pgm_input_data["transformer"] = mock_pgm_array
+    converter.pgm_output_data["transformer"] = mock_pgm_array
+    converter.pgm_nodes_lookup = pd.DataFrame(
+        {"u_pu": mock_pgm_array, "u_degree": mock_pgm_array}, index=mock_pgm_array
+    )
+
+    with patch("power_grid_model_io.converters.pandapower_converter.pd.DataFrame") as mock_pp_df:
+        # Act
+        converter._pp_trafos_output()
+
+        mock_pgm_array.__getitem__.assert_any_call("loading")
+        mock_pp_df.return_value.__setitem__.assert_any_call("loading_percent", ANY)
+        # result
+        converter.pp_output_data.__setitem__.assert_called_once_with("res_trafo", mock_pp_df.return_value)
+
+
+def test_output_trafos__invalid_trafo_loading(converter):
+    # Arrange
+    mock_pgm_array = MagicMock()
+    converter.trafo_loading = "abcd"
+    converter.pp_input_data["trafo"] = MockDf(2)
+    converter.pgm_input_data["transformer"] = mock_pgm_array
+    converter.pgm_output_data["transformer"] = mock_pgm_array
+    converter.pgm_nodes_lookup = pd.DataFrame(
+        {"u_pu": mock_pgm_array, "u_degree": mock_pgm_array}, index=mock_pgm_array
+    )
+
+    with pytest.raises(ValueError, match="Invalid transformer loading type: abcd"):
+        converter._pp_trafos_output()
+
+
 def test_output_trafo3w(converter):
     # Arrange
     mock_pgm_array = MagicMock()

tests/unit/converters/test_tabular_converter.py

@@ -266,9 +266,9 @@ def test_serialize_data(converter: TabularConverter, pgm_node_empty: SingleDatas
     pgm_node_empty["node"]["id"] = [1, 2]
     pgm_node_empty["node"]["u_rated"] = [3.0, 4.0]
     tabular_data = converter._serialize_data(data=pgm_node_empty, extra_info=None)
-    assert len(tabular_data._data) == 1
-    assert (tabular_data.get_column("node", "id") == np.array([1, 2])).all()
-    assert (tabular_data.get_column("node", "u_rated") == np.array([3.0, 4.0])).all()
+    assert len(tabular_data) == 1
+    assert (tabular_data["node"]["id"] == np.array([1, 2])).all()
+    assert (tabular_data["node"]["u_rated"] == np.array([3.0, 4.0])).all()
 
 
 def test_parse_col_def(converter: TabularConverter, tabular_data_no_units_no_substitutions: TabularData):

tests/unit/data_stores/test_csv_dir_store.py

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model project <[email protected]>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+import tempfile
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+
+from power_grid_model_io.data_stores.csv_dir_store import CsvDirStore
+from power_grid_model_io.data_types import TabularData
+
+
+@pytest.fixture()
+def temp_dir():
+    with tempfile.TemporaryDirectory() as tmp:
+        yield Path(tmp).resolve()
+
+
+def touch(file_path: Path):
+    open(file_path, "wb").close()
+
+
+@patch("power_grid_model_io.data_stores.csv_dir_store.pd.read_csv")
+def test_load(mock_read_csv: MagicMock, temp_dir: Path):
+    # Arrange
+    foo_data = MagicMock()
+    bar_data = MagicMock()
+    touch(temp_dir / "foo.csv")
+    touch(temp_dir / "bar.csv")
+    mock_read_csv.side_effect = (foo_data, bar_data)
+    csv_dir = CsvDirStore(temp_dir, bla=True)
+
+    # Act
+    csv_data = csv_dir.load()
+
+    # Assert
+    mock_read_csv.assert_not_called()  # The csv data is not yet loaded
+    assert csv_data["foo"] == foo_data
+    assert csv_data["bar"] == bar_data
+    mock_read_csv.assert_any_call(filepath_or_buffer=temp_dir / "foo.csv", header=[0], bla=True)
+    mock_read_csv.assert_any_call(filepath_or_buffer=temp_dir / "bar.csv", header=[0], bla=True)
+
+
+@patch("power_grid_model_io.data_stores.csv_dir_store.pd.DataFrame.to_csv")
+def test_save(mock_to_csv: MagicMock, temp_dir):
+    # Arrange
+    foo_data = pd.DataFrame()
+    bar_data = pd.DataFrame()
+    data = TabularData(foo=foo_data, bar=bar_data)
+    csv_dir = CsvDirStore(temp_dir, bla=True)
+
+    # Act
+    csv_dir.save(data)
+
+    # Assert
+    mock_to_csv.assert_any_call(path_or_buf=temp_dir / "foo.csv", bla=True)
+    mock_to_csv.assert_any_call(path_or_buf=temp_dir / "bar.csv", bla=True)