Add calamine as engine for reading Excel files (#190)

Author: euronion

docs/release_notes.rst

@@ -34,6 +34,8 @@ Upcoming Release
 
 * Add `purge` and `all` rules to clean all generated outputs and regenerate them (https://github.com/PyPSA/technology-data/pull/187)
 
+* Switch to `python-calamine` as engine for reading Excel files in `pandas`, greatly improving performance (https://github.com/PyPSA/technology-data/pull/188)
+
 `v0.11.0 <https://github.com/PyPSA/technology-data/releases/tag/v0.11.0>`__ (24th January 2025)
 =======================================================================================
 

environment.yaml

@@ -19,13 +19,17 @@ dependencies:
 - xlrd
 - scipy
 - openpyxl>=3.1.2
+- python-calamine
 - packaging
 - pytest
 
 # Development dependencies
 - pre-commit
 - ruff
 
+# Interactive execution
+- ipykernel
+
 - pip:
   - tabula-py
   - currencyconverter

scripts/compile_cost_assumptions.py

@@ -376,6 +376,7 @@ def get_dea_maritime_data(
         index_col=[0, 1],
         usecols="A:F",
         na_values="N/A",
+        engine="calamine",
     )
 
     wished_index = [
@@ -500,6 +501,7 @@ def get_dea_vehicle_data(
         index_col=0,
         usecols="A:F",
         na_values="no data",
+        engine="calamine",
     )
 
     wished_index = [
@@ -667,6 +669,7 @@ def get_data_DEA(
         usecols=usecols,
         skiprows=skiprows,
         na_values="N.A",
+        engine="calamine",
     )
 
     excel.dropna(axis=1, how="all", inplace=True)
@@ -2439,6 +2442,7 @@ def add_gas_storage(
         gas_storage_file_name,
         sheet_name="150 Underground Storage of Gas",
         index_col=1,
+        engine="calamine",
     )
     gas_storage.dropna(axis=1, how="all", inplace=True)
 
@@ -3666,6 +3670,7 @@ def add_energy_storage_database(
             "ref_size_MW": float,
             "EP_ratio_h": float,
         },
+        engine="calamine",
     )
     df = df.drop(columns=["ref_size_MW", "EP_ratio_h"])
     df = df.fillna(df.dtypes.replace({"float64": 0.0, "O": "NULL"}))
@@ -3939,7 +3944,9 @@ def prepare_inflation_rate(fn: str) -> pd.Series:
         inflation rates series
     """
 
-    inflation_rate = pd.read_excel(fn, sheet_name="Sheet 1", index_col=0, header=[8])
+    inflation_rate = pd.read_excel(
+        fn, sheet_name="Sheet 1", index_col=0, header=[8], engine="calamine"
+    )
     inflation_rate = (
         inflation_rate.loc["European Union - 27 countries (from 2020)"].dropna()
     ).loc["2001"::]