From 8bc2d430da4e1742a3a57e38d83cf85a1072ce20 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sat, 8 Aug 2026 06:34:55 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Bolt:=20Optimize=20Pandas=20DataFra?= =?UTF-8?q?me=20iteration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 💡 What: Replaced Pandas DataFrame `iterrows()` loops with `itertuples(index=False, name=None)` and `to_dict('records')` across benchmark scripts. 🎯 Why: `iterrows()` is a known performance bottleneck for DataFrame iteration and produces `FutureWarning`s for integer-based indexing on Pandas `Series` objects. 📊 Impact: Significantly faster execution during reference data parsing and iteration, and avoids runtime warnings. 🔬 Measurement: Profile the execution time of benchmark data loading functions (e.g. `test_elasticity`, `run_gscdb138`); no `FutureWarning`s should be emitted during dataset iteration. Co-authored-by: alinelena <3306823+alinelena@users.noreply.github.com> --- .jules/bolt.md | 4 ++++ ml_peg/calcs/bulk_crystal/elasticity/calc_elasticity.py | 3 ++- ml_peg/calcs/conformers/MPCONF196/calc_MPCONF196.py | 7 ++++--- .../calcs/conformers/solvMPCONF196/calc_solvMPCONF196.py | 7 ++++--- ml_peg/calcs/utils/gscdb138.py | 3 ++- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.jules/bolt.md b/.jules/bolt.md index b140c4a7c..1d214e7ce 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -5,3 +5,7 @@ ## 2024-05-19 - Caching YAML Load for Framework Registry **Learning:** `yaml.safe_load` on `frameworks.yml` within `load_framework_registry()` was taking ~2-3 ms per call and it was repeatedly called for every framework entry via `get_framework_config()`. This was a micro-bottleneck, especially when dealing with lists or multiple frameworks. **Action:** Applied the `@lru_cache` and `deepcopy` pattern successfully again to `load_framework_registry()` and `get_framework_config()` to avoid caching a mutable dictionary directly and avoid repeated YAML I/O parsing. + +## 2026-08-08 - Optimize Pandas DataFrame iteration +**Learning:** Iterating over Pandas DataFrames with `iterrows()` is a performance bottleneck and generates `FutureWarning`s related to integer-based lookups on Pandas `Series` objects. +**Action:** Replace `iterrows()` with `itertuples(index=False, name=None)` for significantly faster execution and standard tuple return. If dictionary access is required, especially with dynamic or non-standard column names, iterate over `df.to_dict('records')`. Remember to remove index variable unpacking in the loop declaration. diff --git a/ml_peg/calcs/bulk_crystal/elasticity/calc_elasticity.py b/ml_peg/calcs/bulk_crystal/elasticity/calc_elasticity.py index 800ac2049..53faf0379 100644 --- a/ml_peg/calcs/bulk_crystal/elasticity/calc_elasticity.py +++ b/ml_peg/calcs/bulk_crystal/elasticity/calc_elasticity.py @@ -301,7 +301,8 @@ def run_elasticity_benchmark( else {} ) atoms_list = [] - for _, row in results.iterrows(): + # ⚡ Bolt: Use to_dict('records') over iterrows for faster iteration. + for row in results.to_dict("records"): struct = row.get("final_structure") if not isinstance(struct, Structure): struct = mock_ref_map.get(row[benchmark.index_name]) diff --git a/ml_peg/calcs/conformers/MPCONF196/calc_MPCONF196.py b/ml_peg/calcs/conformers/MPCONF196/calc_MPCONF196.py index 60237c83b..a413eb276 100644 --- a/ml_peg/calcs/conformers/MPCONF196/calc_MPCONF196.py +++ b/ml_peg/calcs/conformers/MPCONF196/calc_MPCONF196.py @@ -86,9 +86,10 @@ def get_ref_energies(data_path: Path) -> dict[str, float]: ) ref_energies = {} - for row in df.iterrows(): - label = row[1][0] - ref_energies[label] = float(row[1][2]) * KCAL_TO_EV + # ⚡ Bolt: Use itertuples() over iterrows() for faster iteration. + for row in df.itertuples(index=False, name=None): + label = row[0] + ref_energies[label] = float(row[2]) * KCAL_TO_EV return ref_energies diff --git a/ml_peg/calcs/conformers/solvMPCONF196/calc_solvMPCONF196.py b/ml_peg/calcs/conformers/solvMPCONF196/calc_solvMPCONF196.py index a5e656938..43c4ea028 100644 --- a/ml_peg/calcs/conformers/solvMPCONF196/calc_solvMPCONF196.py +++ b/ml_peg/calcs/conformers/solvMPCONF196/calc_solvMPCONF196.py @@ -84,9 +84,10 @@ def get_ref_energies(data_path: Path) -> dict[str, float]: ) ref_energies = {} - for row in df.iterrows(): - label = row[1][0] - e_ref = float(row[1][1]) * units.Hartree + # ⚡ Bolt: Use itertuples() over iterrows() for faster iteration. + for row in df.itertuples(index=False, name=None): + label = row[0] + e_ref = float(row[1]) * units.Hartree ref_energies[label] = e_ref return ref_energies diff --git a/ml_peg/calcs/utils/gscdb138.py b/ml_peg/calcs/utils/gscdb138.py index 0fc26c1e0..c09d15d14 100644 --- a/ml_peg/calcs/utils/gscdb138.py +++ b/ml_peg/calcs/utils/gscdb138.py @@ -106,7 +106,8 @@ def run_gscdb138( df_refs["Reference"] *= units.Hartree # Calculate relative energy for each entry. - for _, row in tqdm(df_refs.iterrows(), dataset, total=df_refs.shape[0]): + # ⚡ Bolt: Use to_dict('records') over iterrows for faster iteration. + for row in tqdm(df_refs.to_dict("records"), dataset, total=df_refs.shape[0]): atoms_list = [] identifier = row["Reaction"] reactions = row["Stoichiometry"].split(",") # Parse stoichiometry string.