From 5da53224a8b1b3b83a0af532f777e72404607da5 Mon Sep 17 00:00:00 2001
From: "google-labs-jules[bot]"
 <161369871+google-labs-jules[bot]@users.noreply.github.com>
Date: Sun, 21 Jun 2026 09:25:46 +0000
Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Bolt:=20[performance=20improvement]?=
 =?UTF-8?q?=20Replace=20iterrows=20with=20to=5Fdict('records')?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replaced extremely slow `df.iterrows()` iterations with `df.to_dict('records')` in `verify_processed_omol25.py` to prevent pandas from wrapping every single row into a new `pd.Series` object, drastically improving iteration performance.

Co-authored-by: alinelena <3306823+alinelena@users.noreply.github.com>
---
 .jules/bolt.md                               |  3 +++
 src/lavello_mlips/verify_processed_omol25.py | 11 ++++++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/.jules/bolt.md b/.jules/bolt.md
index d87bd6d..4232ac9 100644
--- a/.jules/bolt.md
+++ b/.jules/bolt.md
@@ -8,3 +8,6 @@
 ## 2024-03-29 - ASE Custom JSON encoding vs standard JSON
 **Learning:** ASE's custom JSON encoder (`ase.io.jsonio.encode`) will generate dicts with special keys like `__ndarray__` or `__complex__` (e.g. `{"__ndarray__": [[5], "int64", ...]}`). When optimizing JSON deserialization using faster alternatives like `orjson`, it's critical to realize that a normal `json.loads` or `orjson.loads` will deserialize this into a Python dictionary, while ASE's custom `decode` will properly reconstruct the underlying numpy array. Bypassing ASE's decoder without checking for these keys leads to downstream type errors (e.g. `KeyError: '__ndarray__'`).
 **Action:** When replacing or wrapping ASE's jsonio with `orjson`, always fall back to ASE's `decode` if the payload string contains `__ndarray__` or `__complex__` markers, to ensure custom objects are correctly reconstructed.
+## 2024-05-19 - Pandas Iteration Bottleneck
+**Learning:** `df.iterrows()` is an extreme performance anti-pattern in pandas because it wraps every row into a new `pd.Series` object. This causes massive slowdowns on large dataframes like those processed in `verify_processed_omol25.py`.
+**Action:** Always replace `df.iterrows()` with `df.to_dict('records')` (or similar vectorized methods) to convert the dataframe to a list of native Python dictionaries before iterating.
diff --git a/src/lavello_mlips/verify_processed_omol25.py b/src/lavello_mlips/verify_processed_omol25.py
index 9d1c47c..8acb2cb 100644
--- a/src/lavello_mlips/verify_processed_omol25.py
+++ b/src/lavello_mlips/verify_processed_omol25.py
@@ -77,8 +77,13 @@ def main() -> None:
         )
 
     logger.info(f"Loaded {len(df)} records from Parquet.")
-    parquet_by_sha = {row["geom_sha1"]: row for _, row in df.iterrows()}
-    parquet_by_argone_rel = {row["argonne_rel"]: row for _, row in df.iterrows()}
+
+    # ⚡ Bolt Optimization: Replace df.iterrows() with df.to_dict("records")
+    # Why: df.iterrows() creates a slow Pandas Series for every row. Converting
+    # to a list of native Python dicts first is orders of magnitude faster.
+    df_records = df.to_dict("records")
+    parquet_by_sha = {row["geom_sha1"]: row for row in df_records}
+    parquet_by_argone_rel = {row["argonne_rel"]: row for row in df_records}
     logger.info(f"Loading ExtXYZ file from {args.extxyz} (this may take a moment)...")
     all_atoms = read(str(args.extxyz), index=":")
     if not isinstance(all_atoms, list):
@@ -108,7 +113,7 @@ def get_dump_entry(at):
             info = dict(at.info)
             rel = info.get("argonne_rel")
             pq_row = parquet_by_argone_rel.get(rel)
-            pq_data = pq_row.to_dict() if pq_row is not None else None
+            pq_data = pq_row if pq_row is not None else None
             return {"xyz": info, "parquet": pq_data}
 
         duplicates = {