fix(pipeline): fix API pagination, add skip_details fast path, bulk mp_votes insert

- _get_voting_records returns (records, besluit_meta) tuple; paginate via Besluit?expand=Stemming (469/mo vs 8400)
- get_motions(skip_details=True) bypasses per-motion detail chain (3 HTTP calls/motion)
- extract_mp_votes rewritten: bulk DataFrame insert (80k rows in 1.9s), includes party-level actors
- run_pipeline.py fixed: pass db_path not db, handle dict/int return types
- download_past_year.py: skip_details=True default, limit-per-chunk default 50000

pipeline/extract_mp_votes.py

@@ -3,6 +3,7 @@ import logging
 from typing import Optional
 
 import duckdb
+import pandas as pd
 
 from database import MotionDatabase
 
@@ -13,6 +14,11 @@ def extract_mp_votes(db_path: Optional[str] = None, limit: Optional[int] = None)
     """Extract individual MP votes from motions.voting_results and store them
     in the mp_votes table.
 
+    Handles both individual MP votes (ActorNaam contains comma) and party-level
+    votes (ActorNaam has no comma) by treating party names as actors.
+
+    Uses a single DuckDB connection with DataFrame bulk insert for performance.
+
     Returns a dict with summary counts:
       - motions_scanned: number of motions inspected
       - mp_rows_inserted: number of mp_votes rows inserted
@@ -31,42 +37,56 @@ def extract_mp_votes(db_path: Optional[str] = None, limit: Optional[int] = None)
             rows = conn.execute(
                 "SELECT id, voting_results, date FROM motions"
             ).fetchall()
-    finally:
+    except Exception as e:
         conn.close()
+        raise e
 
-    mp_rows_inserted = 0
-    motions_skipped = 0
     motions_scanned = 0
+    motions_skipped = 0
+    batch = []
 
     for motion_id, voting_results_json, date in rows:
         motions_scanned += 1
+
+        # Check if mp_votes already exist for this motion
+        existing = conn.execute(
+            "SELECT COUNT(*) FROM mp_votes WHERE motion_id = ?", (motion_id,)
+        ).fetchone()
+        if existing and existing[0] > 0:
+            _logger.debug("Skipping motion %s, mp_votes already exist", motion_id)
+            motions_skipped += 1
+            continue
+
+        # voting_results may be stored as JSON text or as native JSON; ensure it's a dict
+        if isinstance(voting_results_json, str):
+            voting_results = json.loads(voting_results_json)
+        else:
+            voting_results = voting_results_json
+
+        for actor, vote in (voting_results or {}).items():
+            # Individual MP names contain a comma (e.g. "Last, F.")
+            # Party names have no comma (e.g. "VVD", "GroenLinks-PvdA")
+            party = None if "," in actor else actor
+            batch.append((motion_id, actor, party, vote, str(date) if date else None))
+
+    # Bulk insert via DataFrame for performance (avoids per-row connection overhead)
+    mp_rows_inserted = 0
+    if batch:
         try:
-            if db.mp_votes_exists_for_motion(motion_id):
-                _logger.debug(
-                    "Skipping motion %s because mp_votes already exist", motion_id
-                )
-                motions_skipped += 1
-                continue
-
-            # voting_results may be stored as JSON text or as native JSON; ensure it's a dict
-            if isinstance(voting_results_json, str):
-                voting_results = json.loads(voting_results_json)
-            else:
-                voting_results = voting_results_json
-
-            for actor, vote in (voting_results or {}).items():
-                # Individual MP names contain a comma (e.g. "Last, F.")
-                if "," not in actor:
-                    continue
-
-                inserted_id = db.insert_mp_vote(
-                    motion_id=motion_id, mp_name=actor, vote=vote, date=date, party=None
-                )
-                if inserted_id and inserted_id > 0:
-                    mp_rows_inserted += 1
-
+            df = pd.DataFrame(
+                batch, columns=["motion_id", "mp_name", "party", "vote", "date"]
+            )
+            conn.execute(
+                "INSERT INTO mp_votes (motion_id, mp_name, party, vote, date) SELECT * FROM df"
+            )
+            mp_rows_inserted = len(batch)
+            _logger.info("Bulk inserted %d mp_votes rows", mp_rows_inserted)
         except Exception as e:
-            _logger.error("Error processing motion %s: %s", motion_id, e)
+            _logger.error("Bulk insert failed: %s", e)
+    else:
+        _logger.info("No new mp_votes rows to insert")
+
+    conn.close()
 
     return {
         "motions_scanned": motions_scanned,

pipeline/run_pipeline.py

@@ -115,8 +115,8 @@ def run(args: argparse.Namespace) -> int:
         if not dry_run:
             from pipeline.fetch_mp_metadata import fetch_mp_metadata
 
-            fetched, skipped = fetch_mp_metadata(db)
-            _logger.info("  mp_metadata: fetched=%d skipped=%d", fetched, skipped)
+            n = fetch_mp_metadata(db_path=db.db_path)
+            _logger.info("  mp_metadata: processed=%d", n)
         else:
             _logger.info("  [dry-run] would call fetch_mp_metadata(db)")
     else:
@@ -128,9 +128,12 @@ def run(args: argparse.Namespace) -> int:
         if not dry_run:
             from pipeline.extract_mp_votes import extract_mp_votes
 
-            inserted, skipped = extract_mp_votes(db)
+            result = extract_mp_votes(db_path=db.db_path)
             _logger.info(
-                "  mp_votes: inserted=%d motions skipped=%d", inserted, skipped
+                "  mp_votes: inserted=%d motions_scanned=%d skipped=%d",
+                result["mp_rows_inserted"],
+                result["motions_scanned"],
+                result["motions_skipped"],
             )
         else:
             _logger.info("  [dry-run] would call extract_mp_votes(db)")
@@ -199,11 +202,12 @@ def run(args: argparse.Namespace) -> int:
                     model=args.text_model,
                 )
                 _logger.info(
-                    "  window %s: fused=%d skipped_no_svd=%d skipped_no_text=%d",
+                    "  window %s: fused=%d skipped_no_svd=%d skipped_no_text=%d errors=%d",
                     window_id,
-                    result["fused"],
-                    result.get("skipped_no_svd", 0),
-                    result.get("skipped_no_text", 0),
+                    result.get("inserted", 0),
+                    result.get("skipped_missing_svd", 0),
+                    result.get("skipped_missing_text", 0),
+                    result.get("errors", 0),
                 )
             else:
                 _logger.info("  [dry-run] would fuse window %s", window_id)