feat: complete parliamentary embedding pipeline with full historical coverage

- Add fused (SVD + text) embedding pipeline for annual windows 2016-2026
- Fix store_fused_embedding duplicate bug: DELETE before INSERT (idempotent)
- Add --text-batch-size CLI flag to run_pipeline.py (default 200)
- Add explicit --start-date/--end-date to download_past_year.py
- Backfill mp_votes for all motions (party-level votes, 111k new rows)
- Add similarity cache recompute: 212k rows across 9 annual windows
- Improve ai_provider retry logic, text_pipeline batching
- Improve analysis/political_axis PCA handling and visualizations
- Add diagnostic/utility scripts: compare_svd, generate_compass, inspect_axis, etc.
- Untrack data/motions.db (3.6GB binary), add to .gitignore with outputs/
- Update continuity ledger with full session state

pipeline/run_pipeline.py

@@ -174,7 +174,7 @@ def run(args: argparse.Namespace) -> int:
             from pipeline.text_pipeline import ensure_text_embeddings
 
             stored, existing, no_text, errors = ensure_text_embeddings(
-                db_path=db_path, model=args.text_model
+                db_path=db_path, model=args.text_model, batch_size=args.text_batch_size
             )
             _logger.info(
                 "  embeddings: stored=%d existing=%d no_text=%d errors=%d",
@@ -240,6 +240,12 @@ def build_parser() -> argparse.ArgumentParser:
         default=None,
         help="Text embedding model (default: ai_provider default)",
     )
+    parser.add_argument(
+        "--text-batch-size",
+        type=int,
+        default=200,
+        help="Number of texts per embedding API call (default: 200)",
+    )
     parser.add_argument(
         "--skip-metadata", action="store_true", help="Skip MP metadata fetch"
     )

pipeline/text_pipeline.py

@@ -55,10 +55,11 @@ def _select_text(
 
 
 def ensure_text_embeddings(
-    db_path: Optional[str] = None, model: Optional[str] = None
+    db_path: Optional[str] = None, model: Optional[str] = None, batch_size: int = 50
 ) -> Tuple[int, int, int, int]:
     """Ensure all motions have text embeddings for `model`.
 
+    Uses batched API calls (batch_size texts per HTTP request) for speed.
     Returns tuple (stored_count, skipped_existing, skipped_no_text, errors).
     """
     model = model or DEFAULT_MODEL
@@ -87,14 +88,54 @@ def ensure_text_embeddings(
     skipped_no_text = 0
     errors = 0
 
+    # Separate motions with text from those without
+    with_text: List[Tuple[int, str]] = []
     for motion_id, text in to_process:
         if not text:
             _logger.info("Skipping motion %s: no text available", motion_id)
             skipped_no_text += 1
-            continue
+        else:
+            with_text.append((motion_id, text))
+
+    _logger.info(
+        "Processing %d motions in batches of %d (%d skipped no text, %d already exist)",
+        len(with_text),
+        batch_size,
+        skipped_no_text,
+        existing,
+    )
+
+    # Process in batches
+    for batch_start in range(0, len(with_text), batch_size):
+        batch = with_text[batch_start : batch_start + batch_size]
+        batch_ids = [mid for mid, _ in batch]
+        batch_texts = [txt for _, txt in batch]
 
         try:
-            vec = ai_provider.get_embedding(text, model=model)
+            vecs = ai_provider.get_embeddings_batch(
+                batch_texts, model=model, batch_size=batch_size
+            )
+        except Exception as exc:
+            _logger.error(
+                "Batch embedding failed for motions %s..%s: %s",
+                batch_ids[0],
+                batch_ids[-1],
+                exc,
+            )
+            errors += len(batch)
+            continue
+
+        if len(vecs) != len(batch):
+            _logger.error(
+                "Batch size mismatch: expected %d, got %d embeddings",
+                len(batch),
+                len(vecs),
+            )
+            errors += len(batch)
+            continue
+
+        batch_stored = 0
+        for (motion_id, _text), vec in zip(batch, vecs):
             if not isinstance(vec, list):
                 _logger.warning(
                     "Embedding provider returned non-list for motion %s", motion_id
@@ -102,21 +143,33 @@ def ensure_text_embeddings(
                 errors += 1
                 continue
 
-            res = db.store_embedding(motion_id, model, vec)
-            if res and res > 0:
-                stored += 1
-            else:
+            try:
+                res = db.store_embedding(motion_id, model, vec)
+                if res and res > 0:
+                    stored += 1
+                    batch_stored += 1
+                else:
+                    _logger.error(
+                        "Failed to store embedding for motion %s (store returned %s)",
+                        motion_id,
+                        res,
+                    )
+                    errors += 1
+            except Exception as exc:
                 _logger.error(
-                    "Failed to store embedding for motion %s (store returned %s)",
-                    motion_id,
-                    res,
+                    "Error storing embedding for motion %s: %s", motion_id, exc
                 )
                 errors += 1
-        except Exception as exc:
-            _logger.error(
-                "Error computing/storing embedding for motion %s: %s", motion_id, exc
-            )
-            errors += 1
+
+        _logger.info(
+            "Batch %d-%d: stored %d/%d (total: %d/%d)",
+            batch_start,
+            batch_start + len(batch),
+            batch_stored,
+            len(batch),
+            stored + existing,
+            total_motions,
+        )
 
     skipped_existing = int(existing)
     return stored, skipped_existing, skipped_no_text, errors