deploy to server

pipeline/run_pipeline.py

@@ -202,7 +202,7 @@ def run(args: argparse.Namespace) -> int:
         if not dry_run:
             from pipeline.text_pipeline import ensure_text_embeddings
 
-            stored, existing, no_text, errors = ensure_text_embeddings(
+            stored, existing, no_text, errors, _failed_ids = ensure_text_embeddings(
                 db_path=db_path, model=args.text_model, batch_size=args.text_batch_size
             )
             _logger.info(

pipeline/svd_pipeline.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import re
 from typing import Optional, Dict, List, Tuple
 
 import numpy as np
@@ -63,6 +64,68 @@ _PARTY_NAME_MAP = {
 # Party names for which we have no usable mp_metadata (tiny noise, skip expansion)
 _SKIP_PARTIES = {"Brinkman", "Bontes", "Krol", "Van Kooten-Arissen"}
 
+# Special-character corrections for individual vote name parts
+_NAME_CHAR_FIXES: Dict[str, str] = {
+    "Gündogan": "Gündoğan",
+}
+
+
+def _votes_name_to_meta_format(votes_name: str) -> str:
+    """Convert an mp_votes individual-record name to mp_metadata canonical format.
+
+    mp_votes format:  ``{surname} {lowercase_tussenvoegsel}, {initials} ({FirstName})``
+                      e.g. ``Dijk van, I. (Inge)``   →   ``Van Dijk, I.``
+                           ``Beer de, M.E.E.``        →   ``De Beer, M.E.E.``
+                           ``Abassi el, I.``          →   ``El Abassi, I.``
+                           ``Baarle van, S.R.T.``     →   ``Van Baarle, S.R.T.``
+
+    mp_metadata format: ``{Capital_tussenvoegsel} {Achternaam}, {initials}``
+
+    Steps:
+    1. Split on ``, `` → name_part, initials_part.
+    2. Strip parenthetical first name from initials_part.
+    3. In name_part, isolate trailing lowercase words as tussenvoegsel;
+       the rest is the achternaam.
+    4. Reconstruct as ``{Capitalized tussenvoegsel} {achternaam}, {initials}``.
+    5. Apply special-character fixes.
+    """
+    if "," not in votes_name:
+        return votes_name
+
+    comma_idx = votes_name.index(",")
+    name_part = votes_name[:comma_idx].strip()
+    initials_part = votes_name[comma_idx + 1 :].strip()
+
+    # Remove parenthetical first name, e.g. "(Inge)" or "(Jan-Willem)"
+    initials_part = re.sub(r"\s*\([^)]+\)$", "", initials_part).strip()
+
+    # Split name_part into words; trailing lowercase words are tussenvoegsel
+    words = name_part.split()
+    # Find split point: last run of lowercase words at the end
+    split = len(words)
+    for i in range(len(words) - 1, -1, -1):
+        if words[i][0].islower():
+            split = i
+        else:
+            break
+    achternaam_words = words[:split]
+    tussenvoegsel_words = words[split:]
+
+    if tussenvoegsel_words:
+        # Capitalize the first letter of the first tussenvoegsel word
+        tussenvoegsel_words[0] = tussenvoegsel_words[0].capitalize()
+        canonical = (
+            " ".join(tussenvoegsel_words + achternaam_words) + ", " + initials_part
+        )
+    else:
+        canonical = " ".join(achternaam_words) + ", " + initials_part
+
+    # Apply special-character fixes
+    for bad, good in _NAME_CHAR_FIXES.items():
+        canonical = canonical.replace(bad, good)
+
+    return canonical
+
 
 def _build_expanded_rows(
     db_path: str, start_date: str, end_date: str
@@ -136,9 +199,11 @@ def _build_expanded_rows(
     all_motion_ids = set(motion_individual.keys()) | set(motion_party.keys())
     for mid in all_motion_ids:
         if mid in motion_individual and motion_individual[mid]:
-            # Motion already has individual MP rows — use them directly, skip party rows
+            # Motion already has individual MP rows — convert to mp_metadata name format,
+            # then use directly; skip party rows for this motion.
             for mp_name, vote, date in motion_individual[mid]:
-                expanded.append((mid, mp_name, vote, str(date)))
+                canonical_name = _votes_name_to_meta_format(str(mp_name))
+                expanded.append((mid, canonical_name, vote, str(date)))
         else:
             # Party-only motion — expand each party row to individual MPs
             for party_name, vote, date in motion_party[mid]: