feat(pipeline): add orchestrator CLI, analysis modules, and ActorFractie ingestion

- pipeline/run_pipeline.py: CLI orchestrator for all 5 pipeline phases with
  --dry-run, --skip-*, --window-size, --svd-k, --start/end-date flags
- analysis/{political_axis,trajectory,clustering,visualize}.py: PCA/anchor
  ideological axis, MP drift trajectories, UMAP + KMeans clustering, Plotly HTML output
- api_client.py: capture ActorFractie per individual MP vote (comma in ActorNaam)
  into mp_vote_parties dict on each motion
- database.insert_motion: auto-insert mp_votes rows with party affiliation for
  newly ingested motions when mp_vote_parties is present
- Add scikit-learn to pyproject.toml for KMeans clustering
- tests/test_run_pipeline.py: window generation, dry-run, skip-all paths
- tests/test_analysis.py: PCA axis, anchor axis, trajectory drift, KMeans

Ref: thoughts/shared/plans/2026-03-21-parliamentary-embedding-pipeline-plan.md

analysis/__init__.py

@@ -0,0 +1,8 @@
+"""Analysis modules for the parliamentary embedding pipeline.
+
+Modules:
+    political_axis  — project MP SVD vectors onto ideological axis
+    trajectory      — compute MP drift across aligned windows
+    clustering      — UMAP dimensionality reduction + cluster labelling
+    visualize       — Plotly interactive plots (outputs self-contained HTML)
+"""

analysis/clustering.py

@@ -0,0 +1,130 @@
+"""clustering.py — UMAP dimensionality reduction on fused embeddings.
+
+Reduces fused motion embeddings to 2D (or 3D) for visualisation,
+and optionally labels clusters using KMeans.
+
+Requires: umap-learn, scikit-learn (for KMeans)
+"""
+
+import json
+import logging
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+import duckdb
+
+_logger = logging.getLogger(__name__)
+
+
+def _load_fused_vectors(
+    db_path: str, window_id: Optional[str] = None
+) -> Tuple[List[int], List[str], np.ndarray]:
+    """Load fused embeddings from the DB.
+
+    Returns (motion_ids, window_ids, matrix).
+    Optionally filter by window_id.
+    """
+    conn = duckdb.connect(db_path)
+    if window_id:
+        rows = conn.execute(
+            "SELECT motion_id, window_id, vector FROM fused_embeddings WHERE window_id = ?",
+            (window_id,),
+        ).fetchall()
+    else:
+        rows = conn.execute(
+            "SELECT motion_id, window_id, vector FROM fused_embeddings ORDER BY window_id, motion_id"
+        ).fetchall()
+    conn.close()
+
+    motion_ids, window_ids, vectors = [], [], []
+    for motion_id, wid, vec_json in rows:
+        try:
+            vec = json.loads(vec_json)
+            motion_ids.append(int(motion_id))
+            window_ids.append(wid)
+            vectors.append(vec)
+        except Exception:
+            _logger.warning("Could not parse fused vector for motion %s", motion_id)
+
+    if not vectors:
+        return [], [], np.zeros((0, 0))
+
+    # Pad to common length if needed (shouldn't happen if pipeline is consistent)
+    max_len = max(len(v) for v in vectors)
+    mat = np.zeros((len(vectors), max_len), dtype=float)
+    for i, v in enumerate(vectors):
+        mat[i, : len(v)] = v
+
+    return motion_ids, window_ids, mat
+
+
+def run_umap(
+    db_path: str,
+    window_id: Optional[str] = None,
+    n_components: int = 2,
+    n_neighbors: int = 15,
+    min_dist: float = 0.1,
+    random_state: int = 42,
+) -> Dict:
+    """Run UMAP on fused embeddings and return 2D/3D coordinates.
+
+    Returns:
+        {
+          "motion_ids": [...],
+          "window_ids": [...],
+          "coords": [[x, y], ...],   # or [x, y, z] if n_components=3
+          "n_components": int,
+        }
+    """
+    try:
+        import umap
+    except ImportError:
+        _logger.error("umap-learn is not installed; cannot run UMAP")
+        return {}
+
+    motion_ids, window_ids, mat = _load_fused_vectors(db_path, window_id)
+    if mat.size == 0:
+        _logger.warning("No fused embeddings found for window_id=%s", window_id)
+        return {}
+
+    if mat.shape[0] < n_neighbors + 1:
+        # UMAP requires at least n_neighbors+1 samples
+        n_neighbors = max(2, mat.shape[0] - 1)
+        _logger.warning(
+            "Reduced n_neighbors to %d due to small dataset (%d samples)",
+            n_neighbors,
+            mat.shape[0],
+        )
+
+    reducer = umap.UMAP(
+        n_components=n_components,
+        n_neighbors=n_neighbors,
+        min_dist=min_dist,
+        random_state=random_state,
+    )
+    coords = reducer.fit_transform(mat)
+
+    return {
+        "motion_ids": motion_ids,
+        "window_ids": window_ids,
+        "coords": coords.tolist(),
+        "n_components": n_components,
+    }
+
+
+def cluster_kmeans(
+    coords: np.ndarray, n_clusters: int = 8, random_state: int = 42
+) -> np.ndarray:
+    """Run KMeans on 2D/3D UMAP coordinates.
+
+    Returns array of integer cluster labels (length = len(coords)).
+    """
+    try:
+        from sklearn.cluster import KMeans
+    except ImportError:
+        _logger.error("scikit-learn is not installed; cannot run KMeans")
+        return np.zeros(len(coords), dtype=int)
+
+    n_clusters = min(n_clusters, len(coords))
+    km = KMeans(n_clusters=n_clusters, random_state=random_state, n_init="auto")
+    return km.fit_predict(coords)

analysis/political_axis.py

@@ -0,0 +1,125 @@
+"""political_axis.py — Project MP SVD vectors onto an ideological axis.
+
+Two modes:
+  1. PCA mode (default): compute the first principal component of all MP SVD
+     vectors for a window and project each MP onto it.  The sign is arbitrary
+     but consistent within a window.
+
+  2. Anchor mode: define the axis as the vector from the centroid of
+     ``left_parties`` to the centroid of ``right_parties``.  Project all MPs
+     onto this normalised anchor axis.
+
+Both modes return a dict mapping mp_name → scalar score for the given window.
+"""
+
+import json
+import logging
+from typing import Dict, List, Optional
+
+import numpy as np
+import duckdb
+
+_logger = logging.getLogger(__name__)
+
+
+def _load_mp_svd_vectors(db_path: str, window_id: str) -> Dict[str, np.ndarray]:
+    """Load all MP SVD vectors for a window from svd_vectors table."""
+    conn = duckdb.connect(db_path)
+    rows = conn.execute(
+        "SELECT entity_id, vector FROM svd_vectors WHERE window_id = ? AND entity_type = 'mp'",
+        (window_id,),
+    ).fetchall()
+    conn.close()
+
+    result = {}
+    for mp_name, vec_json in rows:
+        try:
+            result[mp_name] = np.array(json.loads(vec_json), dtype=float)
+        except Exception:
+            _logger.warning("Could not parse SVD vector for MP %s", mp_name)
+    return result
+
+
+def compute_pca_axis(db_path: str, window_id: str) -> Dict[str, float]:
+    """Project MP SVD vectors onto their first principal component.
+
+    Returns {mp_name: score}.  Returns empty dict if fewer than 2 MPs.
+    """
+    mp_vecs = _load_mp_svd_vectors(db_path, window_id)
+    if len(mp_vecs) < 2:
+        _logger.warning(
+            "window %s has only %d MPs; skipping PCA axis", window_id, len(mp_vecs)
+        )
+        return {}
+
+    names = list(mp_vecs.keys())
+    mat = np.vstack([mp_vecs[n] for n in names])  # (n_mps, k)
+
+    # Centre
+    mat_centred = mat - mat.mean(axis=0)
+
+    # First PC via SVD
+    try:
+        _, _, Vt = np.linalg.svd(mat_centred, full_matrices=False)
+        axis = Vt[0]  # (k,)
+    except np.linalg.LinAlgError:
+        _logger.exception("SVD failed in compute_pca_axis for window %s", window_id)
+        return {}
+
+    projections = mat_centred.dot(axis)
+    return {name: float(score) for name, score in zip(names, projections)}
+
+
+def compute_anchor_axis(
+    db_path: str,
+    window_id: str,
+    left_parties: List[str],
+    right_parties: List[str],
+) -> Dict[str, float]:
+    """Project MP SVD vectors onto a left↔right anchor axis.
+
+    The axis runs from the centroid of ``left_parties`` to the centroid of
+    ``right_parties``.  Positive scores are toward the right.
+
+    Returns {mp_name: score}.
+    """
+    mp_vecs = _load_mp_svd_vectors(db_path, window_id)
+    if not mp_vecs:
+        return {}
+
+    # Load party affiliation for this window from mp_metadata
+    conn = duckdb.connect(db_path)
+    rows = conn.execute("SELECT mp_name, party FROM mp_metadata").fetchall()
+    conn.close()
+    party_of = {mp: party for mp, party in rows}
+
+    left_vecs = [
+        mp_vecs[mp]
+        for mp, party in party_of.items()
+        if party in left_parties and mp in mp_vecs
+    ]
+    right_vecs = [
+        mp_vecs[mp]
+        for mp, party in party_of.items()
+        if party in right_parties and mp in mp_vecs
+    ]
+
+    if not left_vecs or not right_vecs:
+        _logger.warning(
+            "window %s: insufficient anchor parties (left=%d, right=%d)",
+            window_id,
+            len(left_vecs),
+            len(right_vecs),
+        )
+        return {}
+
+    left_centroid = np.mean(left_vecs, axis=0)
+    right_centroid = np.mean(right_vecs, axis=0)
+    axis = right_centroid - left_centroid
+    norm = np.linalg.norm(axis)
+    if norm < 1e-10:
+        _logger.warning("Anchor axis has near-zero norm for window %s", window_id)
+        return {}
+    axis = axis / norm
+
+    return {name: float(np.dot(vec, axis)) for name, vec in mp_vecs.items()}

analysis/trajectory.py

@@ -0,0 +1,123 @@
+"""trajectory.py — Compute MP political drift across aligned time windows.
+
+For each MP that appears in multiple windows, computes:
+  - The aligned SVD vector per window
+  - The Euclidean distance between consecutive windows (drift)
+  - Total cumulative drift
+
+Returns a dict keyed by mp_name containing per-window positions and drift scores.
+"""
+
+import json
+import logging
+from typing import Dict, List, Optional
+
+import numpy as np
+import duckdb
+
+_logger = logging.getLogger(__name__)
+
+
+def _load_window_ids(db_path: str) -> List[str]:
+    """Return all distinct window IDs from svd_vectors, in lexicographic order."""
+    conn = duckdb.connect(db_path)
+    rows = conn.execute(
+        "SELECT DISTINCT window_id FROM svd_vectors WHERE entity_type = 'mp' ORDER BY window_id"
+    ).fetchall()
+    conn.close()
+    return [r[0] for r in rows]
+
+
+def _load_mp_vectors_for_window(db_path: str, window_id: str) -> Dict[str, np.ndarray]:
+    conn = duckdb.connect(db_path)
+    rows = conn.execute(
+        "SELECT entity_id, vector FROM svd_vectors WHERE window_id = ? AND entity_type = 'mp'",
+        (window_id,),
+    ).fetchall()
+    conn.close()
+    result = {}
+    for mp_name, vec_json in rows:
+        try:
+            result[mp_name] = np.array(json.loads(vec_json), dtype=float)
+        except Exception:
+            _logger.warning(
+                "Could not parse vector for MP %s window %s", mp_name, window_id
+            )
+    return result
+
+
+def compute_trajectories(
+    db_path: str,
+    window_ids: Optional[List[str]] = None,
+) -> Dict[str, Dict]:
+    """Compute per-MP trajectories across windows.
+
+    Returns:
+        {
+          mp_name: {
+            "windows": [window_id, ...],
+            "vectors": [[...], ...],   # one vector per window
+            "drift": [float, ...],     # consecutive Euclidean distances
+            "total_drift": float,
+          }
+        }
+    Only MPs present in at least 2 windows are included.
+    """
+    if window_ids is None:
+        window_ids = _load_window_ids(db_path)
+
+    if len(window_ids) < 2:
+        _logger.info("Fewer than 2 windows — no trajectories to compute")
+        return {}
+
+    # Collect per-window vectors for each MP
+    mp_data: Dict[str, Dict] = {}
+
+    for wid in window_ids:
+        vecs = _load_mp_vectors_for_window(db_path, wid)
+        for mp_name, vec in vecs.items():
+            if mp_name not in mp_data:
+                mp_data[mp_name] = {"windows": [], "vectors": []}
+            mp_data[mp_name]["windows"].append(wid)
+            mp_data[mp_name]["vectors"].append(vec)
+
+    # Compute drift for MPs with >= 2 windows
+    result = {}
+    for mp_name, data in mp_data.items():
+        if len(data["windows"]) < 2:
+            continue
+        vecs = data["vectors"]
+        drifts = [
+            float(np.linalg.norm(vecs[i + 1] - vecs[i])) for i in range(len(vecs) - 1)
+        ]
+        result[mp_name] = {
+            "windows": data["windows"],
+            "vectors": [v.tolist() for v in vecs],
+            "drift": drifts,
+            "total_drift": float(sum(drifts)),
+        }
+
+    _logger.info(
+        "Trajectories computed for %d MPs across %d windows",
+        len(result),
+        len(window_ids),
+    )
+    return result
+
+
+def top_drifters(trajectories: Dict[str, Dict], n: int = 10) -> List[Dict]:
+    """Return the top-n MPs by total drift, sorted descending.
+
+    Each entry: {"mp_name": ..., "total_drift": ..., "windows": [...]}
+    """
+    ranked = sorted(
+        trajectories.items(), key=lambda kv: kv[1]["total_drift"], reverse=True
+    )
+    return [
+        {
+            "mp_name": mp,
+            "total_drift": data["total_drift"],
+            "windows": data["windows"],
+        }
+        for mp, data in ranked[:n]
+    ]

analysis/visualize.py

@@ -0,0 +1,163 @@
+"""visualize.py — Plotly interactive plots for parliamentary embeddings.
+
+Produces self-contained HTML files.
+
+Functions:
+  plot_umap_scatter   — 2D scatter of fused motion embeddings, coloured by cluster
+  plot_mp_trajectory  — Line plot of MP drift across windows
+  plot_political_axis — Bar chart of MP scores on the ideological axis
+"""
+
+import logging
+from typing import Dict, List, Optional
+
+import numpy as np
+
+_logger = logging.getLogger(__name__)
+
+
+def _require_plotly():
+    try:
+        import plotly.graph_objects as go
+        import plotly.express as px
+
+        return go, px
+    except ImportError:
+        raise ImportError("plotly is not installed. Install it with: uv add plotly")
+
+
+def plot_umap_scatter(
+    motion_ids: List[int],
+    coords: List[List[float]],
+    labels: Optional[List[int]] = None,
+    window_id: Optional[str] = None,
+    output_path: str = "analysis_umap.html",
+) -> str:
+    """Produce a 2D scatter plot of UMAP-reduced fused embeddings.
+
+    Args:
+        motion_ids:  Motion IDs (used as hover labels)
+        coords:      List of [x, y] coordinates
+        labels:      Optional cluster labels (integer per motion)
+        window_id:   Window label for the plot title
+        output_path: Where to write the self-contained HTML
+
+    Returns the output_path on success.
+    """
+    go, px = _require_plotly()
+
+    xs = [c[0] for c in coords]
+    ys = [c[1] for c in coords]
+    color = labels if labels is not None else [0] * len(motion_ids)
+    title = f"UMAP — fused motion embeddings" + (f" ({window_id})" if window_id else "")
+
+    fig = px.scatter(
+        x=xs,
+        y=ys,
+        color=[str(c) for c in color],
+        hover_name=[str(mid) for mid in motion_ids],
+        title=title,
+        labels={"x": "UMAP-1", "y": "UMAP-2", "color": "Cluster"},
+    )
+    fig.write_html(output_path, include_plotlyjs="cdn")
+    _logger.info("UMAP scatter written to %s", output_path)
+    return output_path
+
+
+def plot_mp_trajectory(
+    trajectories: Dict[str, Dict],
+    mp_names: Optional[List[str]] = None,
+    output_path: str = "analysis_trajectory.html",
+) -> str:
+    """Line plot of MP drift across time windows.
+
+    Args:
+        trajectories: Output of analysis.trajectory.compute_trajectories()
+        mp_names:     Subset of MPs to plot (default: all)
+        output_path:  Output HTML file path
+
+    Returns the output_path on success.
+    """
+    go, px = _require_plotly()
+
+    if mp_names is None:
+        mp_names = list(trajectories.keys())
+
+    fig = go.Figure()
+
+    for mp in mp_names:
+        if mp not in trajectories:
+            continue
+        data = trajectories[mp]
+        windows = data["windows"]
+        drifts_cumulative = [0.0] + list(np.cumsum(data["drift"]))
+        # Plot cumulative drift per window transition
+        x_labels = windows[: len(drifts_cumulative)]
+        fig.add_trace(
+            go.Scatter(
+                x=x_labels,
+                y=drifts_cumulative,
+                mode="lines+markers",
+                name=mp,
+            )
+        )
+
+    fig.update_layout(
+        title="MP Political Drift Over Time (Cumulative)",
+        xaxis_title="Window",
+        yaxis_title="Cumulative Drift",
+    )
+    fig.write_html(output_path, include_plotlyjs="cdn")
+    _logger.info("Trajectory plot written to %s", output_path)
+    return output_path
+
+
+def plot_political_axis(
+    scores: Dict[str, float],
+    party_of: Optional[Dict[str, str]] = None,
+    window_id: Optional[str] = None,
+    n_top: int = 30,
+    output_path: str = "analysis_political_axis.html",
+) -> str:
+    """Horizontal bar chart of MP scores on the ideological axis.
+
+    Args:
+        scores:      {mp_name: score} from political_axis module
+        party_of:    Optional {mp_name: party} for colour-coding
+        window_id:   Window label for the title
+        n_top:       Show only the top/bottom n MPs by score
+        output_path: Output HTML path
+
+    Returns the output_path on success.
+    """
+    go, px = _require_plotly()
+
+    # Sort by score
+    sorted_items = sorted(scores.items(), key=lambda kv: kv[1])
+
+    # Take n_top from each end if list is large
+    if len(sorted_items) > 2 * n_top:
+        sorted_items = sorted_items[:n_top] + sorted_items[-n_top:]
+
+    names = [item[0] for item in sorted_items]
+    vals = [item[1] for item in sorted_items]
+    colors = (
+        [party_of.get(n, "Unknown") for n in names]
+        if party_of
+        else ["Unknown"] * len(names)
+    )
+
+    title = "MP Ideological Axis Score" + (f" ({window_id})" if window_id else "")
+
+    fig = px.bar(
+        x=vals,
+        y=names,
+        color=colors,
+        orientation="h",
+        title=title,
+        labels={"x": "Score (← left — right →)", "y": "MP", "color": "Party"},
+    )
+    fig.update_layout(yaxis={"categoryorder": "total ascending"})
+    fig.write_html(output_path, include_plotlyjs="cdn")
+    _logger.info("Political axis chart written to %s", output_path)
+    return output_path

api_client.py

@@ -92,7 +92,12 @@ class TweedeKamerAPI:
 
         # Group records by Besluit_Id (decision/motion)
         motion_groups = defaultdict(
-            lambda: {"votes": {}, "besluit_id": None, "latest_date": None}
+            lambda: {
+                "votes": {},
+                "mp_vote_parties": {},
+                "besluit_id": None,
+                "latest_date": None,
+            }
         )
 
         for record in records:
@@ -120,6 +125,14 @@ class TweedeKamerAPI:
             motion_groups[besluit_id]["votes"][party_name] = vote
             motion_groups[besluit_id]["besluit_id"] = besluit_id
 
+            # For individual MPs (ActorNaam contains comma), also capture their party
+            if "," in party_name:
+                actor_fractie = record.get("ActorFractie")
+                if actor_fractie:
+                    motion_groups[besluit_id]["mp_vote_parties"][party_name] = (
+                        actor_fractie
+                    )
+
             # Track the latest date for this motion
             if (
                 not motion_groups[besluit_id]["latest_date"]
@@ -166,6 +179,7 @@ class TweedeKamerAPI:
                     motion_details["title"], motion_details["description"]
                 ),
                 "voting_results": voting_results,
+                "mp_vote_parties": motion_data["mp_vote_parties"],
                 "winning_margin": winning_margin,
                 "url": f"https://www.tweedekamer.nl/kamerstukken/stemmingsuitslagen/{besluit_id}",
                 "externe_identifier": motion_details.get("externe_identifier"),

database.py

@@ -190,6 +190,33 @@ class MotionDatabase:
             )
 
             conn.close()
+
+            # Also insert mp_vote rows for individual MPs if party data is available.
+            # This only runs for brand-new motions (existing motions are rejected above),
+            # so there is no risk of duplicates — no existence check needed here.
+            mp_vote_parties = motion_data.get("mp_vote_parties", {})
+            voting_results_raw = motion_data.get("voting_results", {})
+            if mp_vote_parties:
+                conn2 = duckdb.connect(self.db_path)
+                row = conn2.execute(
+                    "SELECT id FROM motions WHERE url = ? LIMIT 1",
+                    (motion_data["url"],),
+                ).fetchone()
+                conn2.close()
+                motion_id = row[0] if row else None
+
+                if motion_id is not None:
+                    motion_date = motion_data.get("date", "")
+                    for mp_name, party in mp_vote_parties.items():
+                        vote = voting_results_raw.get(mp_name, "afwezig")
+                        self.insert_mp_vote(
+                            motion_id=motion_id,
+                            mp_name=mp_name,
+                            party=party,
+                            vote=vote,
+                            date=motion_date,
+                        )
+
             return True
 
         except Exception as e:

pipeline/run_pipeline.py

@@ -0,0 +1,261 @@
+"""CLI orchestrator for the parliamentary embedding pipeline.
+
+Runs all phases in sequence:
+  1. fetch_mp_metadata  — pull MP party + tenure from OData
+  2. extract_mp_votes   — parse voting_results JSON → mp_votes rows
+  3. svd per window     — build vote matrix, SVD, Procrustes-align
+  4. text embeddings    — fill any gaps in the embeddings table
+  5. fuse per window    — concatenate SVD + text vectors → fused_embeddings
+
+Usage:
+    uv run python -m pipeline.run_pipeline [options]
+
+Options:
+    --db-path PATH        Path to the DuckDB file (default: data/motions.db)
+    --start-date DATE     Window start (YYYY-MM-DD, default: 2 years ago)
+    --end-date DATE       Window end   (YYYY-MM-DD, default: today)
+    --window-size {quarterly,annual}  Time window granularity (default: quarterly)
+    --svd-k INT           SVD dimensionality (default: 50)
+    --text-model TEXT     Text embedding model name (default: from ai_provider)
+    --skip-metadata       Skip fetching MP metadata from OData
+    --skip-extract        Skip extracting MP votes from voting_results
+    --skip-svd            Skip SVD computation
+    --skip-text           Skip text embedding gap-fill
+    --skip-fusion         Skip vector fusion
+    --dry-run             Print actions but make no DB writes
+"""
+
+import argparse
+import calendar
+import logging
+import sys
+from datetime import date, timedelta
+from typing import List, Tuple
+
+from database import MotionDatabase
+
+_logger = logging.getLogger(__name__)
+
+
+def _generate_windows(
+    start: date, end: date, granularity: str
+) -> List[Tuple[str, str, str]]:
+    """Return list of (window_id, start_str, end_str) tuples.
+
+    window_id format:
+      quarterly → "2024-Q1", "2024-Q2", …
+      annual    → "2024"
+    """
+    windows = []
+    cursor = date(start.year, start.month, 1)
+
+    if granularity == "annual":
+        cursor = date(start.year, 1, 1)
+        while cursor <= end:
+            year_end = date(cursor.year, 12, 31)
+            w_end = min(year_end, end)
+            windows.append((str(cursor.year), cursor.isoformat(), w_end.isoformat()))
+            cursor = date(cursor.year + 1, 1, 1)
+    else:
+        # quarterly
+        quarter_starts = {1: 1, 2: 4, 3: 7, 4: 10}
+        quarter_ends = {1: 3, 2: 6, 3: 9, 4: 12}
+
+        # Align cursor to quarter start
+        q = (cursor.month - 1) // 3 + 1
+        cursor = date(cursor.year, quarter_starts[q], 1)
+
+        while cursor <= end:
+            q = (cursor.month - 1) // 3 + 1
+            q_end_month = quarter_ends[q]
+            last_day = calendar.monthrange(cursor.year, q_end_month)[1]
+            q_end = date(cursor.year, q_end_month, last_day)
+            w_end = min(q_end, end)
+            window_id = f"{cursor.year}-Q{q}"
+            windows.append((window_id, cursor.isoformat(), w_end.isoformat()))
+            cursor = q_end + timedelta(days=1)
+
+    return windows
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the pipeline. Returns exit code (0 = success)."""
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+
+    db_path = args.db_path
+    dry_run = args.dry_run
+
+    if dry_run:
+        _logger.info("DRY RUN — no writes will be made")
+
+    # Resolve date range
+    end_date = date.fromisoformat(args.end_date) if args.end_date else date.today()
+    start_date = (
+        date.fromisoformat(args.start_date)
+        if args.start_date
+        else end_date - timedelta(days=730)
+    )
+
+    _logger.info(
+        "Pipeline run: %s → %s (%s windows), db=%s",
+        start_date,
+        end_date,
+        args.window_size,
+        db_path,
+    )
+
+    db = MotionDatabase(db_path)
+
+    # ── Phase 1: MP metadata ────────────────────────────────────────────────
+    if not args.skip_metadata:
+        _logger.info("Phase 1: fetching MP metadata from OData")
+        if not dry_run:
+            from pipeline.fetch_mp_metadata import fetch_mp_metadata
+
+            fetched, skipped = fetch_mp_metadata(db)
+            _logger.info("  mp_metadata: fetched=%d skipped=%d", fetched, skipped)
+        else:
+            _logger.info("  [dry-run] would call fetch_mp_metadata(db)")
+    else:
+        _logger.info("Phase 1: skipped (--skip-metadata)")
+
+    # ── Phase 2: Extract MP votes ────────────────────────────────────────────
+    if not args.skip_extract:
+        _logger.info("Phase 2: extracting MP votes from voting_results")
+        if not dry_run:
+            from pipeline.extract_mp_votes import extract_mp_votes
+
+            inserted, skipped = extract_mp_votes(db)
+            _logger.info(
+                "  mp_votes: inserted=%d motions skipped=%d", inserted, skipped
+            )
+        else:
+            _logger.info("  [dry-run] would call extract_mp_votes(db)")
+    else:
+        _logger.info("Phase 2: skipped (--skip-extract)")
+
+    # ── Phase 3: SVD per window ──────────────────────────────────────────────
+    if not args.skip_svd:
+        windows = _generate_windows(start_date, end_date, args.window_size)
+        _logger.info("Phase 3: SVD for %d windows (k=%d)", len(windows), args.svd_k)
+        from pipeline.svd_pipeline import run_svd_for_window
+
+        for window_id, w_start, w_end in windows:
+            _logger.info("  window %s: %s → %s", window_id, w_start, w_end)
+            if not dry_run:
+                result = run_svd_for_window(
+                    db=db,
+                    window_id=window_id,
+                    start_date=w_start,
+                    end_date=w_end,
+                    k=args.svd_k,
+                )
+                _logger.info(
+                    "    k_used=%d stored_mp=%d stored_motion=%d",
+                    result["k_used"],
+                    result["stored_mp"],
+                    result["stored_motion"],
+                )
+            else:
+                _logger.info("  [dry-run] would run SVD for window %s", window_id)
+    else:
+        _logger.info("Phase 3: skipped (--skip-svd)")
+
+    # ── Phase 4: Text embeddings ──────────────────────────────────────────────
+    if not args.skip_text:
+        _logger.info("Phase 4: ensuring text embeddings")
+        if not dry_run:
+            from pipeline.text_pipeline import ensure_text_embeddings
+
+            stored, existing, no_text, errors = ensure_text_embeddings(
+                db_path=db_path, model=args.text_model
+            )
+            _logger.info(
+                "  embeddings: stored=%d existing=%d no_text=%d errors=%d",
+                stored,
+                existing,
+                no_text,
+                errors,
+            )
+        else:
+            _logger.info("  [dry-run] would call ensure_text_embeddings")
+    else:
+        _logger.info("Phase 4: skipped (--skip-text)")
+
+    # ── Phase 5: Fusion per window ────────────────────────────────────────────
+    if not args.skip_fusion:
+        windows = _generate_windows(start_date, end_date, args.window_size)
+        _logger.info("Phase 5: fusing vectors for %d windows", len(windows))
+        from pipeline.fusion import fuse_for_window
+
+        for window_id, _w_start, _w_end in windows:
+            if not dry_run:
+                result = fuse_for_window(
+                    window_id=window_id,
+                    db_path=db_path,
+                    model=args.text_model,
+                )
+                _logger.info(
+                    "  window %s: fused=%d skipped_no_svd=%d skipped_no_text=%d",
+                    window_id,
+                    result["fused"],
+                    result.get("skipped_no_svd", 0),
+                    result.get("skipped_no_text", 0),
+                )
+            else:
+                _logger.info("  [dry-run] would fuse window %s", window_id)
+    else:
+        _logger.info("Phase 5: skipped (--skip-fusion)")
+
+    _logger.info("Pipeline complete.")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Parliamentary embedding pipeline orchestrator",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--db-path", default="data/motions.db", help="Path to DuckDB file"
+    )
+    parser.add_argument("--start-date", default=None, help="Window start YYYY-MM-DD")
+    parser.add_argument("--end-date", default=None, help="Window end YYYY-MM-DD")
+    parser.add_argument(
+        "--window-size",
+        choices=["quarterly", "annual"],
+        default="quarterly",
+        help="Time window granularity",
+    )
+    parser.add_argument("--svd-k", type=int, default=50, help="SVD dimensions")
+    parser.add_argument(
+        "--text-model",
+        default=None,
+        help="Text embedding model (default: ai_provider default)",
+    )
+    parser.add_argument(
+        "--skip-metadata", action="store_true", help="Skip MP metadata fetch"
+    )
+    parser.add_argument(
+        "--skip-extract", action="store_true", help="Skip MP vote extraction"
+    )
+    parser.add_argument("--skip-svd", action="store_true", help="Skip SVD computation")
+    parser.add_argument(
+        "--skip-text", action="store_true", help="Skip text embedding gap-fill"
+    )
+    parser.add_argument("--skip-fusion", action="store_true", help="Skip vector fusion")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Print what would happen without writing anything",
+    )
+    return parser
+
+
+if __name__ == "__main__":
+    parser = build_parser()
+    args = parser.parse_args()
+    sys.exit(run(args))

pyproject.toml

@@ -15,4 +15,5 @@ dependencies = [
     "requests>=2.32.4",
     "schedule>=1.2.2",
     "streamlit>=1.48.0",
+    "scikit-learn>=1.8.0",
 ]

tests/test_analysis.py

@@ -0,0 +1,195 @@
+"""Tests for analysis modules: political_axis, trajectory, clustering."""
+
+import json
+import numpy as np
+import pytest
+
+duckdb = pytest.importorskip("duckdb")
+
+
+# ── Helpers ──────────────────────────────────────────────────────────────────
+
+
+def _setup_svd_vectors(db_path: str, window_ids_mp_vecs: dict):
+    """Insert synthetic MP SVD vectors into svd_vectors table.
+
+    window_ids_mp_vecs: {window_id: {mp_name: np.ndarray}}
+    """
+    conn = duckdb.connect(db_path)
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS svd_vectors (
+            id INTEGER,
+            window_id TEXT,
+            entity_type TEXT,
+            entity_id TEXT,
+            vector JSON,
+            model TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """
+    )
+    for wid, mp_vecs in window_ids_mp_vecs.items():
+        for mp_name, vec in mp_vecs.items():
+            conn.execute(
+                "INSERT INTO svd_vectors (window_id, entity_type, entity_id, vector, model) VALUES (?, 'mp', ?, ?, 'test')",
+                (wid, mp_name, json.dumps(vec.tolist())),
+            )
+    conn.close()
+
+
+def _setup_mp_metadata(db_path: str, mp_party: dict):
+    """Insert synthetic MP metadata rows."""
+    conn = duckdb.connect(db_path)
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS mp_metadata (
+            mp_name TEXT,
+            party TEXT,
+            van DATE,
+            tot_en_met DATE,
+            persoon_id TEXT
+        )
+        """
+    )
+    for mp_name, party in mp_party.items():
+        conn.execute(
+            "INSERT INTO mp_metadata (mp_name, party) VALUES (?, ?)",
+            (mp_name, party),
+        )
+    conn.close()
+
+
+# ── political_axis ────────────────────────────────────────────────────────────
+
+
+class TestPoliticalAxis:
+    def test_pca_axis_basic(self, tmp_path):
+        np.random.seed(42)
+        db_path = str(tmp_path / "test.db")
+        n_mps, k = 20, 5
+
+        # Create a low-rank set of MP vectors (they should have a clear first PC)
+        vecs = np.random.randn(n_mps, k)
+        mp_names = [f"MP_{i}" for i in range(n_mps)]
+        _setup_svd_vectors(
+            db_path, {"2024-Q1": {mp_names[i]: vecs[i] for i in range(n_mps)}}
+        )
+
+        from analysis.political_axis import compute_pca_axis
+
+        scores = compute_pca_axis(db_path, "2024-Q1")
+        assert len(scores) == n_mps
+        assert all(isinstance(v, float) for v in scores.values())
+        # Scores should have non-trivial variance
+        vals = list(scores.values())
+        assert np.std(vals) > 0.0
+
+    def test_pca_axis_too_few_mps(self, tmp_path):
+        db_path = str(tmp_path / "test.db")
+        _setup_svd_vectors(db_path, {"w1": {"MP_A": np.array([1.0, 0.0])}})
+
+        from analysis.political_axis import compute_pca_axis
+
+        scores = compute_pca_axis(db_path, "w1")
+        assert scores == {}
+
+    def test_anchor_axis_basic(self, tmp_path):
+        db_path = str(tmp_path / "test.db")
+        # Two clusters clearly separated on dim 0
+        left_vec = np.array([-2.0, 0.0, 0.0])
+        right_vec = np.array([2.0, 0.0, 0.0])
+        mp_vecs = {
+            "Left_A": left_vec + np.array([0.1, 0.0, 0.0]),
+            "Left_B": left_vec - np.array([0.1, 0.0, 0.0]),
+            "Right_A": right_vec + np.array([0.1, 0.0, 0.0]),
+            "Right_B": right_vec - np.array([0.1, 0.0, 0.0]),
+            "Centre": np.array([0.0, 0.0, 0.0]),
+        }
+        _setup_svd_vectors(db_path, {"w1": mp_vecs})
+        _setup_mp_metadata(
+            db_path,
+            {
+                "Left_A": "SP",
+                "Left_B": "SP",
+                "Right_A": "VVD",
+                "Right_B": "VVD",
+                "Centre": "D66",
+            },
+        )
+
+        from analysis.political_axis import compute_anchor_axis
+
+        scores = compute_anchor_axis(
+            db_path, "w1", left_parties=["SP"], right_parties=["VVD"]
+        )
+        assert len(scores) == 5
+        # Left MPs should have negative scores, Right MPs positive
+        assert scores["Left_A"] < scores["Right_A"]
+        assert scores["Left_B"] < scores["Right_B"]
+
+
+# ── trajectory ───────────────────────────────────────────────────────────────
+
+
+class TestTrajectory:
+    def test_basic_trajectory(self, tmp_path):
+        np.random.seed(0)
+        db_path = str(tmp_path / "test.db")
+
+        vec_w1 = {"MP_A": np.array([1.0, 0.0]), "MP_B": np.array([0.0, 1.0])}
+        vec_w2 = {
+            "MP_A": np.array([1.5, 0.5]),
+            "MP_B": np.array([0.0, 1.0]),
+            "MP_C": np.array([2.0, 2.0]),
+        }
+        _setup_svd_vectors(db_path, {"2024-Q1": vec_w1, "2024-Q2": vec_w2})
+
+        from analysis.trajectory import compute_trajectories, top_drifters
+
+        traj = compute_trajectories(db_path)
+
+        # Only MPs appearing in >= 2 windows
+        assert "MP_A" in traj
+        assert "MP_B" in traj
+        assert "MP_C" not in traj  # only in one window
+
+        assert len(traj["MP_A"]["drift"]) == 1
+        assert traj["MP_A"]["total_drift"] > 0.0
+
+        # MP_B didn't move — drift should be 0
+        assert traj["MP_B"]["total_drift"] == pytest.approx(0.0)
+
+        drifters = top_drifters(traj, n=5)
+        assert drifters[0]["mp_name"] == "MP_A"
+
+    def test_fewer_than_2_windows(self, tmp_path):
+        db_path = str(tmp_path / "test.db")
+        _setup_svd_vectors(db_path, {"2024-Q1": {"MP_A": np.array([1.0, 2.0])}})
+
+        from analysis.trajectory import compute_trajectories
+
+        traj = compute_trajectories(db_path)
+        assert traj == {}
+
+
+# ── clustering ────────────────────────────────────────────────────────────────
+
+
+class TestClustering:
+    def test_cluster_kmeans_basic(self):
+        from analysis.clustering import cluster_kmeans
+        import numpy as np
+
+        coords = np.random.randn(20, 2)
+        labels = cluster_kmeans(coords, n_clusters=3)
+        assert len(labels) == 20
+        assert set(labels).issubset({0, 1, 2})
+
+    def test_cluster_kmeans_fewer_points_than_clusters(self):
+        from analysis.clustering import cluster_kmeans
+
+        coords = np.array([[0.0, 0.0], [1.0, 1.0]])
+        labels = cluster_kmeans(coords, n_clusters=5)
+        # Should not crash; n_clusters clamped to len(coords)
+        assert len(labels) == 2

tests/test_run_pipeline.py

@@ -0,0 +1,113 @@
+"""Tests for pipeline/run_pipeline.py"""
+
+import argparse
+import sys
+import pytest
+
+from pipeline.run_pipeline import _generate_windows, build_parser, run
+from datetime import date
+
+
+def test_generate_windows_quarterly():
+    start = date(2024, 1, 1)
+    end = date(2024, 12, 31)
+    windows = _generate_windows(start, end, "quarterly")
+
+    assert len(windows) == 4
+    ids = [w[0] for w in windows]
+    assert ids == ["2024-Q1", "2024-Q2", "2024-Q3", "2024-Q4"]
+
+    # Q1 bounds
+    assert windows[0][1] == "2024-01-01"
+    assert windows[0][2] == "2024-03-31"
+
+    # Q4 bounds
+    assert windows[3][1] == "2024-10-01"
+    assert windows[3][2] == "2024-12-31"
+
+
+def test_generate_windows_annual():
+    start = date(2022, 6, 1)
+    end = date(2024, 3, 31)
+    windows = _generate_windows(start, end, "annual")
+
+    assert len(windows) == 3
+    ids = [w[0] for w in windows]
+    assert ids == ["2022", "2023", "2024"]
+
+    # 2024 should end at end_date, not Dec 31
+    assert windows[2][2] == "2024-03-31"
+
+
+def test_generate_windows_mid_quarter_start():
+    """Starting in the middle of Q2 should still produce a full Q2 window."""
+    start = date(2024, 5, 15)
+    end = date(2024, 9, 30)
+    windows = _generate_windows(start, end, "quarterly")
+
+    ids = [w[0] for w in windows]
+    assert "2024-Q2" in ids
+    assert "2024-Q3" in ids
+
+
+def test_build_parser_defaults():
+    parser = build_parser()
+    args = parser.parse_args([])
+    assert args.db_path == "data/motions.db"
+    assert args.window_size == "quarterly"
+    assert args.svd_k == 50
+    assert args.dry_run is False
+
+
+def test_run_dry_run(tmp_path, monkeypatch):
+    """Dry-run should log actions and return 0 without touching the DB."""
+    db_path = str(tmp_path / "motions.db")
+
+    # Create minimal DB so MotionDatabase initialises
+    from database import MotionDatabase
+
+    MotionDatabase(db_path)
+
+    args = argparse.Namespace(
+        db_path=db_path,
+        start_date="2024-01-01",
+        end_date="2024-03-31",
+        window_size="quarterly",
+        svd_k=10,
+        text_model=None,
+        skip_metadata=False,
+        skip_extract=False,
+        skip_svd=False,
+        skip_text=False,
+        skip_fusion=False,
+        dry_run=True,
+    )
+
+    exit_code = run(args)
+    assert exit_code == 0
+
+
+def test_run_skip_all(tmp_path):
+    """Skipping all phases should still return 0."""
+    db_path = str(tmp_path / "motions.db")
+    from database import MotionDatabase
+
+    MotionDatabase(db_path)
+
+    args = argparse.Namespace(
+        db_path=db_path,
+        start_date="2024-01-01",
+        end_date="2024-03-31",
+        window_size="quarterly",
+        svd_k=10,
+        text_model=None,
+        skip_metadata=True,
+        skip_extract=True,
+        skip_svd=True,
+        skip_text=True,
+        skip_fusion=True,
+        dry_run=False,
+    )
+
+    exit_code = run(args)
+    assert exit_code == 0

uv.lock

@@ -1056,6 +1056,7 @@ dependencies = [
     { name = "pytest" },
     { name = "requests" },
     { name = "schedule" },
+    { name = "scikit-learn" },
     { name = "scipy" },
     { name = "streamlit" },
     { name = "umap-learn" },
@@ -1070,6 +1071,7 @@ requires-dist = [
     { name = "pytest", specifier = ">=9.0.2" },
     { name = "requests", specifier = ">=2.32.4" },
     { name = "schedule", specifier = ">=1.2.2" },
+    { name = "scikit-learn", specifier = ">=1.8.0" },
     { name = "scipy", specifier = ">=1.11" },
     { name = "streamlit", specifier = ">=1.48.0" },
     { name = "umap-learn", specifier = ">=0.5" },