feat(pipeline): add orchestrator CLI, analysis modules, and ActorFractie ingestion

- pipeline/run_pipeline.py: CLI orchestrator for all 5 pipeline phases with
  --dry-run, --skip-*, --window-size, --svd-k, --start/end-date flags
- analysis/{political_axis,trajectory,clustering,visualize}.py: PCA/anchor
  ideological axis, MP drift trajectories, UMAP + KMeans clustering, Plotly HTML output
- api_client.py: capture ActorFractie per individual MP vote (comma in ActorNaam)
  into mp_vote_parties dict on each motion
- database.insert_motion: auto-insert mp_votes rows with party affiliation for
  newly ingested motions when mp_vote_parties is present
- Add scikit-learn to pyproject.toml for KMeans clustering
- tests/test_run_pipeline.py: window generation, dry-run, skip-all paths
- tests/test_analysis.py: PCA axis, anchor axis, trajectory drift, KMeans

Ref: thoughts/shared/plans/2026-03-21-parliamentary-embedding-pipeline-plan.md

analysis/__init__.py

@@ -0,0 +1,8 @@
+"""Analysis modules for the parliamentary embedding pipeline.
+
+Modules:
+    political_axis  — project MP SVD vectors onto ideological axis
+    trajectory      — compute MP drift across aligned windows
+    clustering      — UMAP dimensionality reduction + cluster labelling
+    visualize       — Plotly interactive plots (outputs self-contained HTML)
+"""

analysis/clustering.py

@@ -0,0 +1,130 @@
+"""clustering.py — UMAP dimensionality reduction on fused embeddings.
+
+Reduces fused motion embeddings to 2D (or 3D) for visualisation,
+and optionally labels clusters using KMeans.
+
+Requires: umap-learn, scikit-learn (for KMeans)
+"""
+
+import json
+import logging
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+import duckdb
+
+_logger = logging.getLogger(__name__)
+
+
+def _load_fused_vectors(
+    db_path: str, window_id: Optional[str] = None
+) -> Tuple[List[int], List[str], np.ndarray]:
+    """Load fused embeddings from the DB.
+
+    Returns (motion_ids, window_ids, matrix).
+    Optionally filter by window_id.
+    """
+    conn = duckdb.connect(db_path)
+    if window_id:
+        rows = conn.execute(
+            "SELECT motion_id, window_id, vector FROM fused_embeddings WHERE window_id = ?",
+            (window_id,),
+        ).fetchall()
+    else:
+        rows = conn.execute(
+            "SELECT motion_id, window_id, vector FROM fused_embeddings ORDER BY window_id, motion_id"
+        ).fetchall()
+    conn.close()
+
+    motion_ids, window_ids, vectors = [], [], []
+    for motion_id, wid, vec_json in rows:
+        try:
+            vec = json.loads(vec_json)
+            motion_ids.append(int(motion_id))
+            window_ids.append(wid)
+            vectors.append(vec)
+        except Exception:
+            _logger.warning("Could not parse fused vector for motion %s", motion_id)
+
+    if not vectors:
+        return [], [], np.zeros((0, 0))
+
+    # Pad to common length if needed (shouldn't happen if pipeline is consistent)
+    max_len = max(len(v) for v in vectors)
+    mat = np.zeros((len(vectors), max_len), dtype=float)
+    for i, v in enumerate(vectors):
+        mat[i, : len(v)] = v
+
+    return motion_ids, window_ids, mat
+
+
+def run_umap(
+    db_path: str,
+    window_id: Optional[str] = None,
+    n_components: int = 2,
+    n_neighbors: int = 15,
+    min_dist: float = 0.1,
+    random_state: int = 42,
+) -> Dict:
+    """Run UMAP on fused embeddings and return 2D/3D coordinates.
+
+    Returns:
+        {
+          "motion_ids": [...],
+          "window_ids": [...],
+          "coords": [[x, y], ...],   # or [x, y, z] if n_components=3
+          "n_components": int,
+        }
+    """
+    try:
+        import umap
+    except ImportError:
+        _logger.error("umap-learn is not installed; cannot run UMAP")
+        return {}
+
+    motion_ids, window_ids, mat = _load_fused_vectors(db_path, window_id)
+    if mat.size == 0:
+        _logger.warning("No fused embeddings found for window_id=%s", window_id)
+        return {}
+
+    if mat.shape[0] < n_neighbors + 1:
+        # UMAP requires at least n_neighbors+1 samples
+        n_neighbors = max(2, mat.shape[0] - 1)
+        _logger.warning(
+            "Reduced n_neighbors to %d due to small dataset (%d samples)",
+            n_neighbors,
+            mat.shape[0],
+        )
+
+    reducer = umap.UMAP(
+        n_components=n_components,
+        n_neighbors=n_neighbors,
+        min_dist=min_dist,
+        random_state=random_state,
+    )
+    coords = reducer.fit_transform(mat)
+
+    return {
+        "motion_ids": motion_ids,
+        "window_ids": window_ids,
+        "coords": coords.tolist(),
+        "n_components": n_components,
+    }
+
+
+def cluster_kmeans(
+    coords: np.ndarray, n_clusters: int = 8, random_state: int = 42
+) -> np.ndarray:
+    """Run KMeans on 2D/3D UMAP coordinates.
+
+    Returns array of integer cluster labels (length = len(coords)).
+    """
+    try:
+        from sklearn.cluster import KMeans
+    except ImportError:
+        _logger.error("scikit-learn is not installed; cannot run KMeans")
+        return np.zeros(len(coords), dtype=int)
+
+    n_clusters = min(n_clusters, len(coords))
+    km = KMeans(n_clusters=n_clusters, random_state=random_state, n_init="auto")
+    return km.fit_predict(coords)

analysis/political_axis.py

@@ -0,0 +1,125 @@
+"""political_axis.py — Project MP SVD vectors onto an ideological axis.
+
+Two modes:
+  1. PCA mode (default): compute the first principal component of all MP SVD
+     vectors for a window and project each MP onto it.  The sign is arbitrary
+     but consistent within a window.
+
+  2. Anchor mode: define the axis as the vector from the centroid of
+     ``left_parties`` to the centroid of ``right_parties``.  Project all MPs
+     onto this normalised anchor axis.
+
+Both modes return a dict mapping mp_name → scalar score for the given window.
+"""
+
+import json
+import logging
+from typing import Dict, List, Optional
+
+import numpy as np
+import duckdb
+
+_logger = logging.getLogger(__name__)
+
+
+def _load_mp_svd_vectors(db_path: str, window_id: str) -> Dict[str, np.ndarray]:
+    """Load all MP SVD vectors for a window from svd_vectors table."""
+    conn = duckdb.connect(db_path)
+    rows = conn.execute(
+        "SELECT entity_id, vector FROM svd_vectors WHERE window_id = ? AND entity_type = 'mp'",
+        (window_id,),
+    ).fetchall()
+    conn.close()
+
+    result = {}
+    for mp_name, vec_json in rows:
+        try:
+            result[mp_name] = np.array(json.loads(vec_json), dtype=float)
+        except Exception:
+            _logger.warning("Could not parse SVD vector for MP %s", mp_name)
+    return result
+
+
+def compute_pca_axis(db_path: str, window_id: str) -> Dict[str, float]:
+    """Project MP SVD vectors onto their first principal component.
+
+    Returns {mp_name: score}.  Returns empty dict if fewer than 2 MPs.
+    """
+    mp_vecs = _load_mp_svd_vectors(db_path, window_id)
+    if len(mp_vecs) < 2:
+        _logger.warning(
+            "window %s has only %d MPs; skipping PCA axis", window_id, len(mp_vecs)
+        )
+        return {}
+
+    names = list(mp_vecs.keys())
+    mat = np.vstack([mp_vecs[n] for n in names])  # (n_mps, k)
+
+    # Centre
+    mat_centred = mat - mat.mean(axis=0)
+
+    # First PC via SVD
+    try:
+        _, _, Vt = np.linalg.svd(mat_centred, full_matrices=False)
+        axis = Vt[0]  # (k,)
+    except np.linalg.LinAlgError:
+        _logger.exception("SVD failed in compute_pca_axis for window %s", window_id)
+        return {}
+
+    projections = mat_centred.dot(axis)
+    return {name: float(score) for name, score in zip(names, projections)}
+
+
+def compute_anchor_axis(
+    db_path: str,
+    window_id: str,
+    left_parties: List[str],
+    right_parties: List[str],
+) -> Dict[str, float]:
+    """Project MP SVD vectors onto a left↔right anchor axis.
+
+    The axis runs from the centroid of ``left_parties`` to the centroid of
+    ``right_parties``.  Positive scores are toward the right.
+
+    Returns {mp_name: score}.
+    """
+    mp_vecs = _load_mp_svd_vectors(db_path, window_id)
+    if not mp_vecs:
+        return {}
+
+    # Load party affiliation for this window from mp_metadata
+    conn = duckdb.connect(db_path)
+    rows = conn.execute("SELECT mp_name, party FROM mp_metadata").fetchall()
+    conn.close()
+    party_of = {mp: party for mp, party in rows}
+
+    left_vecs = [
+        mp_vecs[mp]
+        for mp, party in party_of.items()
+        if party in left_parties and mp in mp_vecs
+    ]
+    right_vecs = [
+        mp_vecs[mp]
+        for mp, party in party_of.items()
+        if party in right_parties and mp in mp_vecs
+    ]
+
+    if not left_vecs or not right_vecs:
+        _logger.warning(
+            "window %s: insufficient anchor parties (left=%d, right=%d)",
+            window_id,
+            len(left_vecs),
+            len(right_vecs),
+        )
+        return {}
+
+    left_centroid = np.mean(left_vecs, axis=0)
+    right_centroid = np.mean(right_vecs, axis=0)
+    axis = right_centroid - left_centroid
+    norm = np.linalg.norm(axis)
+    if norm < 1e-10:
+        _logger.warning("Anchor axis has near-zero norm for window %s", window_id)
+        return {}
+    axis = axis / norm
+
+    return {name: float(np.dot(vec, axis)) for name, vec in mp_vecs.items()}

analysis/trajectory.py

@@ -0,0 +1,123 @@
+"""trajectory.py — Compute MP political drift across aligned time windows.
+
+For each MP that appears in multiple windows, computes:
+  - The aligned SVD vector per window
+  - The Euclidean distance between consecutive windows (drift)
+  - Total cumulative drift
+
+Returns a dict keyed by mp_name containing per-window positions and drift scores.
+"""
+
+import json
+import logging
+from typing import Dict, List, Optional
+
+import numpy as np
+import duckdb
+
+_logger = logging.getLogger(__name__)
+
+
+def _load_window_ids(db_path: str) -> List[str]:
+    """Return all distinct window IDs from svd_vectors, in lexicographic order."""
+    conn = duckdb.connect(db_path)
+    rows = conn.execute(
+        "SELECT DISTINCT window_id FROM svd_vectors WHERE entity_type = 'mp' ORDER BY window_id"
+    ).fetchall()
+    conn.close()
+    return [r[0] for r in rows]
+
+
+def _load_mp_vectors_for_window(db_path: str, window_id: str) -> Dict[str, np.ndarray]:
+    conn = duckdb.connect(db_path)
+    rows = conn.execute(
+        "SELECT entity_id, vector FROM svd_vectors WHERE window_id = ? AND entity_type = 'mp'",
+        (window_id,),
+    ).fetchall()
+    conn.close()
+    result = {}
+    for mp_name, vec_json in rows:
+        try:
+            result[mp_name] = np.array(json.loads(vec_json), dtype=float)
+        except Exception:
+            _logger.warning(
+                "Could not parse vector for MP %s window %s", mp_name, window_id
+            )
+    return result
+
+
+def compute_trajectories(
+    db_path: str,
+    window_ids: Optional[List[str]] = None,
+) -> Dict[str, Dict]:
+    """Compute per-MP trajectories across windows.
+
+    Returns:
+        {
+          mp_name: {
+            "windows": [window_id, ...],
+            "vectors": [[...], ...],   # one vector per window
+            "drift": [float, ...],     # consecutive Euclidean distances
+            "total_drift": float,
+          }
+        }
+    Only MPs present in at least 2 windows are included.
+    """
+    if window_ids is None:
+        window_ids = _load_window_ids(db_path)
+
+    if len(window_ids) < 2:
+        _logger.info("Fewer than 2 windows — no trajectories to compute")
+        return {}
+
+    # Collect per-window vectors for each MP
+    mp_data: Dict[str, Dict] = {}
+
+    for wid in window_ids:
+        vecs = _load_mp_vectors_for_window(db_path, wid)
+        for mp_name, vec in vecs.items():
+            if mp_name not in mp_data:
+                mp_data[mp_name] = {"windows": [], "vectors": []}
+            mp_data[mp_name]["windows"].append(wid)
+            mp_data[mp_name]["vectors"].append(vec)
+
+    # Compute drift for MPs with >= 2 windows
+    result = {}
+    for mp_name, data in mp_data.items():
+        if len(data["windows"]) < 2:
+            continue
+        vecs = data["vectors"]
+        drifts = [
+            float(np.linalg.norm(vecs[i + 1] - vecs[i])) for i in range(len(vecs) - 1)
+        ]
+        result[mp_name] = {
+            "windows": data["windows"],
+            "vectors": [v.tolist() for v in vecs],
+            "drift": drifts,
+            "total_drift": float(sum(drifts)),
+        }
+
+    _logger.info(
+        "Trajectories computed for %d MPs across %d windows",
+        len(result),
+        len(window_ids),
+    )
+    return result
+
+
+def top_drifters(trajectories: Dict[str, Dict], n: int = 10) -> List[Dict]:
+    """Return the top-n MPs by total drift, sorted descending.
+
+    Each entry: {"mp_name": ..., "total_drift": ..., "windows": [...]}
+    """
+    ranked = sorted(
+        trajectories.items(), key=lambda kv: kv[1]["total_drift"], reverse=True
+    )
+    return [
+        {
+            "mp_name": mp,
+            "total_drift": data["total_drift"],
+            "windows": data["windows"],
+        }
+        for mp, data in ranked[:n]
+    ]

analysis/visualize.py

@@ -0,0 +1,163 @@
+"""visualize.py — Plotly interactive plots for parliamentary embeddings.
+
+Produces self-contained HTML files.
+
+Functions:
+  plot_umap_scatter   — 2D scatter of fused motion embeddings, coloured by cluster
+  plot_mp_trajectory  — Line plot of MP drift across windows
+  plot_political_axis — Bar chart of MP scores on the ideological axis
+"""
+
+import logging
+from typing import Dict, List, Optional
+
+import numpy as np
+
+_logger = logging.getLogger(__name__)
+
+
+def _require_plotly():
+    try:
+        import plotly.graph_objects as go
+        import plotly.express as px
+
+        return go, px
+    except ImportError:
+        raise ImportError("plotly is not installed. Install it with: uv add plotly")
+
+
+def plot_umap_scatter(
+    motion_ids: List[int],
+    coords: List[List[float]],
+    labels: Optional[List[int]] = None,
+    window_id: Optional[str] = None,
+    output_path: str = "analysis_umap.html",
+) -> str:
+    """Produce a 2D scatter plot of UMAP-reduced fused embeddings.
+
+    Args:
+        motion_ids:  Motion IDs (used as hover labels)
+        coords:      List of [x, y] coordinates
+        labels:      Optional cluster labels (integer per motion)
+        window_id:   Window label for the plot title
+        output_path: Where to write the self-contained HTML
+
+    Returns the output_path on success.
+    """
+    go, px = _require_plotly()
+
+    xs = [c[0] for c in coords]
+    ys = [c[1] for c in coords]
+    color = labels if labels is not None else [0] * len(motion_ids)
+    title = f"UMAP — fused motion embeddings" + (f" ({window_id})" if window_id else "")
+
+    fig = px.scatter(
+        x=xs,
+        y=ys,
+        color=[str(c) for c in color],
+        hover_name=[str(mid) for mid in motion_ids],
+        title=title,
+        labels={"x": "UMAP-1", "y": "UMAP-2", "color": "Cluster"},
+    )
+    fig.write_html(output_path, include_plotlyjs="cdn")
+    _logger.info("UMAP scatter written to %s", output_path)
+    return output_path
+
+
+def plot_mp_trajectory(
+    trajectories: Dict[str, Dict],
+    mp_names: Optional[List[str]] = None,
+    output_path: str = "analysis_trajectory.html",
+) -> str:
+    """Line plot of MP drift across time windows.
+
+    Args:
+        trajectories: Output of analysis.trajectory.compute_trajectories()
+        mp_names:     Subset of MPs to plot (default: all)
+        output_path:  Output HTML file path
+
+    Returns the output_path on success.
+    """
+    go, px = _require_plotly()
+
+    if mp_names is None:
+        mp_names = list(trajectories.keys())
+
+    fig = go.Figure()
+
+    for mp in mp_names:
+        if mp not in trajectories:
+            continue
+        data = trajectories[mp]
+        windows = data["windows"]
+        drifts_cumulative = [0.0] + list(np.cumsum(data["drift"]))
+        # Plot cumulative drift per window transition
+        x_labels = windows[: len(drifts_cumulative)]
+        fig.add_trace(
+            go.Scatter(
+                x=x_labels,
+                y=drifts_cumulative,
+                mode="lines+markers",
+                name=mp,
+            )
+        )
+
+    fig.update_layout(
+        title="MP Political Drift Over Time (Cumulative)",
+        xaxis_title="Window",
+        yaxis_title="Cumulative Drift",
+    )
+    fig.write_html(output_path, include_plotlyjs="cdn")
+    _logger.info("Trajectory plot written to %s", output_path)
+    return output_path
+
+
+def plot_political_axis(
+    scores: Dict[str, float],
+    party_of: Optional[Dict[str, str]] = None,
+    window_id: Optional[str] = None,
+    n_top: int = 30,
+    output_path: str = "analysis_political_axis.html",
+) -> str:
+    """Horizontal bar chart of MP scores on the ideological axis.
+
+    Args:
+        scores:      {mp_name: score} from political_axis module
+        party_of:    Optional {mp_name: party} for colour-coding
+        window_id:   Window label for the title
+        n_top:       Show only the top/bottom n MPs by score
+        output_path: Output HTML path
+
+    Returns the output_path on success.
+    """
+    go, px = _require_plotly()
+
+    # Sort by score
+    sorted_items = sorted(scores.items(), key=lambda kv: kv[1])
+
+    # Take n_top from each end if list is large
+    if len(sorted_items) > 2 * n_top:
+        sorted_items = sorted_items[:n_top] + sorted_items[-n_top:]
+
+    names = [item[0] for item in sorted_items]
+    vals = [item[1] for item in sorted_items]
+    colors = (
+        [party_of.get(n, "Unknown") for n in names]
+        if party_of
+        else ["Unknown"] * len(names)
+    )
+
+    title = "MP Ideological Axis Score" + (f" ({window_id})" if window_id else "")
+
+    fig = px.bar(
+        x=vals,
+        y=names,
+        color=colors,
+        orientation="h",
+        title=title,
+        labels={"x": "Score (← left — right →)", "y": "MP", "color": "Party"},
+    )
+    fig.update_layout(yaxis={"categoryorder": "total ascending"})
+    fig.write_html(output_path, include_plotlyjs="cdn")
+    _logger.info("Political axis chart written to %s", output_path)
+    return output_path