feat: complete parliamentary embedding pipeline with full historical coverage
- Add fused (SVD + text) embedding pipeline for annual windows 2016-2026 - Fix store_fused_embedding duplicate bug: DELETE before INSERT (idempotent) - Add --text-batch-size CLI flag to run_pipeline.py (default 200) - Add explicit --start-date/--end-date to download_past_year.py - Backfill mp_votes for all motions (party-level votes, 111k new rows) - Add similarity cache recompute: 212k rows across 9 annual windows - Improve ai_provider retry logic, text_pipeline batching - Improve analysis/political_axis PCA handling and visualizations - Add diagnostic/utility scripts: compare_svd, generate_compass, inspect_axis, etc. - Untrack data/motions.db (3.6GB binary), add to .gitignore with outputs/ - Update continuity ledger with full session state
This commit is contained in:
@@ -161,6 +161,11 @@ def compute_2d_axes(
|
||||
to load and align windows so the returned coordinates are consistent
|
||||
across windows.
|
||||
"""
|
||||
# Import trajectory helper at runtime so tests can monkeypatch sys.modules
|
||||
import importlib
|
||||
|
||||
_trajectory = importlib.import_module("analysis.trajectory")
|
||||
|
||||
if window_ids is None:
|
||||
window_ids = _trajectory._load_window_ids(db_path)
|
||||
|
||||
@@ -238,6 +243,77 @@ def compute_2d_axes(
|
||||
"pca_residual_used": bool(pca_residual or evr1 > 0.85),
|
||||
}
|
||||
|
||||
# Ensure consistent left/right and progressive/conservative orientation
|
||||
# by checking canonical party centroids and flipping axis signs if needed.
|
||||
try:
|
||||
right_parties = {"PVV", "VVD", "FVD", "BBB", "JA21"}
|
||||
left_parties = {"SP", "PvdA", "GroenLinks", "GroenLinks-PvdA", "DENK"}
|
||||
cons_parties = {"PVV", "VVD", "FVD", "CDA", "SGP", "BBB", "JA21"}
|
||||
prog_parties = {
|
||||
"GroenLinks",
|
||||
"PvdA",
|
||||
"PvdD",
|
||||
"SP",
|
||||
"GroenLinks-PvdA",
|
||||
"DENK",
|
||||
}
|
||||
|
||||
# Build mapping of entity -> vector from stacked matrix M
|
||||
ent_to_vec = {ent: vec for (wid, ent), vec in zip(entity_index, M)}
|
||||
|
||||
def _centroid_for_party_set(party_set):
|
||||
vecs = []
|
||||
for p in party_set:
|
||||
if p in ent_to_vec:
|
||||
vecs.append(ent_to_vec[p])
|
||||
try:
|
||||
conn = duckdb.connect(db_path)
|
||||
rows = conn.execute(
|
||||
"SELECT mp_name, party FROM mp_metadata"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
except Exception:
|
||||
rows = []
|
||||
for mp_name, party in rows:
|
||||
if party in party_set and mp_name in ent_to_vec:
|
||||
vecs.append(ent_to_vec[mp_name])
|
||||
if not vecs:
|
||||
return None
|
||||
return np.mean(np.vstack(vecs), axis=0)
|
||||
|
||||
# X-axis: left vs right
|
||||
left_cent = _centroid_for_party_set(left_parties)
|
||||
right_cent = _centroid_for_party_set(right_parties)
|
||||
if left_cent is not None and right_cent is not None:
|
||||
left_proj = float(np.dot(left_cent - M.mean(axis=0), comp1_hat))
|
||||
right_proj = float(np.dot(right_cent - M.mean(axis=0), comp1_hat))
|
||||
if right_proj < left_proj:
|
||||
_logger.info(
|
||||
"Flipping PCA x-axis to match canonical left/right orientation (right_proj=%.3f left_proj=%.3f)",
|
||||
right_proj,
|
||||
left_proj,
|
||||
)
|
||||
axes["x_axis"] = -axes["x_axis"]
|
||||
|
||||
# Y-axis: progressive vs conservative — prefer positive = conservative
|
||||
prog_cent = _centroid_for_party_set(prog_parties)
|
||||
cons_cent = _centroid_for_party_set(cons_parties)
|
||||
if prog_cent is not None and cons_cent is not None:
|
||||
prog_proj = float(np.dot(prog_cent - M.mean(axis=0), comp2_hat))
|
||||
cons_proj = float(np.dot(cons_cent - M.mean(axis=0), comp2_hat))
|
||||
# We want positive Y to mean 'progressive'. If the progressive
|
||||
# centroid currently projects lower than the conservative centroid,
|
||||
# flip the sign so progressive > conservative.
|
||||
if prog_proj < cons_proj:
|
||||
_logger.info(
|
||||
"Flipping PCA y-axis so positive Y corresponds to progressive (prog_proj=%.3f cons_proj=%.3f)",
|
||||
prog_proj,
|
||||
cons_proj,
|
||||
)
|
||||
axes["y_axis"] = -axes["y_axis"]
|
||||
except Exception:
|
||||
_logger.debug("Could not auto-orient PCA axes; leaving signs as-is")
|
||||
|
||||
# warn if PCA is effectively 1-D
|
||||
if evr1 > 0.85 and not pca_residual:
|
||||
_logger.warning(
|
||||
|
||||
+127
-12
@@ -27,6 +27,58 @@ def _require_plotly():
|
||||
raise ImportError("plotly is not installed. Install it with: uv add plotly")
|
||||
|
||||
|
||||
def _load_party_map(db_path: str = "data/motions.db") -> Dict[str, str]:
|
||||
"""Build a party mapping mp_name -> party.
|
||||
|
||||
Prefers mp_metadata where available; otherwise uses majority-party from mp_votes.
|
||||
Returns a dict of mp_name -> party (strings).
|
||||
"""
|
||||
try:
|
||||
import duckdb
|
||||
except Exception:
|
||||
_logger.debug("duckdb not available when building party map")
|
||||
return {}
|
||||
|
||||
conn = duckdb.connect(db_path)
|
||||
try:
|
||||
# metadata-based mapping
|
||||
rows = conn.execute(
|
||||
"SELECT mp_name, party FROM mp_metadata WHERE party IS NOT NULL"
|
||||
).fetchall()
|
||||
meta_map = {r[0]: r[1] for r in rows}
|
||||
|
||||
# majority-party heuristic from mp_votes
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT mp_name, party, COUNT(*) as n
|
||||
FROM mp_votes
|
||||
WHERE party IS NOT NULL
|
||||
GROUP BY mp_name, party
|
||||
"""
|
||||
).fetchall()
|
||||
counts: Dict[str, List[tuple]] = {}
|
||||
for mp_name, party, n in rows:
|
||||
counts.setdefault(mp_name, []).append((party, n))
|
||||
maj_map: Dict[str, str] = {}
|
||||
for mp_name, arr in counts.items():
|
||||
maj_map[mp_name] = max(arr, key=lambda x: x[1])[0]
|
||||
|
||||
merged = dict(maj_map)
|
||||
# prefer metadata mapping when available
|
||||
merged.update(meta_map)
|
||||
_logger.info(
|
||||
"Built party map: %d from mp_votes majority, %d from mp_metadata",
|
||||
len(maj_map),
|
||||
len(meta_map),
|
||||
)
|
||||
return merged
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def plot_umap_scatter(
|
||||
motion_ids: List[int],
|
||||
coords: List[List[float]],
|
||||
@@ -194,6 +246,7 @@ def plot_political_compass(
|
||||
try:
|
||||
import duckdb # type: ignore
|
||||
|
||||
conn = None
|
||||
try:
|
||||
conn = duckdb.connect(database="data/motions.db", read_only=True)
|
||||
df = conn.execute("SELECT mp_name, party FROM mp_metadata").fetchdf()
|
||||
@@ -206,10 +259,11 @@ def plot_political_compass(
|
||||
len(party_of),
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
except ImportError:
|
||||
_logger.debug("duckdb not installed; proceeding without party mapping")
|
||||
except Exception as e:
|
||||
@@ -221,8 +275,18 @@ def plot_political_compass(
|
||||
scaled_ys = ys
|
||||
if axis_def and y_scale is None:
|
||||
evr = axis_def.get("explained_variance_ratio") if axis_def else None
|
||||
if evr and isinstance(evr, (list, tuple)) and len(evr) >= 2:
|
||||
evr1, evr2 = evr[0], evr[1]
|
||||
# Accept lists/tuples or numpy arrays; avoid ambiguous truth checks
|
||||
evr_list = None
|
||||
if evr is not None:
|
||||
try:
|
||||
evr_list = list(evr)
|
||||
except Exception:
|
||||
try:
|
||||
evr_list = [float(evr)]
|
||||
except Exception:
|
||||
evr_list = None
|
||||
if evr_list is not None and len(evr_list) >= 2:
|
||||
evr1, evr2 = float(evr_list[0]), float(evr_list[1])
|
||||
if evr2 < 1e-6:
|
||||
scale_guess = 1.0
|
||||
else:
|
||||
@@ -237,30 +301,42 @@ def plot_political_compass(
|
||||
elif axis_def and y_scale is not None:
|
||||
scaled_ys = [y * float(y_scale) for y in ys]
|
||||
|
||||
# mark unknowns differently
|
||||
unknown_flags = [1 if parties[i] == "Unknown" else 0 for i in range(len(names))]
|
||||
# mark unknowns differently: use descriptive labels so the legend doesn't
|
||||
# show numeric symbol values like "PVV, 0" when color and symbol combine.
|
||||
unknown_labels = [
|
||||
"Unknown" if parties[i] == "Unknown" else "Known" for i in range(len(names))
|
||||
]
|
||||
|
||||
fig = px.scatter(
|
||||
x=xs,
|
||||
y=scaled_ys,
|
||||
color=parties,
|
||||
symbol=unknown_flags,
|
||||
symbol=unknown_labels,
|
||||
hover_name=names,
|
||||
title=f"Political Compass ({window_id})",
|
||||
labels={
|
||||
"x": "Left ← — → Right",
|
||||
"y": "Progressive ← — → Conservative",
|
||||
"color": "Party",
|
||||
"symbol": "Unknown",
|
||||
"symbol": "Known?",
|
||||
},
|
||||
)
|
||||
fig.update_traces(marker=dict(size=8, opacity=0.85))
|
||||
# annotate explained variance if available
|
||||
if axis_def and axis_def.get("method") == "pca":
|
||||
evr = axis_def.get("explained_variance_ratio")
|
||||
if evr and len(evr) >= 2:
|
||||
evr_list = None
|
||||
if evr is not None:
|
||||
try:
|
||||
evr_list = list(evr)
|
||||
except Exception:
|
||||
try:
|
||||
evr_list = [float(evr)]
|
||||
except Exception:
|
||||
evr_list = None
|
||||
if evr_list is not None and len(evr_list) >= 2:
|
||||
fig.update_layout(
|
||||
title=f"Political Compass ({window_id}) — PCA EVR PC1={evr[0] * 100:.1f}%, PC2={evr[1] * 100:.1f}%"
|
||||
title=f"Political Compass ({window_id}) — PCA EVR PC1={evr_list[0] * 100:.1f}%, PC2={evr_list[1] * 100:.1f}%"
|
||||
)
|
||||
fig.write_html(output_path, include_plotlyjs="cdn")
|
||||
_logger.info("Political compass written to %s", output_path)
|
||||
@@ -309,6 +385,45 @@ def plot_2d_trajectories(
|
||||
)
|
||||
)
|
||||
|
||||
# Add an arrow indicating the final direction (only one arrow per MP to
|
||||
# avoid clutter). Use an annotation with an arrowhead from the penultimate
|
||||
# to the last point and label the endpoint with the MP name.
|
||||
try:
|
||||
if len(xs) >= 2:
|
||||
x0, y0 = xs[-2], ys[-2]
|
||||
x1, y1 = xs[-1], ys[-1]
|
||||
# small style choices — subtle arrow and a short label
|
||||
fig.add_annotation(
|
||||
x=x1,
|
||||
y=y1,
|
||||
ax=x0,
|
||||
ay=y0,
|
||||
xref="x",
|
||||
yref="y",
|
||||
axref="x",
|
||||
ayref="y",
|
||||
showarrow=True,
|
||||
arrowhead=3,
|
||||
arrowsize=1.0,
|
||||
arrowwidth=1.2,
|
||||
arrowcolor="rgba(0,0,0,0.6)",
|
||||
opacity=0.8,
|
||||
)
|
||||
# endpoint label slightly offset to reduce overlap with marker
|
||||
fig.add_annotation(
|
||||
x=x1,
|
||||
y=y1,
|
||||
xref="x",
|
||||
yref="y",
|
||||
text=mp,
|
||||
showarrow=False,
|
||||
xanchor="left",
|
||||
yanchor="bottom",
|
||||
font=dict(size=10, color="rgba(0,0,0,0.8)"),
|
||||
)
|
||||
except Exception:
|
||||
_logger.exception("Failed to add arrow/label for MP %s", mp)
|
||||
|
||||
fig.update_layout(
|
||||
title="MP Trajectories on Political Compass",
|
||||
xaxis_title="Left ← — → Right",
|
||||
|
||||
Reference in New Issue
Block a user