cleanup: archive stale scripts and delete orphaned generate_extra_charts
Archives 8 one-off/backfill/research scripts to scripts/archive/: - compare_svd_exclude_parties.py (diagnostic) - compute_test_batch.py (test utility) - fill_mp_votes_parties.py (backfill) - generate_compass.py (generates to deleted outputs/) - inspect_axis.py (diagnostic) - qa_similarity.py (QA script, references deleted thoughts/ledgers/) - recompute_svd.py (one-off recompute) - semantic_gravity_examples.py (research) Deletes: - generate_extra_charts.py (0 references, generates to deleted outputs/) - tests/test_qa_similarity.py (test for archived script) Adds: - scripts/archive/README.md explaining archive purpose - docs/plans/2026-05-01-001-scripts-audit-cleanup-plan.md
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
# Archived scripts
|
||||
#
|
||||
# These scripts are preserved for reference but are no longer actively
|
||||
# maintained or run. They include one-off diagnostics, backfill utilities,
|
||||
# and research scripts from early pipeline development.
|
||||
#
|
||||
# Git history preserves everything; this directory is just a convenience.
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Compare PCA axes with and without party-level vectors present.
|
||||
|
||||
Generates diagnostics and HTML plots (when plotly available) into outputs/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, List
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if ROOT not in sys.path:
|
||||
sys.path.insert(0, ROOT)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger("compare_svd_exclude_parties")
|
||||
|
||||
|
||||
def main(argv: List[str] | None = None):
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--db", default="data/motions.db")
|
||||
p.add_argument("--out", default="outputs")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
|
||||
try:
|
||||
from analysis import trajectory as traj
|
||||
from analysis.visualize import (
|
||||
_load_party_map,
|
||||
plot_political_compass,
|
||||
plot_2d_trajectories,
|
||||
)
|
||||
import numpy as np
|
||||
except Exception as e:
|
||||
logger.exception("Failed to import analysis modules: %s", e)
|
||||
raise
|
||||
|
||||
window_ids = traj._load_window_ids(args.db)
|
||||
if not window_ids:
|
||||
logger.error("No SVD windows found")
|
||||
return 1
|
||||
latest = sorted(window_ids)[-1]
|
||||
|
||||
# load raw vectors for latest window
|
||||
conn = None
|
||||
try:
|
||||
# build party name set from mp_metadata
|
||||
import duckdb
|
||||
|
||||
conn = duckdb.connect(args.db)
|
||||
rows = conn.execute(
|
||||
"SELECT DISTINCT party FROM mp_metadata WHERE party IS NOT NULL"
|
||||
).fetchall()
|
||||
party_names = set(r[0] for r in rows if r[0])
|
||||
finally:
|
||||
if conn:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
raw = traj._load_mp_vectors_for_window(args.db, latest)
|
||||
# group by vector JSON-like key
|
||||
groups: Dict[str, List[str]] = {}
|
||||
for ent, vec in raw.items():
|
||||
key = tuple([round(float(x), 8) for x in vec.tolist()])
|
||||
groups.setdefault(str(key), []).append(ent)
|
||||
|
||||
group_list = sorted(groups.items(), key=lambda kv: len(kv[1]), reverse=True)
|
||||
|
||||
top_groups = [(len(v), v[:8]) for k, v in group_list[:20]]
|
||||
logger.info("Top duplicate groups (count, sample entities): %s", top_groups)
|
||||
|
||||
# entities that are party names
|
||||
party_entities = [ent for ent in raw.keys() if ent in party_names]
|
||||
logger.info(
|
||||
"Found %d party-like entities in svd_vectors for %s",
|
||||
len(party_entities),
|
||||
latest,
|
||||
)
|
||||
|
||||
# Build aligned windows excluding party-level entities
|
||||
raw_window_vecs = {
|
||||
wid: traj._load_mp_vectors_for_window(args.db, wid) for wid in window_ids
|
||||
}
|
||||
# create filtered copy that removes party-level entity ids
|
||||
filtered_window_vecs = {
|
||||
wid: {ent: vec for ent, vec in d.items() if ent not in party_names}
|
||||
for wid, d in raw_window_vecs.items()
|
||||
}
|
||||
|
||||
aligned_filtered = traj._procrustes_align_windows(filtered_window_vecs)
|
||||
# stack and compute PCA
|
||||
all_vecs = []
|
||||
entity_index = []
|
||||
for wid, d in aligned_filtered.items():
|
||||
for ent, v in d.items():
|
||||
n = np.linalg.norm(v)
|
||||
all_vecs.append(v / n if n > 1e-10 else v)
|
||||
entity_index.append((wid, ent))
|
||||
|
||||
if not all_vecs:
|
||||
logger.error("No vectors left after excluding parties — aborting")
|
||||
return 2
|
||||
|
||||
M = np.vstack(all_vecs)
|
||||
Mc = M - M.mean(axis=0)
|
||||
try:
|
||||
U, s, Vt = np.linalg.svd(Mc, full_matrices=False)
|
||||
except Exception:
|
||||
logger.exception("SVD failed on filtered data")
|
||||
return 3
|
||||
|
||||
sv2 = s**2
|
||||
evr = sv2 / (sv2.sum() + 1e-20)
|
||||
logger.info("Filtered PCA EVR top2: %s", evr[:2].tolist())
|
||||
|
||||
comp1 = Vt[0]
|
||||
comp1_hat = comp1 / (np.linalg.norm(comp1) + 1e-12)
|
||||
comp2 = Vt[1] if Vt.shape[0] > 1 else np.zeros_like(comp1)
|
||||
comp2_hat = comp2 / (np.linalg.norm(comp2) + 1e-12)
|
||||
|
||||
# project filtered entities for latest window
|
||||
filtered_positions = {}
|
||||
global_mean = M.mean(axis=0)
|
||||
for (wid, ent), vec in zip(entity_index, M):
|
||||
if wid != latest:
|
||||
continue
|
||||
v_centered = vec - global_mean
|
||||
x = float(np.dot(v_centered, comp1_hat))
|
||||
y = float(np.dot(v_centered, comp2_hat))
|
||||
filtered_positions[ent] = (x, y)
|
||||
|
||||
# save JSON and small report
|
||||
out_json = os.path.join(args.out, "svd_filtered_positions.json")
|
||||
with open(out_json, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"latest": latest,
|
||||
"positions": filtered_positions,
|
||||
"evr": evr[:2].tolist(),
|
||||
},
|
||||
f,
|
||||
indent=2,
|
||||
)
|
||||
logger.info("Wrote filtered positions to %s", out_json)
|
||||
|
||||
# Also generate plots if plotly available
|
||||
try:
|
||||
party_map = _load_party_map(args.db)
|
||||
# positions_by_window format expected by plot functions — include only latest
|
||||
positions_by_window = {latest: filtered_positions}
|
||||
pcomp_out = os.path.join(args.out, f"political_compass_filtered_{latest}.html")
|
||||
plot_political_compass(
|
||||
positions_by_window,
|
||||
window_id=latest,
|
||||
party_of=party_map,
|
||||
axis_def={"method": "pca", "explained_variance_ratio": evr[:2]},
|
||||
output_path=pcomp_out,
|
||||
)
|
||||
logger.info("Wrote filtered compass to %s", pcomp_out)
|
||||
# simple trajectory plotting for filtered set — top movers by count
|
||||
traj_out = os.path.join(args.out, f"trajectories_filtered_{latest}.html")
|
||||
# Build simple per-MP coords across windows for filtered set
|
||||
mp_coords = {}
|
||||
for wid in window_ids:
|
||||
for ent, coord in aligned_filtered.get(wid, {}).items():
|
||||
if ent not in mp_coords:
|
||||
mp_coords[ent] = []
|
||||
mp_coords[ent].append((wid, tuple(coord.tolist())))
|
||||
# pick MPs with at least 2 windows
|
||||
names = [n for n, v in mp_coords.items() if len(v) >= 2]
|
||||
plot_2d_trajectories(
|
||||
{
|
||||
wid: {
|
||||
n: mp_coords[n][i][1]
|
||||
for n in names
|
||||
for i, (w, _) in enumerate(mp_coords[n])
|
||||
if w == wid
|
||||
}
|
||||
for wid in window_ids
|
||||
},
|
||||
mp_names=names[:50],
|
||||
output_path=traj_out,
|
||||
)
|
||||
logger.info("Wrote filtered trajectories to %s", traj_out)
|
||||
except Exception:
|
||||
logger.exception("Plotting filtered results failed — plots skipped")
|
||||
|
||||
# console summary
|
||||
print("Top duplicate groups (count, sample):")
|
||||
for k, v in group_list[:20]:
|
||||
print(len(v), v[:6])
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Compute summaries and embeddings for a small test batch of motions.
|
||||
|
||||
Usage:
|
||||
# dry-run (no network calls)
|
||||
python scripts/compute_test_batch.py --limit 20 --dry-run
|
||||
|
||||
# run (will call AI provider; requires OPENROUTER_API_KEY)
|
||||
python scripts/compute_test_batch.py --limit 20
|
||||
|
||||
This script is intentionally simple and intended for manual invocation.
|
||||
It will update motions.layman_explanation and store embeddings via db.store_embedding if available.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from typing import List
|
||||
|
||||
import duckdb
|
||||
|
||||
from config import config
|
||||
import ai_provider
|
||||
from database import db
|
||||
from summarizer import MotionSummarizer
|
||||
|
||||
|
||||
logger = logging.getLogger("compute_test_batch")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
|
||||
def fetch_motion_candidates(limit: int) -> List[dict]:
|
||||
conn = duckdb.connect(config.DATABASE_PATH)
|
||||
try:
|
||||
# Prefer motions that still lack a layman_explanation so we don't re-process recent ones
|
||||
rows = conn.execute(
|
||||
"SELECT id, title, description FROM motions WHERE layman_explanation IS NULL OR layman_explanation = '' ORDER BY created_at DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [{"id": r[0], "title": r[1], "description": r[2] or ""} for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def process_batch(limit: int = 20, dry_run: bool = False):
|
||||
summarizer = MotionSummarizer()
|
||||
motions = fetch_motion_candidates(limit)
|
||||
logger.info("Found %d motions to process", len(motions))
|
||||
|
||||
conn = duckdb.connect(config.DATABASE_PATH)
|
||||
try:
|
||||
for i, m in enumerate(motions, start=1):
|
||||
mid = m["id"]
|
||||
title = m["title"]
|
||||
desc = m["description"]
|
||||
logger.info(
|
||||
"[%d/%d] Processing motion id=%s title=%s", i, len(motions), mid, title
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
logger.info(
|
||||
"Dry run: would generate summary and embedding for motion %s", mid
|
||||
)
|
||||
continue
|
||||
|
||||
# Generate summary
|
||||
summary = summarizer.generate_layman_explanation(title, desc)
|
||||
# Update DB
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE motions SET layman_explanation = ? WHERE id = ?",
|
||||
(summary, mid),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Failed to update motion %s: %s", mid, e)
|
||||
|
||||
# Compute embedding and store
|
||||
try:
|
||||
emb = ai_provider.get_embedding(summary)
|
||||
store_fn = getattr(db, "store_embedding", None)
|
||||
if callable(store_fn):
|
||||
store_fn(mid, "text-embedding-3-small", emb)
|
||||
logger.info("Stored embedding for motion %s", mid)
|
||||
else:
|
||||
logger.warning(
|
||||
"No store_embedding available on db; skipping storage"
|
||||
)
|
||||
except ai_provider.ProviderError as e:
|
||||
logger.exception(
|
||||
"Failed to compute/store embedding for motion %s: %s", mid, e
|
||||
)
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--limit", type=int, default=20, help="Number of motions to process")
|
||||
p.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Do not call external APIs; just show what would run",
|
||||
)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
if args.dry_run:
|
||||
logger.info("Running in dry-run mode; no network calls will be made")
|
||||
|
||||
# Safety: confirm when not dry-run
|
||||
if not args.dry_run:
|
||||
confirm = (
|
||||
input(
|
||||
f"This will call the AI provider for {args.limit} motions and may incur cost. Continue? (y/N): "
|
||||
)
|
||||
.strip()
|
||||
.lower()
|
||||
)
|
||||
if confirm not in ("y", "yes"):
|
||||
logger.info("Aborting per user choice")
|
||||
sys.exit(0)
|
||||
|
||||
process_batch(limit=args.limit, dry_run=args.dry_run)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Backfill missing mp_votes.party values from mp_metadata and co-voting inference.
|
||||
|
||||
Multi-tier strategy:
|
||||
1) Tussenvoegsel-aware name match against mp_metadata.
|
||||
2) Majority party already recorded in mp_votes for the same MP.
|
||||
3) Looser last-name-token match against mp_metadata.
|
||||
4) Co-voting inference: for MPs still unresolved, find which party's MPs
|
||||
they vote identically with most often, using a Jaccard-style overlap.
|
||||
|
||||
Usage:
|
||||
uv run python3 scripts/fill_mp_votes_parties.py --db data/motions.db
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
|
||||
import duckdb
|
||||
|
||||
logger = logging.getLogger("fill_mp_votes_parties")
|
||||
|
||||
|
||||
_TUSSENVOEGSEL = {
|
||||
"van de",
|
||||
"van den",
|
||||
"van der",
|
||||
"van het",
|
||||
"van",
|
||||
"de",
|
||||
"den",
|
||||
"der",
|
||||
"het",
|
||||
"ter",
|
||||
"ten",
|
||||
"el",
|
||||
"al",
|
||||
"in 't",
|
||||
}
|
||||
|
||||
# Build a regex that matches any known tussenvoegsel (longest first to avoid
|
||||
# partial matches like "van" eating the "van" in "van der").
|
||||
_TV_PATTERN = re.compile(
|
||||
r"\b("
|
||||
+ "|".join(re.escape(tv) for tv in sorted(_TUSSENVOEGSEL, key=len, reverse=True))
|
||||
+ r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def normalize_mp_key(name: str) -> str:
|
||||
"""Produce a canonical key that matches regardless of tussenvoegsel position.
|
||||
|
||||
Both "Burg van der, E." (mp_votes style) and "Van der Burg, E."
|
||||
(mp_metadata style) should produce the same key. Also strips diacritics
|
||||
so "Kostić, I." matches "Kostic, I.".
|
||||
|
||||
Strategy: split into pre-comma and post-comma parts. From the pre-comma
|
||||
part, extract any tussenvoegsel tokens and the remaining lastname.
|
||||
Canonical key = "lastname tussenvoegsel initials", all lowercased.
|
||||
"""
|
||||
if not name:
|
||||
return ""
|
||||
# Strip diacritics: NFD decompose then drop combining marks
|
||||
s = unicodedata.normalize("NFD", name)
|
||||
s = "".join(c for c in s if unicodedata.category(c) != "Mn")
|
||||
# remove parenthetical fullnames e.g. "(Christine)"
|
||||
s = re.sub(r"\s*\(.*?\)", "", s).strip()
|
||||
# remove dots and commas for splitting but keep the comma position
|
||||
# Split on first comma: last_part, initials_part
|
||||
parts = s.split(",", 1)
|
||||
last_part = parts[0].strip()
|
||||
initials_part = parts[1].strip() if len(parts) > 1 else ""
|
||||
|
||||
# Clean initials: remove dots
|
||||
initials = re.sub(r"\.", "", initials_part).strip().lower()
|
||||
|
||||
# From last_part, extract tussenvoegsel and lastname
|
||||
last_lower = last_part.lower()
|
||||
# Find all tussenvoegsel matches
|
||||
found_tv = []
|
||||
remaining = last_lower
|
||||
for m in _TV_PATTERN.finditer(last_lower):
|
||||
found_tv.append(m.group(0).lower())
|
||||
# Remove tussenvoegsel tokens from remaining to get the pure lastname
|
||||
remaining = _TV_PATTERN.sub("", last_lower).strip()
|
||||
remaining = re.sub(r"\s+", " ", remaining).strip()
|
||||
|
||||
# Sort tussenvoegsel to canonical order
|
||||
tv_str = " ".join(sorted(found_tv)) if found_tv else ""
|
||||
|
||||
# Build canonical key: "lastname tv initials"
|
||||
key_parts = [remaining]
|
||||
if tv_str:
|
||||
key_parts.append(tv_str)
|
||||
if initials:
|
||||
key_parts.append(initials)
|
||||
return " ".join(key_parts)
|
||||
|
||||
|
||||
def pick_preferred_party(records: list) -> str | None:
|
||||
# records: list of dicts with keys party, van, tot
|
||||
# prefer active membership
|
||||
for r in records:
|
||||
if r.get("tot") is None and r.get("party"):
|
||||
return r.get("party")
|
||||
# otherwise pick most recent van
|
||||
best = None
|
||||
best_date = None
|
||||
for r in records:
|
||||
van = r.get("van")
|
||||
try:
|
||||
d = datetime.fromisoformat(van).date() if van else None
|
||||
except Exception:
|
||||
d = None
|
||||
if d and (best_date is None or d > best_date):
|
||||
best_date = d
|
||||
best = r
|
||||
if best:
|
||||
return best.get("party")
|
||||
# fallback to any party present
|
||||
for r in records:
|
||||
if r.get("party"):
|
||||
return r.get("party")
|
||||
return None
|
||||
|
||||
|
||||
def _infer_party_by_covoting(conn, mp_name: str, min_overlap: int = 10) -> str | None:
|
||||
"""Infer party by finding which known-party MPs vote identically most often.
|
||||
|
||||
For each motion where *mp_name* voted, find all other MPs who cast the
|
||||
same vote AND already have a party assigned. The party with the highest
|
||||
agreement count wins, provided the overlap exceeds *min_overlap*.
|
||||
"""
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT other.party, COUNT(*) AS agreement
|
||||
FROM mp_votes me
|
||||
JOIN mp_votes other
|
||||
ON me.motion_id = other.motion_id
|
||||
AND me.vote = other.vote
|
||||
WHERE me.mp_name = ?
|
||||
AND other.mp_name != ?
|
||||
AND other.party IS NOT NULL
|
||||
AND other.party != ''
|
||||
AND other.mp_name LIKE '%,%'
|
||||
GROUP BY other.party
|
||||
ORDER BY agreement DESC
|
||||
LIMIT 5
|
||||
""",
|
||||
(mp_name, mp_name),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return None
|
||||
|
||||
best_party, best_count = rows[0]
|
||||
if best_count < min_overlap:
|
||||
return None
|
||||
|
||||
# Require meaningful margin over second-best to avoid ambiguous assignment
|
||||
if len(rows) > 1:
|
||||
second_count = rows[1][1]
|
||||
# Best must have at least 20% more agreement than runner-up
|
||||
if best_count < second_count * 1.2:
|
||||
logger.debug(
|
||||
"Co-voting ambiguous for %s: %s=%d vs %s=%d",
|
||||
mp_name,
|
||||
best_party,
|
||||
best_count,
|
||||
rows[1][0],
|
||||
second_count,
|
||||
)
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
"Co-voting inferred %s -> %s (agreement=%d)",
|
||||
mp_name,
|
||||
best_party,
|
||||
best_count,
|
||||
)
|
||||
return best_party
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--db", default="data/motions.db")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
conn = duckdb.connect(args.db)
|
||||
|
||||
# Load mp_metadata
|
||||
md_rows = conn.execute(
|
||||
"SELECT mp_name, party, van, tot_en_met FROM mp_metadata"
|
||||
).fetchall()
|
||||
|
||||
metadata = defaultdict(list)
|
||||
for mp_name, party, van, tot in md_rows:
|
||||
key = normalize_mp_key(mp_name)
|
||||
metadata[key].append(
|
||||
{"mp_name": mp_name, "party": party, "van": van, "tot": tot}
|
||||
)
|
||||
|
||||
# Build majority-party mapping from existing mp_votes (non-null parties)
|
||||
party_counts = defaultdict(lambda: defaultdict(int))
|
||||
rows_counts = conn.execute(
|
||||
"SELECT mp_name, party, COUNT(*) FROM mp_votes WHERE party IS NOT NULL AND party != '' GROUP BY mp_name, party"
|
||||
).fetchall()
|
||||
for mp_name, party, cnt in rows_counts:
|
||||
key = normalize_mp_key(mp_name)
|
||||
party_counts[key][party] += cnt
|
||||
|
||||
majority_by_norm = {
|
||||
k: max(v.items(), key=lambda kv: kv[1])[0] for k, v in party_counts.items()
|
||||
}
|
||||
|
||||
# Target mp_votes rows: individual MPs (contain comma) with NULL or empty party
|
||||
target_rows = conn.execute(
|
||||
"SELECT id, mp_name FROM mp_votes WHERE (party IS NULL OR party = '') AND mp_name LIKE '%,%'"
|
||||
).fetchall()
|
||||
|
||||
updated = 0
|
||||
# Track MPs that need co-voting inference (tier 4) — collect after tiers 1-3
|
||||
covote_candidates: dict[str, list[int]] = defaultdict(list) # mp_name -> [ids]
|
||||
|
||||
for id_, mp_name in target_rows:
|
||||
key = normalize_mp_key(mp_name)
|
||||
chosen_party = None
|
||||
|
||||
# 1) exact normalized metadata match
|
||||
if key in metadata:
|
||||
chosen_party = pick_preferred_party(metadata[key])
|
||||
|
||||
# 2) fallback to majority observed in mp_votes
|
||||
if not chosen_party:
|
||||
chosen_party = majority_by_norm.get(key)
|
||||
|
||||
# 3) try looser substring matches on lastname token
|
||||
if not chosen_party:
|
||||
tokens = key.split()
|
||||
if tokens:
|
||||
lastname = tokens[0]
|
||||
# find metadata keys that start with lastname
|
||||
for meta_key, recs in metadata.items():
|
||||
if meta_key.split()[0] == lastname:
|
||||
chosen_party = pick_preferred_party(recs)
|
||||
if chosen_party:
|
||||
break
|
||||
|
||||
if chosen_party:
|
||||
conn.execute(
|
||||
"UPDATE mp_votes SET party = ? WHERE id = ?", (chosen_party, id_)
|
||||
)
|
||||
updated += 1
|
||||
else:
|
||||
covote_candidates[mp_name].append(id_)
|
||||
|
||||
# 4) Co-voting inference for remaining unresolved MPs
|
||||
for mp_name, ids in covote_candidates.items():
|
||||
inferred = _infer_party_by_covoting(conn, mp_name)
|
||||
if inferred:
|
||||
for id_ in ids:
|
||||
conn.execute(
|
||||
"UPDATE mp_votes SET party = ? WHERE id = ?", (inferred, id_)
|
||||
)
|
||||
updated += 1
|
||||
|
||||
conn.close()
|
||||
logger.info("Updated %d mp_votes rows with party info", updated)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Generate political compass and 2D trajectories HTML outputs.
|
||||
|
||||
This script computes 2D axes using residual-PCA (or anchor), applies the
|
||||
party-fill helper to colour MPs, and writes self-contained HTML files into
|
||||
an outputs/ directory.
|
||||
|
||||
Usage:
|
||||
python scripts/generate_compass.py --db data/motions.db --out outputs --method pca --pca-residual
|
||||
|
||||
The script is defensive: if required optional libraries (duckdb, plotly,
|
||||
scipy) are missing it will log and exit without raising an uncaught exception.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
# Ensure project root is on sys.path so `import analysis.*` works when the
|
||||
# script is executed from the repository root or from scripts/ directly.
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if ROOT not in sys.path:
|
||||
sys.path.insert(0, ROOT)
|
||||
|
||||
|
||||
logger = logging.getLogger("generate_compass")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
|
||||
def main(argv: Optional[list] = None):
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--db", default="data/motions.db", help="Path to duckdb database")
|
||||
p.add_argument("--out", default="outputs", help="Output directory")
|
||||
p.add_argument("--method", choices=["pca", "anchor"], default="pca")
|
||||
p.add_argument(
|
||||
"--pca-residual", action="store_true", help="Use residual PCA for second axis"
|
||||
)
|
||||
p.add_argument(
|
||||
"--y-scale",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Optional manual y-axis scale multiplier",
|
||||
)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
# Lazy imports so the script exits gracefully if deps missing
|
||||
try:
|
||||
from analysis.political_axis import compute_2d_axes
|
||||
from analysis.visualize import (
|
||||
plot_political_compass,
|
||||
plot_2d_trajectories,
|
||||
_load_party_map,
|
||||
)
|
||||
except Exception as e: # pragma: no cover - runtime helper
|
||||
logger.exception("Required analysis modules could not be imported: %s", e)
|
||||
sys.exit(1)
|
||||
|
||||
# Ensure output dir exists
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
|
||||
logger.info(
|
||||
"Computing 2D axes (method=%s pca_residual=%s)", args.method, args.pca_residual
|
||||
)
|
||||
|
||||
try:
|
||||
positions_by_window, axis_def = compute_2d_axes(
|
||||
args.db,
|
||||
method=args.method,
|
||||
pca_residual=args.pca_residual,
|
||||
normalize_vectors=True,
|
||||
)
|
||||
except Exception as e: # defensive
|
||||
logger.exception("compute_2d_axes failed: %s", e)
|
||||
sys.exit(1)
|
||||
|
||||
if not positions_by_window:
|
||||
logger.error("No positions produced — aborting")
|
||||
sys.exit(1)
|
||||
|
||||
# pick latest window (lexicographic order is used elsewhere in codebase)
|
||||
window_id = sorted(positions_by_window.keys())[-1]
|
||||
|
||||
# Build party mapping to colour points
|
||||
try:
|
||||
party_map = _load_party_map(args.db)
|
||||
except Exception:
|
||||
logger.exception("Failed to build party map; proceeding without it")
|
||||
party_map = None
|
||||
|
||||
# Output files
|
||||
compass_out = os.path.join(
|
||||
args.out, f"political_compass_{args.method}_{window_id}.html"
|
||||
)
|
||||
traj_out = os.path.join(args.out, f"trajectories_compass_{args.method}_top50.html")
|
||||
|
||||
try:
|
||||
plot_political_compass(
|
||||
positions_by_window,
|
||||
window_id=window_id,
|
||||
party_of=party_map,
|
||||
axis_def=axis_def,
|
||||
y_scale=args.y_scale,
|
||||
output_path=compass_out,
|
||||
)
|
||||
logger.info("Wrote compass to %s", compass_out)
|
||||
except Exception:
|
||||
logger.exception("Failed to write political compass")
|
||||
|
||||
try:
|
||||
# Build 2D trajectories from the already-computed positions_by_window so
|
||||
# we keep the same PCA/anchor axes (compute_2d_trajectories would call
|
||||
# compute_2d_axes again which may use different defaults).
|
||||
import numpy as _np
|
||||
|
||||
window_ids = sorted(positions_by_window.keys())
|
||||
|
||||
mp_data = {}
|
||||
for wid in window_ids:
|
||||
pos = positions_by_window.get(wid, {})
|
||||
for mp_name, coord in pos.items():
|
||||
mp_data.setdefault(mp_name, {"windows": [], "coords": []})
|
||||
mp_data[mp_name]["windows"].append(wid)
|
||||
mp_data[mp_name]["coords"].append(tuple(coord))
|
||||
|
||||
trajs = {}
|
||||
for mp_name, data in mp_data.items():
|
||||
if len(data["windows"]) < 2:
|
||||
continue
|
||||
coords = [_np.array(c, dtype=float) for c in data["coords"]]
|
||||
step_vecs = [coords[i + 1] - coords[i] for i in range(len(coords) - 1)]
|
||||
mags = [float(_np.linalg.norm(v)) for v in step_vecs]
|
||||
trajs[mp_name] = {
|
||||
"windows": data["windows"],
|
||||
"coords": [[float(c[0]), float(c[1])] for c in coords],
|
||||
"step_vectors": [[float(v[0]), float(v[1])] for v in step_vecs],
|
||||
"step_magnitudes": mags,
|
||||
"total_magnitude": float(sum(mags)),
|
||||
}
|
||||
|
||||
ranked = sorted(
|
||||
trajs.items(), key=lambda kv: kv[1]["total_magnitude"], reverse=True
|
||||
)
|
||||
top_names = [mp for mp, _ in ranked[:50]] if ranked else None
|
||||
|
||||
plot_2d_trajectories(
|
||||
positions_by_window, mp_names=top_names, output_path=traj_out
|
||||
)
|
||||
logger.info("Wrote trajectories to %s", traj_out)
|
||||
except Exception:
|
||||
logger.exception("Failed to compute/write trajectories")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Inspect PCA axes and per-MP projections for diagnostics.
|
||||
|
||||
Usage:
|
||||
uv run python3 scripts/inspect_axis.py --db data/motions.db --out outputs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, List
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if ROOT not in sys.path:
|
||||
sys.path.insert(0, ROOT)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger("inspect_axis")
|
||||
|
||||
|
||||
def main(argv: List[str] | None = None):
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--db", default="data/motions.db")
|
||||
p.add_argument("--out", default="outputs")
|
||||
p.add_argument("--method", choices=["pca", "anchor"], default="pca")
|
||||
p.add_argument("--pca-residual", action="store_true")
|
||||
p.add_argument("--normalize", action="store_true", default=True)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
|
||||
try:
|
||||
from analysis.political_axis import compute_2d_axes
|
||||
from analysis.visualize import _load_party_map
|
||||
except Exception as e:
|
||||
logger.exception("Failed to import analysis modules: %s", e)
|
||||
raise
|
||||
|
||||
positions_by_window, axes = compute_2d_axes(
|
||||
args.db,
|
||||
method=args.method,
|
||||
pca_residual=args.pca_residual,
|
||||
normalize_vectors=args.normalize,
|
||||
)
|
||||
|
||||
if not positions_by_window:
|
||||
logger.error("No positions produced")
|
||||
return 2
|
||||
|
||||
latest = sorted(positions_by_window.keys())[-1]
|
||||
pos = positions_by_window[latest]
|
||||
|
||||
names = list(pos.keys())
|
||||
coords = list(pos.values())
|
||||
xs = [c[0] for c in coords]
|
||||
ys = [c[1] for c in coords]
|
||||
|
||||
import numpy as _np
|
||||
|
||||
x_std = float(_np.std(xs))
|
||||
y_std = float(_np.std(ys))
|
||||
x_min, x_max = min(xs), max(xs)
|
||||
y_min, y_max = min(ys), max(ys)
|
||||
|
||||
party_map = _load_party_map(args.db)
|
||||
|
||||
# load mp_votes counts
|
||||
try:
|
||||
import duckdb
|
||||
|
||||
conn = duckdb.connect(args.db)
|
||||
rows = conn.execute(
|
||||
"SELECT mp_name, COUNT(*) FROM mp_votes GROUP BY mp_name"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
vote_counts = {r[0]: int(r[1]) for r in rows}
|
||||
except Exception:
|
||||
vote_counts = {}
|
||||
|
||||
# extremes
|
||||
sorted_by_x = sorted(pos.items(), key=lambda kv: kv[1][0])
|
||||
sorted_by_y = sorted(pos.items(), key=lambda kv: kv[1][1])
|
||||
|
||||
def info_for(name: str):
|
||||
party = party_map.get(name)
|
||||
count = vote_counts.get(name, None)
|
||||
x, y = pos.get(name, (None, None))
|
||||
return {"name": name, "party": party, "count": count, "x": x, "y": y}
|
||||
|
||||
report = {
|
||||
"db": args.db,
|
||||
"latest_window": latest,
|
||||
"n_entities": len(names),
|
||||
"x_std": x_std,
|
||||
"y_std": y_std,
|
||||
"x_min": x_min,
|
||||
"x_max": x_max,
|
||||
"y_min": y_min,
|
||||
"y_max": y_max,
|
||||
"evr": axes.get("explained_variance_ratio") if axes else None,
|
||||
"top_left_by_x": [info_for(n) for n, _ in sorted_by_x[:10]],
|
||||
"top_right_by_x": [info_for(n) for n, _ in sorted_by_x[-10:]],
|
||||
"top_by_y": [info_for(n) for n, _ in sorted_by_y[-10:]],
|
||||
"bottom_by_y": [info_for(n) for n, _ in sorted_by_y[:10]],
|
||||
}
|
||||
|
||||
# count how many are near-center along x within small fraction of std
|
||||
threshold = 0.2 * x_std if x_std > 0 else 0.01
|
||||
near_center = [n for n, (x, y) in pos.items() if abs(x) < threshold]
|
||||
report["near_center_count"] = len(near_center)
|
||||
report["near_center_sample"] = near_center[:40]
|
||||
|
||||
# check duplicate coordinate pairs
|
||||
coord_pairs = [(_np.round(c[0], 6), _np.round(c[1], 6)) for c in coords]
|
||||
unique_coords = set(coord_pairs)
|
||||
report["n_unique_coords"] = len(unique_coords)
|
||||
report["n_total_entities"] = len(names)
|
||||
|
||||
# look up particular MPs
|
||||
for q in ("Ouwehand", "Keijzer", "Mona"):
|
||||
found = [n for n in names if q.lower() in n.lower()]
|
||||
report[f"matches_{q}"] = [info_for(n) for n in found]
|
||||
|
||||
out_json = os.path.join(args.out, "inspect_axis.json")
|
||||
with open(out_json, "w", encoding="utf-8") as f:
|
||||
json.dump(report, f, indent=2)
|
||||
|
||||
logger.info("Wrote inspection to %s", out_json)
|
||||
print(json.dumps(report, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Quick QA script that samples motions and checks similarity cache quality.
|
||||
|
||||
Writes a short JSON summary into thoughts/ledgers/qa_similarity_{ts}.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def sample_motion_ids(sample_size: int) -> List[int]:
|
||||
# naive: select all motion ids from DB and sample
|
||||
# Prefer any dynamically-provided database object from the 'database'
|
||||
# module so tests can inject a fake via sys.modules.
|
||||
try:
|
||||
database_mod = __import__("database")
|
||||
db_obj = getattr(database_mod, "db", None)
|
||||
if db_obj and hasattr(db_obj, "sample_motions"):
|
||||
return db_obj.sample_motions(sample_size)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
conn = (
|
||||
__import__("duckdb").connect(db.db_path) if __import__("duckdb") else None
|
||||
)
|
||||
except Exception:
|
||||
conn = None
|
||||
|
||||
if conn is None:
|
||||
# fallback: read from motions.json if present (file-backed mode)
|
||||
# Not implemented: return empty
|
||||
return []
|
||||
|
||||
try:
|
||||
rows = conn.execute("SELECT id FROM motions").fetchall()
|
||||
conn.close()
|
||||
ids = [r[0] for r in rows]
|
||||
if not ids:
|
||||
return []
|
||||
return random.sample(ids, min(sample_size, len(ids)))
|
||||
except Exception:
|
||||
if conn:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def run_qa(db_path: str, sample_size: int = 50, top_k: int = 5) -> dict:
|
||||
summary = {
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"sample_size": sample_size,
|
||||
"top_k": top_k,
|
||||
"results": [],
|
||||
}
|
||||
|
||||
ids = sample_motion_ids(sample_size)
|
||||
if not ids:
|
||||
summary["error"] = "no motion ids available"
|
||||
return summary
|
||||
|
||||
# Resolve db at runtime so tests can substitute a fake module
|
||||
try:
|
||||
database_mod = __import__("database")
|
||||
db_obj = getattr(database_mod, "db", None)
|
||||
except Exception:
|
||||
db_obj = None
|
||||
|
||||
for mid in ids:
|
||||
if db_obj and hasattr(db_obj, "get_cached_similarities"):
|
||||
sims = db_obj.get_cached_similarities(mid, top_k=top_k)
|
||||
else:
|
||||
# fallback: attempt to call module-level db if present
|
||||
try:
|
||||
from database import db as fallback_db
|
||||
|
||||
sims = fallback_db.get_cached_similarities(
|
||||
mid, vector_type="fused", top_k=top_k
|
||||
)
|
||||
except Exception:
|
||||
sims = []
|
||||
# heuristics: count how many top_k have score >= 0.99999 and different target ids
|
||||
suspicious = 0
|
||||
for r in sims:
|
||||
try:
|
||||
score = float(r.get("score", 0.0))
|
||||
target = (
|
||||
r.get("target_motion_id")
|
||||
if r.get("target_motion_id") is not None
|
||||
else r.get("id")
|
||||
)
|
||||
if score > 0.99999 and int(target) != int(mid):
|
||||
suspicious += 1
|
||||
except Exception:
|
||||
# Be tolerant of unexpected structures in similarity rows
|
||||
continue
|
||||
summary["results"].append(
|
||||
{"motion_id": mid, "top_k": len(sims), "suspicious": suspicious}
|
||||
)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def main(db_path: str | None = None, sample_size: int = 50, top_k: int = 5) -> dict:
|
||||
"""Wrapper used by CLI and tests.
|
||||
|
||||
When called with no args, this behaves like the prior CLI entrypoint and
|
||||
will parse command-line args and write a ledger file. Tests call main()
|
||||
directly with explicit parameters and expect a dict summary to be
|
||||
returned (and a ledger to be written). To maintain compatibility we
|
||||
support both usage patterns.
|
||||
"""
|
||||
# If invoked as CLI, db_path will be None and we should parse args and
|
||||
# write the ledger file as before.
|
||||
if db_path is None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
parser = argparse.ArgumentParser(description="QA similarity cache sampler")
|
||||
parser.add_argument("--db-path", required=False, help="Path to motions.db")
|
||||
parser.add_argument("--sample-size", type=int, default=50)
|
||||
parser.add_argument("--top-k", type=int, default=5)
|
||||
args = parser.parse_args()
|
||||
db_path = args.db_path or db.db_path
|
||||
sample_size = args.sample_size
|
||||
top_k = args.top_k
|
||||
|
||||
summary = run_qa(db_path or db.db_path, sample_size=sample_size, top_k=top_k)
|
||||
# Provide a convenience mapping of motion_id -> result for easier consumption
|
||||
# by callers/tests which expect a `motions` mapping.
|
||||
summary["motions"] = {r["motion_id"]: r for r in summary.get("results", [])}
|
||||
ledger_dir = os.path.join("thoughts", "ledgers")
|
||||
os.makedirs(ledger_dir, exist_ok=True)
|
||||
ts = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
path = os.path.join(ledger_dir, f"qa_similarity_{ts}.json")
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump(summary, fh, ensure_ascii=False, indent=2)
|
||||
print(f"Wrote QA summary to {path}")
|
||||
return {"ledger_path": path, **summary}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Recompute per-window SVD into a fresh DB copy and re-run 2D axes.
|
||||
|
||||
This script copies the current data/motions.db to a new file (data/motions_recompute.db),
|
||||
clears any existing svd_vectors rows for the target windows in the new DB, runs
|
||||
SVD on each window, then computes 2D axes and writes compass + trajectories into
|
||||
outputs_recomputed/ for inspection.
|
||||
|
||||
Usage:
|
||||
uv run python3 scripts/recompute_svd.py --db data/motions.db --out outputs_recomputed
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import calendar
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import date
|
||||
from typing import List, Tuple
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if ROOT not in sys.path:
|
||||
sys.path.insert(0, ROOT)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger("recompute_svd")
|
||||
|
||||
|
||||
def year_bounds(window_id: str) -> Tuple[str, str]:
|
||||
"""Return (start_date, end_date) for an annual window_id like '2024'.
|
||||
|
||||
Quarterly window IDs (containing '-Q') are not supported — this script
|
||||
only processes annual windows.
|
||||
"""
|
||||
if "-Q" in window_id:
|
||||
raise ValueError(
|
||||
f"Quarterly window '{window_id}' is not supported. "
|
||||
"Only annual windows should be recomputed."
|
||||
)
|
||||
y = int(window_id)
|
||||
start = date(y, 1, 1).isoformat()
|
||||
end = date(y, 12, 31).isoformat()
|
||||
return start, end
|
||||
|
||||
|
||||
def main(argv: List[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--db", default="data/motions.db")
|
||||
p.add_argument("--out", default="outputs_recomputed")
|
||||
p.add_argument("--k", type=int, default=50)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
|
||||
# Copy DB to a new file so we don't clobber originals
|
||||
src = args.db
|
||||
dst = os.path.splitext(src)[0] + "_recompute.db"
|
||||
logger.info("Copying %s -> %s", src, dst)
|
||||
shutil.copyfile(src, dst)
|
||||
|
||||
# Lazy imports
|
||||
try:
|
||||
from database import MotionDatabase
|
||||
from pipeline.svd_pipeline import run_svd_for_window
|
||||
from analysis.political_axis import compute_2d_axes
|
||||
from analysis.visualize import (
|
||||
plot_political_compass,
|
||||
plot_2d_trajectories,
|
||||
_load_party_map,
|
||||
)
|
||||
from analysis import trajectory as traj
|
||||
except Exception as e:
|
||||
logger.exception("Import failed: %s", e)
|
||||
return 2
|
||||
|
||||
# build MotionDatabase pointing to new file
|
||||
db = MotionDatabase(dst)
|
||||
|
||||
# find windows from original DB via trajectory helper
|
||||
all_window_ids = traj._load_window_ids(src)
|
||||
# Only process annual windows — quarterly windows are excluded from all PCA/SVD computation
|
||||
window_ids = [w for w in all_window_ids if "-Q" not in w]
|
||||
if not window_ids:
|
||||
logger.error("No annual windows found in source DB %s", src)
|
||||
return 3
|
||||
|
||||
logger.info("Will recompute SVD for annual windows: %s", window_ids)
|
||||
|
||||
# clear existing svd_vectors rows for these windows in dst DB
|
||||
import duckdb
|
||||
|
||||
conn = duckdb.connect(dst)
|
||||
try:
|
||||
conn.execute(
|
||||
"DELETE FROM svd_vectors WHERE window_id IN ({})".format(
|
||||
",".join([f"'{w}'" for w in window_ids])
|
||||
)
|
||||
)
|
||||
conn.commit()
|
||||
logger.info("Cleared existing svd_vectors rows for windows in %s", dst)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Run SVD per window
|
||||
for wid in window_ids:
|
||||
start, end = year_bounds(wid)
|
||||
logger.info("Running SVD for %s (%s -> %s) k=%d", wid, start, end, args.k)
|
||||
res = run_svd_for_window(
|
||||
db=db, window_id=wid, start_date=start, end_date=end, k=args.k
|
||||
)
|
||||
logger.info("SVD result for %s: %s", wid, res)
|
||||
|
||||
# Recompute 2D axes and plots from the recomputed DB
|
||||
logger.info("Computing 2D axes (pca_residual=True) from recomputed DB")
|
||||
positions_by_window, axes = compute_2d_axes(
|
||||
dst, method="pca", pca_residual=True, normalize_vectors=True
|
||||
)
|
||||
if not positions_by_window:
|
||||
logger.error("No positions returned from compute_2d_axes on recomputed DB")
|
||||
return 5
|
||||
|
||||
latest = sorted(positions_by_window.keys())[-1]
|
||||
party_map = _load_party_map(dst)
|
||||
|
||||
compass_out = os.path.join(args.out, f"political_compass_recomputed_{latest}.html")
|
||||
traj_out = os.path.join(args.out, f"trajectories_recomputed_{latest}_top50.html")
|
||||
|
||||
plot_political_compass(
|
||||
positions_by_window,
|
||||
window_id=latest,
|
||||
party_of=party_map,
|
||||
axis_def=axes,
|
||||
output_path=compass_out,
|
||||
)
|
||||
logger.info("Wrote recomputed compass to %s", compass_out)
|
||||
|
||||
# compute simple trajectories from positions_by_window
|
||||
# build per-MP coords
|
||||
mp_coords = {}
|
||||
for wid in sorted(positions_by_window.keys()):
|
||||
for mp, coord in positions_by_window[wid].items():
|
||||
mp_coords.setdefault(mp, []).append((wid, coord))
|
||||
|
||||
names = [n for n, v in mp_coords.items() if len(v) >= 2]
|
||||
plot_2d_trajectories(positions_by_window, mp_names=names[:50], output_path=traj_out)
|
||||
logger.info("Wrote recomputed trajectories to %s", traj_out)
|
||||
|
||||
# write a short diagnostic JSON (convert numpy arrays to lists)
|
||||
import json
|
||||
import numpy as _np
|
||||
|
||||
def _to_serializable(o):
|
||||
if isinstance(o, _np.ndarray):
|
||||
return o.tolist()
|
||||
if isinstance(o, (_np.floating, _np.integer)):
|
||||
return float(o)
|
||||
raise TypeError(f"Object of type {type(o)} is not JSON serializable")
|
||||
|
||||
diag = {"windows": window_ids, "axes": axes}
|
||||
with open(
|
||||
os.path.join(args.out, "recompute_diag.json"), "w", encoding="utf-8"
|
||||
) as f:
|
||||
json.dump(diag, f, indent=2, default=_to_serializable)
|
||||
|
||||
logger.info("Recompute complete; outputs in %s and DB copy at %s", args.out, dst)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,286 @@
|
||||
"""semantic_gravity_examples.py — Show concrete motion examples for SVD axes across windows.
|
||||
|
||||
For each axis and window, finds motions closest to the semantic gravity vector,
|
||||
providing concrete examples of what the axis "means" in that period.
|
||||
|
||||
Usage:
|
||||
uv run python scripts/semantic_gravity_examples.py --db data/motions.db --axis 1
|
||||
uv run python scripts/semantic_gravity_examples.py --db data/motions.db --all
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
import duckdb
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _load_fused_embeddings_with_titles(
|
||||
con: duckdb.DuckDBPyConnection, window_id: str
|
||||
) -> List[Tuple[int, np.ndarray, str]]:
|
||||
"""Load fused embeddings with motion titles for a window."""
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT f.motion_id, f.vector, m.title
|
||||
FROM fused_embeddings f
|
||||
JOIN motions m ON f.motion_id = m.id
|
||||
WHERE f.window_id = ?
|
||||
""",
|
||||
[window_id],
|
||||
).fetchall()
|
||||
|
||||
result = []
|
||||
for motion_id, raw_vec, title in rows:
|
||||
if isinstance(raw_vec, str):
|
||||
vec = json.loads(raw_vec)
|
||||
elif isinstance(raw_vec, (bytes, bytearray)):
|
||||
vec = json.loads(raw_vec.decode())
|
||||
elif isinstance(raw_vec, list):
|
||||
vec = raw_vec
|
||||
else:
|
||||
vec = list(raw_vec)
|
||||
result.append(
|
||||
(
|
||||
motion_id,
|
||||
np.array([float(v) if v is not None else 0.0 for v in vec]),
|
||||
title or "",
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _load_motion_scores(
|
||||
con: duckdb.DuckDBPyConnection, window_id: str
|
||||
) -> Dict[int, np.ndarray]:
|
||||
"""Load SVD scores for a window. Returns {motion_id: score_array}."""
|
||||
rows = con.execute(
|
||||
"SELECT entity_id, vector FROM svd_vectors WHERE window_id = ? AND entity_type = 'motion'",
|
||||
[window_id],
|
||||
).fetchall()
|
||||
|
||||
result = {}
|
||||
for entity_id, raw_vec in rows:
|
||||
if isinstance(raw_vec, str):
|
||||
vec = json.loads(raw_vec)
|
||||
elif isinstance(raw_vec, (bytes, bytearray)):
|
||||
vec = json.loads(raw_vec.decode())
|
||||
elif isinstance(raw_vec, list):
|
||||
vec = raw_vec
|
||||
else:
|
||||
vec = list(raw_vec)
|
||||
result[int(entity_id)] = np.array(
|
||||
[float(v) if v is not None else 0.0 for v in vec]
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def compute_semantic_gravity_examples(
|
||||
con: duckdb.DuckDBPyConnection,
|
||||
windows: List[str],
|
||||
axis: int,
|
||||
n_examples: int = 5,
|
||||
n_components: int = 10,
|
||||
) -> Dict:
|
||||
"""Find motions closest to semantic gravity for an axis across windows."""
|
||||
comp_idx = axis - 1
|
||||
results = {}
|
||||
|
||||
for w in windows:
|
||||
# Load data
|
||||
motion_scores = _load_motion_scores(con, w)
|
||||
embeddings_data = _load_fused_embeddings_with_titles(con, w)
|
||||
|
||||
if not motion_scores or not embeddings_data:
|
||||
continue
|
||||
|
||||
# Build motion_id -> embedding mapping
|
||||
embeddings_by_id = {mid: (vec, title) for mid, vec, title in embeddings_data}
|
||||
|
||||
# Find common motions
|
||||
common = [m for m in motion_scores if m in embeddings_by_id]
|
||||
if len(common) < 10:
|
||||
continue
|
||||
|
||||
# Compute semantic gravity (weighted mean by absolute SVD score on this axis)
|
||||
valid_embeddings = []
|
||||
weights = []
|
||||
for m_id in common:
|
||||
scores = motion_scores[m_id]
|
||||
if comp_idx < len(scores):
|
||||
valid_embeddings.append(embeddings_by_id[m_id][0])
|
||||
weights.append(abs(scores[comp_idx]))
|
||||
|
||||
if not valid_embeddings or sum(weights) == 0:
|
||||
continue
|
||||
|
||||
# Align dimensions
|
||||
dim = min(len(v) for v in valid_embeddings)
|
||||
vectors = np.array([v[:dim] for v in valid_embeddings])
|
||||
weights = np.array(weights[: len(vectors)])
|
||||
gravity = np.average(vectors, axis=0, weights=weights)
|
||||
|
||||
# Find motions closest to gravity (highest cosine similarity)
|
||||
similarities = []
|
||||
for m_id in common:
|
||||
vec, title = embeddings_by_id[m_id]
|
||||
vec = vec[:dim]
|
||||
norm_g = np.linalg.norm(gravity)
|
||||
norm_v = np.linalg.norm(vec)
|
||||
if norm_g > 0 and norm_v > 0:
|
||||
sim = np.dot(gravity, vec) / (norm_g * norm_v)
|
||||
similarities.append((sim, m_id, title))
|
||||
|
||||
# Sort by similarity and get top examples
|
||||
similarities.sort(reverse=True)
|
||||
top_positive = [s for s in similarities if s[0] > 0][:n_examples]
|
||||
top_negative = [s for s in similarities if s[0] < 0][-n_examples:][::-1]
|
||||
|
||||
# Get extreme motions (highest absolute loading on this axis)
|
||||
extreme = sorted(
|
||||
common, key=lambda m: abs(motion_scores[m][comp_idx]), reverse=True
|
||||
)[:n_examples]
|
||||
extreme_motions = []
|
||||
for m_id in extreme:
|
||||
score = motion_scores[m_id][comp_idx]
|
||||
title = embeddings_by_id.get(m_id, (None, ""))[1]
|
||||
extreme_motions.append((score, m_id, title))
|
||||
|
||||
results[w] = {
|
||||
"gravity": gravity,
|
||||
"top_similar": top_positive,
|
||||
"top_dissimilar": top_negative,
|
||||
"extreme": extreme_motions,
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _get_annual_windows(con: duckdb.DuckDBPyConnection) -> List[str]:
|
||||
"""Get list of annual windows that have fused embeddings, sorted by year."""
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT DISTINCT f.window_id
|
||||
FROM fused_embeddings f
|
||||
JOIN svd_vectors s ON f.window_id = s.window_id AND s.entity_type = 'motion'
|
||||
WHERE f.window_id NOT LIKE '%-Q%'
|
||||
ORDER BY f.window_id
|
||||
"""
|
||||
).fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
def format_results(results: Dict, axis: int) -> str:
|
||||
"""Format results as markdown."""
|
||||
lines = [
|
||||
f"# Semantic Gravity Examples for Axis {axis}",
|
||||
"",
|
||||
f"Shows motions closest to semantic gravity (weighted mean embedding) for each window.",
|
||||
"This represents the 'typical' motion on this axis.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
|
||||
for window in sorted(results.keys()):
|
||||
data = results[window]
|
||||
gravity = data["gravity"]
|
||||
|
||||
lines.append(f"## {window}")
|
||||
lines.append("")
|
||||
|
||||
# Positive-pole extreme motions
|
||||
lines.append("### Extreme Positive Motions (high positive loading)")
|
||||
for score, m_id, title in data["extreme"]:
|
||||
if score > 0:
|
||||
lines.append(
|
||||
f"- **[{score:+.3f}]** {title[:100]}{'...' if len(title) > 100 else ''}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Negative-pole extreme motions
|
||||
lines.append("### Extreme Negative Motions (high negative loading)")
|
||||
for score, m_id, title in data["extreme"]:
|
||||
if score < 0:
|
||||
lines.append(
|
||||
f"- **[{score:+.3f}]** {title[:100]}{'...' if len(title) > 100 else ''}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Motions closest to semantic gravity
|
||||
lines.append("### Most Representative Motions (closest to semantic gravity)")
|
||||
for sim, m_id, title in data["top_similar"]:
|
||||
lines.append(
|
||||
f"- **[{sim:.3f}]** {title[:100]}{'...' if len(title) > 100 else ''}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main(argv: List[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Find semantic gravity examples for SVD axes"
|
||||
)
|
||||
p.add_argument("--db", default="data/motions.db", help="Path to motions database")
|
||||
p.add_argument("--axis", type=int, default=1, help="SVD axis to analyze (1-10)")
|
||||
p.add_argument(
|
||||
"--windows", nargs="+", help="Specific windows (default: all annual windows)"
|
||||
)
|
||||
p.add_argument(
|
||||
"--n-examples",
|
||||
type=int,
|
||||
default=5,
|
||||
help="Number of example motions per category",
|
||||
)
|
||||
p.add_argument("--output", help="Output file (default: print to stdout)")
|
||||
|
||||
args = p.parse_args(argv)
|
||||
|
||||
if not os.path.exists(args.db):
|
||||
print(f"Error: Database not found: {args.db}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
con = duckdb.connect(database=args.db, read_only=True)
|
||||
try:
|
||||
# Determine windows
|
||||
if args.windows:
|
||||
windows = args.windows
|
||||
else:
|
||||
windows = _get_annual_windows(con)
|
||||
print(f"Found {len(windows)} annual windows: {windows}", file=sys.stderr)
|
||||
|
||||
if len(windows) < 2:
|
||||
print("Need at least 2 windows for analysis", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Run analysis
|
||||
print(
|
||||
f"Computing semantic gravity examples for Axis {args.axis}...",
|
||||
file=sys.stderr,
|
||||
)
|
||||
results = compute_semantic_gravity_examples(
|
||||
con, windows, args.axis, args.n_examples
|
||||
)
|
||||
|
||||
# Format output
|
||||
output = format_results(results, args.axis)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "w") as f:
|
||||
f.write(output)
|
||||
print(f"Results written to {args.output}", file=sys.stderr)
|
||||
else:
|
||||
print(output)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user