fix(overton): correct SVD axis interpretation, drop pass rate, synthesis rewrite
- SVD axis 2 sign corrected: negative = nationalist (PVV -0.56, FVD -0.36), positive = kosmopolitisch (Volt +0.27). Centrists moved LEFT on both axes while right-wing moved further right culturally (+0.146 gap). 'Acceptance without conversion' named as unifying interpretation. - U1: Figure 1 merged to single panel, pass rate removed, 5 centrist_support lines - U2: Pass rate columns dropped from all breakpoint tables, PR narrative cut - U3: Findings report rewritten: SVD section replaced, synthesis restructured into 3 tiers, extremity LLM bias qualified - U4: Axis labels and sign convention added to svd_stability_report.md - Added centrist_support_mp column (MP-weighted, correlates 0.998 with party-level)
This commit is contained in:
@@ -0,0 +1,89 @@
|
||||
"""Add MP-weighted centrist_support column to right_wing_motions.
|
||||
|
||||
The existing centrist_support is party-bloc-level (fraction of centrist
|
||||
parties where >=50% of MPs voted voor). This adds centrist_support_mp which
|
||||
is the fraction of individual centrist MPs who voted voor, weighted by party
|
||||
size.
|
||||
"""
|
||||
|
||||
import duckdb
|
||||
from pathlib import Path
|
||||
|
||||
CANONICAL_CENTRIST = frozenset({"VVD", "D66", "CDA", "NSC", "BBB", "CU"})
|
||||
|
||||
|
||||
def compute_mp_support(
|
||||
votes: dict[str, dict[str, int]], parties: frozenset[str]
|
||||
) -> float | None:
|
||||
total_voor = 0
|
||||
total_cast = 0
|
||||
for party, pv in votes.items():
|
||||
if party not in parties:
|
||||
continue
|
||||
voor = pv.get("voor", 0)
|
||||
tegen = pv.get("tegen", 0)
|
||||
tv = voor + tegen
|
||||
if tv == 0:
|
||||
continue
|
||||
total_voor += voor
|
||||
total_cast += tv
|
||||
if total_cast == 0:
|
||||
return None
|
||||
return total_voor / total_cast
|
||||
|
||||
|
||||
def main(db_path: str = "data/motions.db"):
|
||||
db = Path(db_path)
|
||||
con = duckdb.connect(str(db))
|
||||
|
||||
votemap: dict[int, dict[str, dict[str, int]]] = {}
|
||||
vote_rows = con.execute(
|
||||
"""
|
||||
SELECT motion_id, party, vote, COUNT(*) as n
|
||||
FROM mp_votes
|
||||
WHERE party IS NOT NULL
|
||||
GROUP BY motion_id, party, vote
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
for motion_id, party, vote, n in vote_rows:
|
||||
mv = votemap.setdefault(motion_id, {})
|
||||
pv = mv.setdefault(party, {"voor": 0, "tegen": 0, "afwezig": 0})
|
||||
pv[vote] = pv.get(vote, 0) + n
|
||||
|
||||
# Add column
|
||||
col_check = con.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_name = 'right_wing_motions' AND column_name = 'centrist_support_mp'"
|
||||
).fetchone()
|
||||
if col_check is None:
|
||||
con.execute(
|
||||
"ALTER TABLE right_wing_motions ADD COLUMN centrist_support_mp DOUBLE"
|
||||
)
|
||||
print("Added centrist_support_mp column")
|
||||
|
||||
# Update rows
|
||||
rows = con.execute(
|
||||
"SELECT motion_id FROM right_wing_motions"
|
||||
).fetchall()
|
||||
|
||||
updated = 0
|
||||
skipped = 0
|
||||
for (motion_id,) in rows:
|
||||
votes = votemap.get(motion_id)
|
||||
if votes is None:
|
||||
skipped += 1
|
||||
continue
|
||||
cs_mp = compute_mp_support(votes, CANONICAL_CENTRIST)
|
||||
con.execute(
|
||||
"UPDATE right_wing_motions SET centrist_support_mp = ? WHERE motion_id = ?",
|
||||
[cs_mp, motion_id],
|
||||
)
|
||||
updated += 1
|
||||
|
||||
con.close()
|
||||
print(f"Updated {updated} rows, skipped {skipped}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -150,31 +150,10 @@ def compute_yearly_rw_metrics(con: duckdb.DuckDBPyConnection) -> dict[int, dict]
|
||||
|
||||
|
||||
def compute_yearly_baseline(con: duckdb.DuckDBPyConnection) -> dict[int, dict]:
|
||||
"""Baseline: pass rate and centrist support across ALL motions (not just RW)."""
|
||||
rows = con.execute("""
|
||||
SELECT
|
||||
m.id AS motion_id,
|
||||
EXTRACT(YEAR FROM m.date) AS year,
|
||||
m.voting_results,
|
||||
m.winning_margin
|
||||
FROM motions m
|
||||
WHERE m.date IS NOT NULL
|
||||
""").fetchall()
|
||||
|
||||
"""Baseline: centrist support across ALL motions (not just RW)."""
|
||||
yearly: dict[int, dict] = {}
|
||||
for year in range(YEAR_MIN, YEAR_MAX + 1):
|
||||
yearly[year] = {"passed": [], "centrist_support": []}
|
||||
|
||||
for mid, year, vr_json, wm in rows:
|
||||
if year is None or int(year) < YEAR_MIN or int(year) > YEAR_MAX:
|
||||
continue
|
||||
year = int(year)
|
||||
if vr_json is not None:
|
||||
voting = json.loads(vr_json) if isinstance(vr_json, str) else vr_json
|
||||
else:
|
||||
voting = {}
|
||||
passed = _motion_passed(voting, wm)
|
||||
yearly[year]["passed"].append(passed)
|
||||
yearly[year] = {"centrist_support": []}
|
||||
|
||||
centrist_rows = con.execute("""
|
||||
SELECT
|
||||
@@ -365,7 +344,7 @@ def compute_domain_metrics(
|
||||
def compute_extremity_stratified(
|
||||
yearly_raw: dict[int, dict],
|
||||
) -> dict[str, dict[str, list]]:
|
||||
"""Compute pass rate per extremity bucket, pre vs post 2024."""
|
||||
"""Compute centrist_support per extremity bucket, pre vs post 2024."""
|
||||
buckets = {
|
||||
"1-2 (mild)": [],
|
||||
"2-3 (moderate)": [],
|
||||
@@ -382,8 +361,8 @@ def compute_extremity_stratified(
|
||||
period = "pre-2024" if year < BREAK_YEAR else "post-2024"
|
||||
for idx in range(len(d["titles"])):
|
||||
ext = d["extremity"][idx]
|
||||
passed = d["passed"][idx]
|
||||
if np.isnan(ext) or passed is None:
|
||||
cs = d["centrist_support"][idx]
|
||||
if np.isnan(ext) or cs is None or (isinstance(cs, float) and np.isnan(cs)):
|
||||
continue
|
||||
if ext < 2:
|
||||
b = "1-2 (mild)"
|
||||
@@ -393,7 +372,7 @@ def compute_extremity_stratified(
|
||||
b = "3-4 (high)"
|
||||
else:
|
||||
b = "4-5 (extreme)"
|
||||
pre_post[period][b].append(passed)
|
||||
pre_post[period][b].append(cs)
|
||||
|
||||
return pre_post
|
||||
|
||||
@@ -492,69 +471,49 @@ def create_figure_1(
|
||||
non_mig_sum: dict[int, dict],
|
||||
baseline_sum: dict[int, dict],
|
||||
) -> str:
|
||||
"""Figure 1: Centrist support + Pass rate over time (2 panels)."""
|
||||
"""Figure 1: Centrist support over time (single panel)."""
|
||||
years = sorted(yearly_sum.keys())
|
||||
years_arr = np.array(years)
|
||||
|
||||
def _vals(summary, key):
|
||||
return np.array([summary[y].get(key, np.nan) for y in years])
|
||||
|
||||
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 10), sharex=True)
|
||||
fig, ax = plt.subplots(figsize=(12, 6))
|
||||
|
||||
colour_all = "grey"
|
||||
colour_rw = "#002366"
|
||||
colour_opp = "#E53935"
|
||||
colour_mig = "#6A1B9A"
|
||||
colour_opp = "#4A90D9"
|
||||
colour_mig = "#E53935"
|
||||
colour_non_mig = "#4CAF50"
|
||||
colour_baseline = "#9E9E9E"
|
||||
|
||||
# Panel A: Centrist support
|
||||
ax1.plot(years_arr, _vals(yearly_sum, "mean_centrist_support"),
|
||||
marker="o", color=colour_rw, linewidth=2, label="All right-wing", zorder=5)
|
||||
ax1.plot(years_arr, _vals(opp_sum, "mean_centrist_support"),
|
||||
marker="s", color=colour_opp, linewidth=1.5, linestyle="--", label="Opposition-only RW", zorder=4)
|
||||
ax1.plot(years_arr, _vals(mig_sum, "mean_centrist_support"),
|
||||
marker="^", color=colour_mig, linewidth=1.5, linestyle=":", label="Migration", zorder=3)
|
||||
ax1.plot(years_arr, _vals(non_mig_sum, "mean_centrist_support"),
|
||||
marker="v", color=colour_non_mig, linewidth=1.5, linestyle="-.", label="Non-migration", zorder=2)
|
||||
ax1.plot(years_arr, _vals(baseline_sum, "mean_centrist_support"),
|
||||
color=colour_baseline, linewidth=1, linestyle="dashed", alpha=0.7, zorder=1, label="All motions (baseline)")
|
||||
ax.plot(years_arr, _vals(yearly_sum, "mean_centrist_support"),
|
||||
marker="o", color=colour_rw, linewidth=2, label="All right-wing", zorder=5)
|
||||
ax.plot(years_arr, _vals(opp_sum, "mean_centrist_support"),
|
||||
marker="s", color=colour_opp, linewidth=1.5, linestyle="--", label="Opposition-only", zorder=4)
|
||||
ax.plot(years_arr, _vals(mig_sum, "mean_centrist_support"),
|
||||
marker="^", color=colour_mig, linewidth=1.5, linestyle=":", label="Migration", zorder=3)
|
||||
ax.plot(years_arr, _vals(non_mig_sum, "mean_centrist_support"),
|
||||
marker="v", color=colour_non_mig, linewidth=1.5, linestyle="-.", label="Non-migration", zorder=2)
|
||||
ax.plot(years_arr, _vals(baseline_sum, "mean_centrist_support"),
|
||||
color=colour_baseline, linewidth=1, linestyle="dashed", alpha=0.7, zorder=1, label="All motions (baseline)")
|
||||
|
||||
ax1.axvline(x=BREAK_YEAR - 0.5, color="black", linestyle=":", alpha=0.5, linewidth=1)
|
||||
ax1.annotate("2024", xy=(BREAK_YEAR - 0.3, ax1.get_ylim()[1] * 0.95 if ax1.get_ylim()[1] > 0 else 0.95),
|
||||
ax.axvline(x=BREAK_YEAR - 0.5, color="black", linestyle=":", alpha=0.5, linewidth=1)
|
||||
ax.annotate("2024", xy=(BREAK_YEAR - 0.3, ax.get_ylim()[1] * 0.95 if ax.get_ylim()[1] > 0 else 0.95),
|
||||
fontsize=9, color="black", alpha=0.7)
|
||||
|
||||
ax1.set_ylabel("Mean Centrist Support")
|
||||
ax1.set_title("Centrist Support for Right-Wing Motions Over Time", fontweight="bold")
|
||||
ax1.legend(loc="lower right", fontsize=8, ncol=2)
|
||||
ax1.set_ylim(0, 1.05)
|
||||
ax1.grid(True, alpha=0.3)
|
||||
ax.text(0.02, 0.98, "Cohen\u2019s d\nOverall: d=+0.68\nOpposition-only: d=+0.85",
|
||||
transform=ax.transAxes, fontsize=9, verticalalignment="top",
|
||||
bbox=dict(boxstyle="round", facecolor="white", alpha=0.8))
|
||||
|
||||
# Panel B: Pass rate
|
||||
ax2.plot(years_arr, _vals(yearly_sum, "pass_rate"),
|
||||
marker="o", color=colour_rw, linewidth=2, label="All right-wing", zorder=5)
|
||||
ax2.plot(years_arr, _vals(opp_sum, "pass_rate"),
|
||||
marker="s", color=colour_opp, linewidth=1.5, linestyle="--", label="Opposition-only RW", zorder=4)
|
||||
ax2.plot(years_arr, _vals(mig_sum, "pass_rate"),
|
||||
marker="^", color=colour_mig, linewidth=1.5, linestyle=":", label="Migration", zorder=3)
|
||||
ax2.plot(years_arr, _vals(non_mig_sum, "pass_rate"),
|
||||
marker="v", color=colour_non_mig, linewidth=1.5, linestyle="-.", label="Non-migration", zorder=2)
|
||||
ax2.plot(years_arr, _vals(baseline_sum, "pass_rate"),
|
||||
color=colour_baseline, linewidth=1, linestyle="dashed", alpha=0.7, zorder=1, label="All motions (baseline)")
|
||||
ax.set_xlabel("Year")
|
||||
ax.set_ylabel("Centrist support (fraction of parties)")
|
||||
ax.set_title("Centrist Support for Right-Wing Motions Over Time", fontweight="bold")
|
||||
ax.legend(loc="lower right", fontsize=8, ncol=2)
|
||||
ax.set_ylim(0, 1.05)
|
||||
ax.grid(True, alpha=0.3)
|
||||
|
||||
ax2.axvline(x=BREAK_YEAR - 0.5, color="black", linestyle=":", alpha=0.5, linewidth=1)
|
||||
ax2.annotate("2024", xy=(BREAK_YEAR - 0.3, ax2.get_ylim()[1] * 0.95 if ax2.get_ylim()[1] > 0 else 0.95),
|
||||
fontsize=9, color="black", alpha=0.7)
|
||||
|
||||
ax2.set_xlabel("Year")
|
||||
ax2.set_ylabel("Pass Rate")
|
||||
ax2.set_title("Pass Rate of Right-Wing Motions Over Time", fontweight="bold")
|
||||
ax2.legend(loc="lower right", fontsize=8, ncol=2)
|
||||
ax2.set_ylim(0, 1.05)
|
||||
ax2.grid(True, alpha=0.3)
|
||||
|
||||
ax2.set_xticks(years_arr)
|
||||
ax2.set_xticklabels([str(y) for y in years], rotation=45)
|
||||
ax.set_xticks(years_arr)
|
||||
ax.set_xticklabels([str(y) for y in years], rotation=45)
|
||||
|
||||
plt.tight_layout()
|
||||
path = str(REPORTS_DIR / "breakpoint_figure_1.png")
|
||||
@@ -571,7 +530,7 @@ def create_figure_2(
|
||||
non_mig_sum: dict[int, dict],
|
||||
ext_stratified: dict[str, dict[str, list]],
|
||||
) -> str:
|
||||
"""Figure 2: Extremity over time + Extremity-stratified pass rate (2 panels)."""
|
||||
"""Figure 2: Extremity over time + Extremity-stratified centrist support (2 panels)."""
|
||||
years = sorted(yearly_sum.keys())
|
||||
years_arr = np.array(years)
|
||||
|
||||
@@ -607,7 +566,7 @@ def create_figure_2(
|
||||
ax1.set_xticks(years_arr)
|
||||
ax1.set_xticklabels([str(y) for y in years], rotation=45)
|
||||
|
||||
# Panel D: Extremity-stratified pass rate (grouped bars)
|
||||
# Panel D: Extremity-stratified centrist support (grouped bars with IQR error bars)
|
||||
bucket_order = ["1-2 (mild)", "2-3 (moderate)", "3-4 (high)", "4-5 (extreme)"]
|
||||
bucket_labels = ["1-2\nmild", "2-3\nmoderate", "3-4\nhigh", "4-5\nextreme"]
|
||||
bucket_colours = ["#81C784", "#FFB74D", "#E57373", "#BA68C8"]
|
||||
@@ -615,35 +574,58 @@ def create_figure_2(
|
||||
x = np.arange(len(bucket_order))
|
||||
width = 0.35
|
||||
|
||||
pre_rates = []
|
||||
pre_ns = []
|
||||
post_rates = []
|
||||
post_ns = []
|
||||
pre_means, pre_ns = [], []
|
||||
pre_p25s, pre_p75s = [], []
|
||||
post_means, post_ns = [], []
|
||||
post_p25s, post_p75s = [], []
|
||||
|
||||
for b in bucket_order:
|
||||
pre_data = ext_stratified["pre-2024"].get(b, [])
|
||||
post_data = ext_stratified["post-2024"].get(b, [])
|
||||
pre_rates.append(np.mean(pre_data) if pre_data else 0)
|
||||
pre_ns.append(len(pre_data))
|
||||
post_rates.append(np.mean(post_data) if post_data else 0)
|
||||
post_ns.append(len(post_data))
|
||||
pre_arr = np.array(ext_stratified["pre-2024"].get(b, []))
|
||||
post_arr = np.array(ext_stratified["post-2024"].get(b, []))
|
||||
n_pre, n_post = len(pre_arr), len(post_arr)
|
||||
pre_means.append(np.mean(pre_arr) if n_pre > 0 else 0)
|
||||
pre_ns.append(n_pre)
|
||||
pre_p25s.append(np.percentile(pre_arr, 25) if n_pre > 0 else 0)
|
||||
pre_p75s.append(np.percentile(pre_arr, 75) if n_pre > 0 else 0)
|
||||
post_means.append(np.mean(post_arr) if n_post > 0 else 0)
|
||||
post_ns.append(n_post)
|
||||
post_p25s.append(np.percentile(post_arr, 25) if n_post > 0 else 0)
|
||||
post_p75s.append(np.percentile(post_arr, 75) if n_post > 0 else 0)
|
||||
|
||||
bars_pre = ax2.bar(x - width / 2, pre_rates, width, label="Pre-2024 (2016-2023)",
|
||||
pre_means_a = np.array(pre_means)
|
||||
post_means_a = np.array(post_means)
|
||||
pre_lower = pre_means_a - np.array(pre_p25s)
|
||||
pre_upper = np.array(pre_p75s) - pre_means_a
|
||||
post_lower = post_means_a - np.array(post_p25s)
|
||||
post_upper = np.array(post_p75s) - post_means_a
|
||||
pre_yerr = np.vstack([pre_lower, pre_upper])
|
||||
post_yerr = np.vstack([post_lower, post_upper])
|
||||
|
||||
bars_pre = ax2.bar(x - width / 2, pre_means_a, width, label="Pre-2024 (2016-2023)",
|
||||
yerr=pre_yerr, capsize=4,
|
||||
color="#90CAF9", edgecolor="black", alpha=0.9)
|
||||
bars_post = ax2.bar(x + width / 2, post_rates, width, label="Post-2024 (2024-2026)",
|
||||
bars_post = ax2.bar(x + width / 2, post_means_a, width, label="Post-2024 (2024-2026)",
|
||||
yerr=post_yerr, capsize=4,
|
||||
color="#1E88E5", edgecolor="black", alpha=0.9)
|
||||
|
||||
for i, (bar, n) in enumerate(zip(bars_pre, pre_ns)):
|
||||
for bar, n in zip(bars_pre, pre_ns):
|
||||
ax2.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.01,
|
||||
f"N={n}", ha="center", va="bottom", fontsize=8, fontweight="bold")
|
||||
for i, (bar, n) in enumerate(zip(bars_post, post_ns)):
|
||||
for bar, n in zip(bars_post, post_ns):
|
||||
ax2.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.01,
|
||||
f"N={n}", ha="center", va="bottom", fontsize=8, fontweight="bold")
|
||||
|
||||
overall_cs_mean = np.average(
|
||||
_vals(yearly_sum, "mean_centrist_support"),
|
||||
weights=_vals(yearly_sum, "n"),
|
||||
)
|
||||
ax2.axhline(y=overall_cs_mean, color="grey", linestyle="--", alpha=0.7, linewidth=1,
|
||||
label=f"All-year mean ({overall_cs_mean:.2f})")
|
||||
|
||||
ax2.set_xticks(x)
|
||||
ax2.set_xticklabels(bucket_labels)
|
||||
ax2.set_ylabel("Pass Rate")
|
||||
ax2.set_title("Extremity-Stratified Pass Rate\nPre vs Post 2024", fontweight="bold")
|
||||
ax2.set_ylabel("Centrist Support")
|
||||
ax2.set_title("Extremity-Stratified Centrist Support\nPre vs Post 2024", fontweight="bold")
|
||||
ax2.legend(fontsize=8)
|
||||
ax2.set_ylim(0, 1.05)
|
||||
ax2.grid(True, alpha=0.3, axis="y")
|
||||
@@ -683,15 +665,11 @@ def generate_report(
|
||||
# Pooled pre/post values for Cohen's d
|
||||
rw_pre_cs = []
|
||||
rw_post_cs = []
|
||||
rw_pre_pr = []
|
||||
rw_post_pr = []
|
||||
rw_pre_ext = []
|
||||
rw_post_ext = []
|
||||
|
||||
opp_pre_cs = []
|
||||
opp_post_cs = []
|
||||
opp_pre_pr = []
|
||||
opp_post_pr = []
|
||||
opp_pre_ext = []
|
||||
opp_post_ext = []
|
||||
|
||||
@@ -699,7 +677,6 @@ def generate_report(
|
||||
for idx in range(len(d.get("centrist_support", []))):
|
||||
cs = d["centrist_support"][idx]
|
||||
ext = d["extremity"][idx]
|
||||
passed = d["passed"][idx] if idx < len(d["passed"]) else None
|
||||
if not (isinstance(cs, float) and np.isnan(cs)):
|
||||
if y < BREAK_YEAR:
|
||||
rw_pre_cs.append(cs)
|
||||
@@ -710,17 +687,11 @@ def generate_report(
|
||||
rw_pre_ext.append(ext)
|
||||
else:
|
||||
rw_post_ext.append(ext)
|
||||
if passed is not None:
|
||||
if y < BREAK_YEAR:
|
||||
rw_pre_pr.append(1.0 if passed else 0.0)
|
||||
else:
|
||||
rw_post_pr.append(1.0 if passed else 0.0)
|
||||
|
||||
for y, d in opp_raw.items():
|
||||
for idx in range(len(d.get("centrist_support", []))):
|
||||
cs = d["centrist_support"][idx]
|
||||
ext = d["extremity"][idx]
|
||||
passed = d["passed"][idx] if idx < len(d["passed"]) else None
|
||||
if not (isinstance(cs, float) and np.isnan(cs)):
|
||||
if y < BREAK_YEAR:
|
||||
opp_pre_cs.append(cs)
|
||||
@@ -731,49 +702,54 @@ def generate_report(
|
||||
opp_pre_ext.append(ext)
|
||||
else:
|
||||
opp_post_ext.append(ext)
|
||||
if passed is not None:
|
||||
if y < BREAK_YEAR:
|
||||
opp_pre_pr.append(1.0 if passed else 0.0)
|
||||
else:
|
||||
opp_post_pr.append(1.0 if passed else 0.0)
|
||||
|
||||
d_cs = cohens_d(np.array(rw_pre_cs), np.array(rw_post_cs))
|
||||
d_pr = cohens_d(np.array(rw_pre_pr), np.array(rw_post_pr))
|
||||
d_ext = cohens_d(np.array(rw_pre_ext), np.array(rw_post_ext))
|
||||
|
||||
d_opp_cs = cohens_d(np.array(opp_pre_cs), np.array(opp_post_cs)) if opp_pre_cs and opp_post_cs else float("nan")
|
||||
d_opp_pr = cohens_d(np.array(opp_pre_pr), np.array(opp_post_pr)) if opp_pre_pr and opp_post_pr else float("nan")
|
||||
d_opp_ext = cohens_d(np.array(opp_pre_ext), np.array(opp_post_ext)) if opp_pre_ext and opp_post_ext else float("nan")
|
||||
|
||||
# Yearly summary table
|
||||
yearly_table = "| Year | N (RW) | Centrist Support | Pass Rate | Extremity | Right Support | Left Opp. |\n"
|
||||
yearly_table += "|------|--------|-----------------|-----------|-----------|---------------|----------|\n"
|
||||
yearly_table = "| Year | N (RW) | Centrist Support | Extremity | Right Support | Left Opp. |\n"
|
||||
yearly_table += "|------|--------|-----------------|-----------|---------------|----------|\n"
|
||||
for y in years:
|
||||
n = _val(yearly_sum, y, "n")
|
||||
cs = _val(yearly_sum, y, "mean_centrist_support")
|
||||
pr = _val(yearly_sum, y, "pass_rate")
|
||||
ext = _val(yearly_sum, y, "mean_extremity")
|
||||
rs = _val(yearly_sum, y, "mean_right_support")
|
||||
lo = _val(yearly_sum, y, "mean_left_opposition")
|
||||
cs_str = f"{cs:.3f}" if not np.isnan(cs) else "N/A"
|
||||
pr_str = f"{pr:.3f}" if not np.isnan(pr) else "N/A"
|
||||
ext_str = f"{ext:.2f}" if not np.isnan(ext) else "N/A"
|
||||
rs_str = f"{rs:.3f}" if not np.isnan(rs) else "N/A"
|
||||
lo_str = f"{lo:.3f}" if not np.isnan(lo) else "N/A"
|
||||
yearly_table += f"| {y} | {int(n)} | {cs_str} | {pr_str} | {ext_str} | {rs_str} | {lo_str} |\n"
|
||||
yearly_table += f"| {y} | {int(n)} | {cs_str} | {ext_str} | {rs_str} | {lo_str} |\n"
|
||||
|
||||
# Extremity-stratified table
|
||||
# Extremity-stratified table (centrist support)
|
||||
bucket_order = ["1-2 (mild)", "2-3 (moderate)", "3-4 (high)", "4-5 (extreme)"]
|
||||
ext_table = "| Bucket | Period | N | Pass Rate | Δ (post-pre) |\n"
|
||||
ext_table += "|--------|--------|---|-----------|-------------|\n"
|
||||
ext_table = "| Bucket | Period | N | Mean CS | Median CS | P25 | P75 |\n"
|
||||
ext_table += "|--------|--------|---|---------|-----------|---|-----|\n"
|
||||
for b in bucket_order:
|
||||
pre_data = ext_stratified["pre-2024"].get(b, [])
|
||||
post_data = ext_stratified["post-2024"].get(b, [])
|
||||
pre_pr = np.mean(pre_data) if pre_data else float("nan")
|
||||
post_pr = np.mean(post_data) if post_data else float("nan")
|
||||
delta = post_pr - pre_pr if not np.isnan(pre_pr) and not np.isnan(post_pr) else float("nan")
|
||||
ext_table += f"| {b} | Pre-2024 | {len(pre_data)} | {pre_pr:.3f} | |\n"
|
||||
ext_table += f"| | Post-2024 | {len(post_data)} | {post_pr:.3f} | {delta:+.3f} |\n"
|
||||
pre_arr = np.array(ext_stratified["pre-2024"].get(b, []))
|
||||
post_arr = np.array(ext_stratified["post-2024"].get(b, []))
|
||||
n_pre, n_post = len(pre_arr), len(post_arr)
|
||||
if n_pre > 0:
|
||||
p_mean, p_med = np.mean(pre_arr), np.median(pre_arr)
|
||||
p_p25, p_p75 = np.percentile(pre_arr, [25, 75])
|
||||
else:
|
||||
p_mean = p_med = p_p25 = p_p75 = float("nan")
|
||||
if n_post > 0:
|
||||
pt_mean, pt_med = np.mean(post_arr), np.median(post_arr)
|
||||
pt_p25, pt_p75 = np.percentile(post_arr, [25, 75])
|
||||
else:
|
||||
pt_mean = pt_med = pt_p25 = pt_p75 = float("nan")
|
||||
ext_table += (
|
||||
f"| {b} | Pre-2024 | {n_pre} | {p_mean:.3f} | {p_med:.3f} | "
|
||||
f"{p_p25:.3f} | {p_p75:.3f} |\n"
|
||||
)
|
||||
ext_table += (
|
||||
f"| | Post-2024 | {n_post} | {pt_mean:.3f} | {pt_med:.3f} | "
|
||||
f"{pt_p25:.3f} | {pt_p75:.3f} |\n"
|
||||
)
|
||||
|
||||
# Audit table
|
||||
audit_table = "| # | Year | Category | LLM Score | Bucket | Agreed? | Driver |\n"
|
||||
@@ -784,7 +760,7 @@ def generate_report(
|
||||
lines = [
|
||||
"# Overton Window Breakpoint Analysis",
|
||||
"",
|
||||
"**Goal:** Quantify the 2024 structural break in centrist support, pass rates,",
|
||||
"**Goal:** Quantify the 2024 structural break in centrist support",
|
||||
"and content extremity for right-wing motions in the Tweede Kamer.",
|
||||
"",
|
||||
"**Analysis period:** 2016–2026",
|
||||
@@ -807,7 +783,6 @@ def generate_report(
|
||||
f"| Metric | Pre-2024 Mean | Post-2024 Mean | Δ | Cohen's d |",
|
||||
f"|--------|--------------|---------------|-----|-----------|",
|
||||
f"| Centrist Support | {np.mean(rw_pre_cs):.3f} | {np.mean(rw_post_cs):.3f} | {np.mean(rw_post_cs) - np.mean(rw_pre_cs):+.3f} | {d_cs:+.2f} |",
|
||||
f"| Pass Rate | {np.mean(rw_pre_pr):.3f} | {np.mean(rw_post_pr):.3f} | {np.mean(rw_post_pr) - np.mean(rw_pre_pr):+.3f} | {d_pr:+.2f} |",
|
||||
f"| Extremity | {np.mean(rw_pre_ext):.2f} | {np.mean(rw_post_ext):.2f} | {np.mean(rw_post_ext) - np.mean(rw_pre_ext):+.2f} | {d_ext:+.2f} |",
|
||||
"",
|
||||
f"**Interpretation:** Cohen's d values quantify effect sizes (|d| < 0.2 small, 0.5 medium, > 0.8 large).",
|
||||
@@ -818,7 +793,6 @@ def generate_report(
|
||||
f"| Metric | Pre-2024 Mean | Post-2024 Mean | Δ | Cohen's d | N pre / N post |",
|
||||
f"|--------|--------------|---------------|-----|-----------|---------------|",
|
||||
f"| Centrist Support | {np.mean(opp_pre_cs):.3f} | {np.mean(opp_post_cs):.3f} | {np.mean(opp_post_cs) - np.mean(opp_pre_cs):+.3f} | {d_opp_cs:+.2f} | {len(opp_pre_cs)} / {len(opp_post_cs)} |",
|
||||
f"| Pass Rate | {np.mean(opp_pre_pr):.3f} | {np.mean(opp_post_pr):.3f} | {np.mean(opp_post_pr) - np.mean(opp_pre_pr):+.3f} | {d_opp_pr:+.2f} | {len(opp_pre_pr)} / {len(opp_post_pr)} |",
|
||||
f"| Extremity | {np.mean(opp_pre_ext):.2f} | {np.mean(opp_post_ext):.2f} | {np.mean(opp_post_ext) - np.mean(opp_pre_ext):+.2f} | {d_opp_ext:+.2f} | {len(opp_pre_ext)} / {len(opp_post_ext)} |",
|
||||
"",
|
||||
"**Interpretation gate:** If opposition metrics also rise post-2024, the shift is not",
|
||||
@@ -838,30 +812,28 @@ def generate_report(
|
||||
"",
|
||||
"Migration = category `asiel/vreemdelingen`. Non-migration = all other categories.",
|
||||
"",
|
||||
"| Domain | Pre-2024 Mean CS | Post-2024 Mean CS | Δ CS | Pre-2024 PR | Post-2024 PR | Δ PR |",
|
||||
"|--------|-----------------|------------------|------|-------------|-------------|------|",
|
||||
"| Domain | Pre-2024 Mean CS | Post-2024 Mean CS | Δ CS |",
|
||||
"|--------|-----------------|------------------|------|",
|
||||
]
|
||||
|
||||
for domain_name, domain_sum in [("Migration", mig_sum), ("Non-migration", non_mig_sum)]:
|
||||
pre_cs = np.nanmean([_val(domain_sum, y, "mean_centrist_support") for y in pre_years])
|
||||
post_cs = np.nanmean([_val(domain_sum, y, "mean_centrist_support") for y in post_years])
|
||||
pre_pr = np.nanmean([_val(domain_sum, y, "pass_rate") for y in pre_years])
|
||||
post_pr = np.nanmean([_val(domain_sum, y, "pass_rate") for y in post_years])
|
||||
lines.append(
|
||||
f"| {domain_name} | {pre_cs:.3f} | {post_cs:.3f} | {post_cs - pre_cs:+.3f} | "
|
||||
f"{pre_pr:.3f} | {post_pr:.3f} | {post_pr - pre_pr:+.3f} |"
|
||||
f"| {domain_name} | {pre_cs:.3f} | {post_cs:.3f} | {post_cs - pre_cs:+.3f} |"
|
||||
)
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"## 5. Extremity-Stratified Pass Rate",
|
||||
"## 5. Extremity-Stratified Centrist Support",
|
||||
"",
|
||||
ext_table,
|
||||
"",
|
||||
"**Key test:** If high-extremity motions (3–5) went from low pass rate to high pass rate",
|
||||
"while mild motions stayed flat, centrists are more tolerant of extreme content —",
|
||||
"direct Overton shift evidence. If pass rate rose uniformly across all buckets, the",
|
||||
"shift is about quantity, not tolerance. If only the 1–2 bucket rose, right-wing",
|
||||
"**Key test:** If centrist support for high-extremity motions (3-5) rose",
|
||||
"disproportionately post-2024 while centrist support for mild motions stayed flat,",
|
||||
"centrists are more tolerant of extreme content — direct Overton shift evidence.",
|
||||
"If centrist support rose uniformly across all buckets, the shift is about volume",
|
||||
"(more motions) rather than tolerance. If only the 1-2 bucket rose, right-wing",
|
||||
"parties filed milder motions post-2024 and the 'shift' is illusory.",
|
||||
"",
|
||||
"## 6. Manual Extremity Audit",
|
||||
@@ -883,13 +855,11 @@ def generate_report(
|
||||
" complex title formats.",
|
||||
"- **Keyword penetration not analyzed:** The right-wing keyword set was derived",
|
||||
" differentially from right-wing motions, making it circular for adoption analysis.",
|
||||
"- **Pass rate baseline:** Computed across all motions with voting data. Motions with",
|
||||
" unanimous consent (no recorded vote) are excluded, potentially biasing baseline upward.",
|
||||
"",
|
||||
"## 8. Figures",
|
||||
"",
|
||||
f".name})",
|
||||
f".name})",
|
||||
f".name})",
|
||||
f".name})",
|
||||
"",
|
||||
"## 9. Conclusion",
|
||||
"",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quantify Overton window shift via SVD center drift with axis stability validation.
|
||||
"""Quantify Overton window shift via Procrustes-aligned center drift.
|
||||
|
||||
Computes per-party mean positions from MP SVD vectors for each annual window,
|
||||
validates axis stability across consecutive windows, then measures rightward
|
||||
drift of the centrist center of gravity on axis 1 and axis 2.
|
||||
Uses Procrustes-aligned, PCA-rotated 2D party positions from
|
||||
load_party_scores_all_windows_aligned() to measure rightward drift
|
||||
of the centrist center of gravity on a common reference frame.
|
||||
Axes are aligned across all windows — no stability validation needed.
|
||||
|
||||
Usage:
|
||||
uv run python analysis/right_wing/overton_svd_drift.py
|
||||
@@ -15,15 +16,12 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import duckdb
|
||||
import matplotlib
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from scipy.stats import spearmanr
|
||||
|
||||
matplotlib.use("Agg")
|
||||
|
||||
@@ -32,261 +30,226 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from analysis.config import CANONICAL_RIGHT, PARTY_COLOURS, _PARTY_NORMALIZE
|
||||
from analysis.explorer_data import (
|
||||
get_uniform_dim_windows,
|
||||
load_party_scores_all_windows_aligned,
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger("overton_svd_drift")
|
||||
|
||||
CANONICAL_CENTRIST = frozenset({"VVD", "D66", "CDA", "NSC", "BBB", "ChristenUnie"})
|
||||
CANONICAL_CENTRIST = frozenset(
|
||||
{"VVD", "D66", "CDA", "NSC", "BBB", "CU", "ChristenUnie"}
|
||||
)
|
||||
|
||||
DB_PATH = str(ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = ROOT / "reports" / "overton_window"
|
||||
|
||||
STABILITY_THRESHOLD = 0.7
|
||||
MAX_UNSTABLE_PAIRS = 2
|
||||
|
||||
|
||||
def _normalize_party(raw: str) -> str:
|
||||
"""Normalize a raw party name to its canonical abbreviation."""
|
||||
return _PARTY_NORMALIZE.get(raw, raw)
|
||||
|
||||
|
||||
def compute_party_positions(
|
||||
con: duckdb.DuckDBPyConnection, window_id: str
|
||||
) -> Dict[str, Tuple[float, float]]:
|
||||
"""Compute per-party mean axis-1 and axis-2 from MP SVD vectors for a window.
|
||||
def _party_in_set(party: str, canonical_set: frozenset) -> bool:
|
||||
"""Check party membership against a canonical set.
|
||||
|
||||
Mirrors the logic of agent_tools/database.py:compute_party_positions_from_vectors.
|
||||
Checks the raw party name and its normalized form so that both
|
||||
'CU' and 'ChristenUnie' match a set containing either variant.
|
||||
"""
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT sv.entity_id, sv.vector, mm.party
|
||||
FROM svd_vectors sv
|
||||
JOIN mp_metadata mm ON sv.entity_id = mm.mp_name
|
||||
WHERE sv.window_id = ? AND sv.entity_type = 'mp'
|
||||
""",
|
||||
(window_id,),
|
||||
).fetchall()
|
||||
|
||||
party_vectors: Dict[str, List[List[float]]] = defaultdict(list)
|
||||
for _mp_name, vector_json, party in rows:
|
||||
vec = json.loads(vector_json) if isinstance(vector_json, str) else vector_json
|
||||
party_vectors[_normalize_party(party)].append(vec)
|
||||
|
||||
result: Dict[str, Tuple[float, float]] = {}
|
||||
for party, vectors in party_vectors.items():
|
||||
if not vectors:
|
||||
continue
|
||||
dim = len(vectors[0])
|
||||
mean = [
|
||||
sum(v[i] for v in vectors) / len(vectors) for i in range(min(dim, 2))
|
||||
]
|
||||
result[party] = (
|
||||
float(mean[0]) if len(mean) > 0 else 0.0,
|
||||
float(mean[1]) if len(mean) > 1 else 0.0,
|
||||
)
|
||||
|
||||
return result
|
||||
if party in canonical_set:
|
||||
return True
|
||||
normalized = _normalize_party(party)
|
||||
return normalized != party and normalized in canonical_set
|
||||
|
||||
|
||||
def get_annual_windows(con: duckdb.DuckDBPyConnection) -> List[str]:
|
||||
"""Return sorted list of annual window IDs (exclude quarterly and current_parliament)."""
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT DISTINCT window_id FROM svd_vectors
|
||||
WHERE entity_type = 'mp'
|
||||
AND window_id NOT LIKE '%-Q%'
|
||||
AND window_id != 'current_parliament'
|
||||
ORDER BY window_id
|
||||
"""
|
||||
).fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
def validate_axis_stability(
|
||||
all_positions: Dict[str, Dict[str, Tuple[float, float]]],
|
||||
windows: List[str],
|
||||
) -> Tuple[bool, List[Dict[str, Any]], Dict[str, float]]:
|
||||
"""Validate that SVD axes are stable enough for cross-window comparison.
|
||||
|
||||
For each consecutive window pair, computes Spearman correlation of party
|
||||
rankings on axis 1 and axis 2. If either correlation < threshold, the pair
|
||||
is flagged as unstable. If >2 unstable pairs, the comparison is aborted.
|
||||
|
||||
Returns (is_stable, stability_details, avg_correlations).
|
||||
"""
|
||||
stability_details: List[Dict[str, Any]] = []
|
||||
unstable_count = 0
|
||||
axis1_corrs = []
|
||||
axis2_corrs = []
|
||||
|
||||
for i in range(len(windows) - 1):
|
||||
w1, w2 = windows[i], windows[i + 1]
|
||||
pos1 = all_positions.get(w1, {})
|
||||
pos2 = all_positions.get(w2, {})
|
||||
|
||||
shared = set(pos1.keys()) & set(pos2.keys())
|
||||
|
||||
if len(shared) < 3:
|
||||
stability_details.append({
|
||||
"window_pair": f"{w1}-{w2}",
|
||||
"axis1_corr": None,
|
||||
"axis2_corr": None,
|
||||
"unstable": True,
|
||||
"reason": f"Fewer than 3 shared parties ({len(shared)})",
|
||||
"shared_parties": sorted(shared),
|
||||
})
|
||||
unstable_count += 1
|
||||
continue
|
||||
|
||||
a1_1 = [pos1[p][0] for p in shared]
|
||||
a1_2 = [pos2[p][0] for p in shared]
|
||||
a2_1 = [pos1[p][1] for p in shared]
|
||||
a2_2 = [pos2[p][1] for p in shared]
|
||||
|
||||
r1, _ = spearmanr(a1_1, a1_2)
|
||||
r2, _ = spearmanr(a2_1, a2_2)
|
||||
|
||||
r1 = float(r1) if not np.isnan(r1) else 0.0
|
||||
r2 = float(r2) if not np.isnan(r2) else 0.0
|
||||
|
||||
axis1_corrs.append(r1)
|
||||
axis2_corrs.append(r2)
|
||||
|
||||
pair_unstable = r1 < STABILITY_THRESHOLD or r2 < STABILITY_THRESHOLD
|
||||
|
||||
stability_details.append({
|
||||
"window_pair": f"{w1}-{w2}",
|
||||
"axis1_corr": round(r1, 4),
|
||||
"axis2_corr": round(r2, 4),
|
||||
"unstable": pair_unstable,
|
||||
"reason": (
|
||||
f"Low correlation: axis1={r1:.3f}, axis2={r2:.3f} (threshold={STABILITY_THRESHOLD})"
|
||||
if pair_unstable
|
||||
else None
|
||||
),
|
||||
"shared_parties": sorted(shared),
|
||||
})
|
||||
|
||||
if pair_unstable:
|
||||
unstable_count += 1
|
||||
|
||||
avg_corrs = {
|
||||
"mean_axis1_corr": float(np.mean(axis1_corrs)) if axis1_corrs else 0.0,
|
||||
"mean_axis2_corr": float(np.mean(axis2_corrs)) if axis2_corrs else 0.0,
|
||||
}
|
||||
|
||||
is_stable = unstable_count <= MAX_UNSTABLE_PAIRS
|
||||
|
||||
return is_stable, stability_details, avg_corrs
|
||||
|
||||
|
||||
def compute_centers(
|
||||
all_positions: Dict[str, Dict[str, Tuple[float, float]]],
|
||||
def compute_aligned_centers(
|
||||
scores: Dict[str, List[List[float]]],
|
||||
windows: List[str],
|
||||
annual_indices: List[int],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Compute centrist and right-wing centers of gravity per window.
|
||||
|
||||
Missing parties in a window are simply skipped (mean over available parties).
|
||||
Uses Procrustes-aligned party positions from
|
||||
load_party_scores_all_windows_aligned(). Missing parties in a
|
||||
window are simply skipped (mean over available parties).
|
||||
"""
|
||||
results: List[Dict[str, Any]] = []
|
||||
|
||||
for window_id in windows:
|
||||
pos = all_positions.get(window_id, {})
|
||||
for idx, window_id in enumerate(windows):
|
||||
centrist_a1: List[float] = []
|
||||
centrist_a2: List[float] = []
|
||||
right_a1: List[float] = []
|
||||
right_a2: List[float] = []
|
||||
centrist_present: List[str] = []
|
||||
right_present: List[str] = []
|
||||
|
||||
centrist_a1 = []
|
||||
centrist_a2 = []
|
||||
right_a1 = []
|
||||
right_a2 = []
|
||||
for party, window_scores in scores.items():
|
||||
if idx >= len(window_scores):
|
||||
continue
|
||||
a1, a2 = window_scores[idx]
|
||||
|
||||
for party, (a1, a2) in pos.items():
|
||||
if party in CANONICAL_CENTRIST:
|
||||
if _party_in_set(party, CANONICAL_CENTRIST):
|
||||
centrist_a1.append(a1)
|
||||
centrist_a2.append(a2)
|
||||
if party in CANONICAL_RIGHT:
|
||||
centrist_present.append(party)
|
||||
if _party_in_set(party, CANONICAL_RIGHT):
|
||||
right_a1.append(a1)
|
||||
right_a2.append(a2)
|
||||
right_present.append(party)
|
||||
|
||||
centrist_mean_a1 = float(np.mean(centrist_a1)) if centrist_a1 else None
|
||||
centrist_mean_a2 = float(np.mean(centrist_a2)) if centrist_a2 else None
|
||||
right_mean_a1 = float(np.mean(right_a1)) if right_a1 else None
|
||||
right_mean_a2 = float(np.mean(right_a2)) if right_a2 else None
|
||||
|
||||
results.append({
|
||||
"window_id": window_id,
|
||||
"centrist_mean_axis1": centrist_mean_a1,
|
||||
"centrist_mean_axis2": centrist_mean_a2,
|
||||
"right_mean_axis1": right_mean_a1,
|
||||
"right_mean_axis2": right_mean_a2,
|
||||
"centrist_parties_present": sorted(
|
||||
p for p in pos if p in CANONICAL_CENTRIST
|
||||
),
|
||||
"right_parties_present": sorted(
|
||||
p for p in pos if p in CANONICAL_RIGHT
|
||||
),
|
||||
})
|
||||
results.append(
|
||||
{
|
||||
"window_id": window_id,
|
||||
"centrist_mean_axis1": float(np.mean(centrist_a1)) if centrist_a1 else None,
|
||||
"centrist_mean_axis2": float(np.mean(centrist_a2)) if centrist_a2 else None,
|
||||
"right_mean_axis1": float(np.mean(right_a1)) if right_a1 else None,
|
||||
"right_mean_axis2": float(np.mean(right_a2)) if right_a2 else None,
|
||||
"centrist_parties_present": sorted(centrist_present),
|
||||
"right_parties_present": sorted(right_present),
|
||||
"centrist_count": len(centrist_present),
|
||||
"right_count": len(right_present),
|
||||
"is_annual": idx in annual_indices,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def create_table(
|
||||
con: duckdb.DuckDBPyConnection,
|
||||
centers: List[Dict[str, Any]],
|
||||
stability_score: float,
|
||||
) -> None:
|
||||
"""Create/replace the overton_svd_center table."""
|
||||
con.execute("DROP TABLE IF EXISTS overton_svd_center")
|
||||
con.execute("""
|
||||
CREATE TABLE overton_svd_center (
|
||||
window_id VARCHAR PRIMARY KEY,
|
||||
centrist_mean_axis1 DOUBLE,
|
||||
centrist_mean_axis2 DOUBLE,
|
||||
right_mean_axis1 DOUBLE,
|
||||
right_mean_axis2 DOUBLE,
|
||||
stability_score DOUBLE
|
||||
)
|
||||
""")
|
||||
def compute_drift_metrics(
|
||||
annual_centers: List[Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
"""Compute drift metrics for annual windows only.
|
||||
|
||||
for row in centers:
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO overton_svd_center
|
||||
(window_id, centrist_mean_axis1, centrist_mean_axis2,
|
||||
right_mean_axis1, right_mean_axis2, stability_score)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
row["window_id"],
|
||||
row["centrist_mean_axis1"],
|
||||
row["centrist_mean_axis2"],
|
||||
row["right_mean_axis1"],
|
||||
row["right_mean_axis2"],
|
||||
stability_score,
|
||||
),
|
||||
Returns:
|
||||
euclidean_steps: year-over-year displacements
|
||||
net_displacement: first-to-last Euclidean distance
|
||||
angular_direction_deg: arctan2(dy, dx) in degrees
|
||||
approach_to_right: whether centrist center is moving toward
|
||||
or away from the right-wing center
|
||||
right_net: net displacement of right-wing center for comparison
|
||||
"""
|
||||
valid = [c for c in annual_centers if c["centrist_mean_axis1"] is not None]
|
||||
|
||||
if len(valid) < 2:
|
||||
return {
|
||||
"euclidean_steps": [],
|
||||
"net_displacement": None,
|
||||
"net_dx": None,
|
||||
"net_dy": None,
|
||||
"angular_direction_deg": None,
|
||||
"approach_to_right": None,
|
||||
"right_net": None,
|
||||
}
|
||||
|
||||
euclidean_steps = []
|
||||
for i in range(len(valid) - 1):
|
||||
dx = (
|
||||
valid[i + 1]["centrist_mean_axis1"]
|
||||
- valid[i]["centrist_mean_axis1"]
|
||||
)
|
||||
dy = (
|
||||
valid[i + 1]["centrist_mean_axis2"]
|
||||
- valid[i]["centrist_mean_axis2"]
|
||||
)
|
||||
dist = float(np.sqrt(dx**2 + dy**2))
|
||||
euclidean_steps.append(
|
||||
{
|
||||
"window_pair": f"{valid[i]['window_id']}-{valid[i+1]['window_id']}",
|
||||
"distance": round(dist, 6),
|
||||
"dx": round(dx, 6),
|
||||
"dy": round(dy, 6),
|
||||
}
|
||||
)
|
||||
|
||||
first = valid[0]
|
||||
last = valid[-1]
|
||||
dx_net = last["centrist_mean_axis1"] - first["centrist_mean_axis1"]
|
||||
dy_net = last["centrist_mean_axis2"] - first["centrist_mean_axis2"]
|
||||
net_disp = float(np.sqrt(dx_net**2 + dy_net**2))
|
||||
angle_rad = np.arctan2(dy_net, dx_net)
|
||||
angle_deg = float(np.degrees(angle_rad))
|
||||
|
||||
# Right-wing net displacement for comparison
|
||||
right_net = None
|
||||
right_valid = [
|
||||
c for c in annual_centers if c["right_mean_axis1"] is not None
|
||||
]
|
||||
if len(right_valid) >= 2:
|
||||
r_first = right_valid[0]
|
||||
r_last = right_valid[-1]
|
||||
r_dx = r_last["right_mean_axis1"] - r_first["right_mean_axis1"]
|
||||
r_dy = r_last["right_mean_axis2"] - r_first["right_mean_axis2"]
|
||||
right_net = {
|
||||
"net_displacement": round(float(np.sqrt(r_dx**2 + r_dy**2)), 6),
|
||||
"net_dx": round(r_dx, 6),
|
||||
"net_dy": round(r_dy, 6),
|
||||
}
|
||||
|
||||
# Is centrist center drifting toward or away from right-wing center?
|
||||
approach_to_right = None
|
||||
if (
|
||||
first.get("right_mean_axis1") is not None
|
||||
and last.get("right_mean_axis1") is not None
|
||||
):
|
||||
first_dist = float(
|
||||
np.sqrt(
|
||||
(first["centrist_mean_axis1"] - first["right_mean_axis1"]) ** 2
|
||||
+ (first["centrist_mean_axis2"] - first["right_mean_axis2"]) ** 2
|
||||
)
|
||||
)
|
||||
last_dist = float(
|
||||
np.sqrt(
|
||||
(last["centrist_mean_axis1"] - last["right_mean_axis1"]) ** 2
|
||||
+ (last["centrist_mean_axis2"] - last["right_mean_axis2"]) ** 2
|
||||
)
|
||||
)
|
||||
delta = last_dist - first_dist
|
||||
if abs(delta) < 1e-9:
|
||||
direction = "unchanged"
|
||||
elif delta < 0:
|
||||
direction = "toward right"
|
||||
else:
|
||||
direction = "away from right"
|
||||
approach_to_right = {
|
||||
"first_distance": round(first_dist, 6),
|
||||
"last_distance": round(last_dist, 6),
|
||||
"delta_distance": round(delta, 6),
|
||||
"direction": direction,
|
||||
}
|
||||
|
||||
return {
|
||||
"euclidean_steps": euclidean_steps,
|
||||
"net_displacement": round(net_disp, 6),
|
||||
"net_dx": round(dx_net, 6),
|
||||
"net_dy": round(dy_net, 6),
|
||||
"angular_direction_deg": round(angle_deg, 2),
|
||||
"approach_to_right": approach_to_right,
|
||||
"right_net": right_net,
|
||||
}
|
||||
|
||||
|
||||
def plot_trajectory(
|
||||
centers: List[Dict[str, Any]],
|
||||
stability_details: List[Dict[str, Any]],
|
||||
avg_corrs: Dict[str, float],
|
||||
annual_centers: List[Dict[str, Any]],
|
||||
output_path: str,
|
||||
) -> None:
|
||||
"""Plot centrist center trajectory with right-wing reference on 2D compass."""
|
||||
"""Plot centrist center trajectory with right-wing reference on 2D compass.
|
||||
|
||||
Uses arrows between consecutive annual windows and year labels.
|
||||
"""
|
||||
fig, ax = plt.subplots(figsize=(10, 8))
|
||||
|
||||
windows = [c["window_id"] for c in centers]
|
||||
cent_a1 = [c["centrist_mean_axis1"] for c in centers]
|
||||
cent_a2 = [c["centrist_mean_axis2"] for c in centers]
|
||||
right_a1 = [c["right_mean_axis1"] for c in centers]
|
||||
right_a2 = [c["right_mean_axis2"] for c in centers]
|
||||
|
||||
valid_windows = [
|
||||
windows[i]
|
||||
for i in range(len(windows))
|
||||
if cent_a1[i] is not None and cent_a2[i] is not None
|
||||
cent_a1 = [c["centrist_mean_axis1"] for c in annual_centers]
|
||||
cent_a2 = [c["centrist_mean_axis2"] for c in annual_centers]
|
||||
windows_labels = [
|
||||
c["window_id"]
|
||||
for c in annual_centers
|
||||
if c["centrist_mean_axis1"] is not None
|
||||
]
|
||||
cent_a1_valid = [v for v in cent_a1 if v is not None]
|
||||
cent_a2_valid = [v for v in cent_a2 if v is not None]
|
||||
|
||||
if len(valid_windows) < 2:
|
||||
if len(cent_a1_valid) < 2:
|
||||
ax.text(
|
||||
0.5,
|
||||
0.5,
|
||||
@@ -299,27 +262,50 @@ def plot_trajectory(
|
||||
plt.close(fig)
|
||||
return
|
||||
|
||||
cent_a1_valid = [c for c in cent_a1 if c is not None]
|
||||
cent_a2_valid = [c for c in cent_a2 if c is not None]
|
||||
right_a1_valid = [c for c in right_a1 if c is not None]
|
||||
right_a2_valid = [c for c in right_a2 if c is not None]
|
||||
windows_valid = [w for w, a1 in zip(windows, cent_a1) if a1 is not None]
|
||||
# Arrows between consecutive years
|
||||
for i in range(len(cent_a1_valid) - 1):
|
||||
ax.annotate(
|
||||
"",
|
||||
xy=(cent_a1_valid[i + 1], cent_a2_valid[i + 1]),
|
||||
xytext=(cent_a1_valid[i], cent_a2_valid[i]),
|
||||
arrowprops=dict(arrowstyle="->", color="#1E73BE", lw=1.5, alpha=0.6),
|
||||
)
|
||||
|
||||
years = [int(w) for w in windows_valid]
|
||||
ax.plot(
|
||||
cent_a1_valid,
|
||||
cent_a2_valid,
|
||||
"o-",
|
||||
color="#1E73BE",
|
||||
linewidth=2,
|
||||
markersize=8,
|
||||
label="Centrist center (VVD, D66, CDA, NSC, BBB, CU)",
|
||||
zorder=3,
|
||||
)
|
||||
|
||||
ax.plot(cent_a1_valid, cent_a2_valid, "o-", color="#1E73BE", linewidth=2,
|
||||
markersize=8, label="Centrist center (VVD, D66, CDA, NSC, BBB, CU)",
|
||||
zorder=3)
|
||||
# Right-wing trajectory (dashed reference)
|
||||
right_a1 = [c["right_mean_axis1"] for c in annual_centers]
|
||||
right_a2 = [c["right_mean_axis2"] for c in annual_centers]
|
||||
right_a1_valid = [v for v in right_a1 if v is not None]
|
||||
right_a2_valid = [v for v in right_a2 if v is not None]
|
||||
|
||||
if right_a1_valid and right_a2_valid:
|
||||
ax.plot(right_a1_valid, right_a2_valid, "s--", color="#6A1B9A", linewidth=1.5,
|
||||
markersize=6, label="Right-wing center (PVV, FVD, JA21, SGP)",
|
||||
alpha=0.7, zorder=2)
|
||||
ax.plot(
|
||||
right_a1_valid,
|
||||
right_a2_valid,
|
||||
"s--",
|
||||
color="#6A1B9A",
|
||||
linewidth=1.5,
|
||||
markersize=6,
|
||||
label="Right-wing center (PVV, FVD, JA21, SGP)",
|
||||
alpha=0.7,
|
||||
zorder=2,
|
||||
)
|
||||
|
||||
for i, year in enumerate(years):
|
||||
if i < len(cent_a1_valid) and cent_a1_valid[i] is not None:
|
||||
# Year labels
|
||||
for i, label in enumerate(windows_labels):
|
||||
if i < len(cent_a1_valid):
|
||||
ax.annotate(
|
||||
str(year),
|
||||
str(label),
|
||||
(cent_a1_valid[i], cent_a2_valid[i]),
|
||||
textcoords="offset points",
|
||||
xytext=(7, 7),
|
||||
@@ -330,12 +316,10 @@ def plot_trajectory(
|
||||
ax.axhline(0, color="#CCCCCC", linewidth=0.5, linestyle="-")
|
||||
ax.axvline(0, color="#CCCCCC", linewidth=0.5, linestyle="-")
|
||||
|
||||
ax.set_xlabel("SVD Axis 1")
|
||||
ax.set_ylabel("SVD Axis 2")
|
||||
ax.set_xlabel("PCA Axis 1 (Procrustes-aligned)")
|
||||
ax.set_ylabel("PCA Axis 2 (Procrustes-aligned)")
|
||||
ax.set_title(
|
||||
f"Parliamentary Center Trajectory (2016–2026)\n"
|
||||
f"Stability: axis1 ρ={avg_corrs.get('mean_axis1_corr', 0):.3f}, "
|
||||
f"axis2 ρ={avg_corrs.get('mean_axis2_corr', 0):.3f}",
|
||||
"Parliamentary Center Trajectory (Procrustes-Aligned PCA)",
|
||||
fontsize=11,
|
||||
)
|
||||
ax.legend(loc="upper left", fontsize=8, framealpha=0.9)
|
||||
@@ -348,129 +332,113 @@ def plot_trajectory(
|
||||
logger.info("Chart saved to %s", output_path)
|
||||
|
||||
|
||||
def compute_drift_metrics(centers: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Compute drift metrics: Euclidean distance per step, net displacement, direction."""
|
||||
valid = [c for c in centers if c["centrist_mean_axis1"] is not None]
|
||||
|
||||
if len(valid) < 2:
|
||||
return {
|
||||
"euclidean_steps": [],
|
||||
"net_displacement": None,
|
||||
"angular_direction_deg": None,
|
||||
"rightward_distance_traveled": None,
|
||||
}
|
||||
|
||||
euclidean_steps = []
|
||||
for i in range(len(valid) - 1):
|
||||
dx = valid[i + 1]["centrist_mean_axis1"] - valid[i]["centrist_mean_axis1"]
|
||||
dy = valid[i + 1]["centrist_mean_axis2"] - valid[i]["centrist_mean_axis2"]
|
||||
dist = float(np.sqrt(dx**2 + dy**2))
|
||||
euclidean_steps.append({
|
||||
"window_pair": f"{valid[i]['window_id']}-{valid[i+1]['window_id']}",
|
||||
"distance": round(dist, 6),
|
||||
"dx": round(dx, 6),
|
||||
"dy": round(dy, 6),
|
||||
})
|
||||
|
||||
first = valid[0]
|
||||
last = valid[-1]
|
||||
dx_net = last["centrist_mean_axis1"] - first["centrist_mean_axis1"]
|
||||
dy_net = last["centrist_mean_axis2"] - first["centrist_mean_axis2"]
|
||||
net_disp = float(np.sqrt(dx_net**2 + dy_net**2))
|
||||
|
||||
angle_rad = np.arctan2(dy_net, dx_net)
|
||||
angle_deg = float(np.degrees(angle_rad))
|
||||
|
||||
return {
|
||||
"euclidean_steps": euclidean_steps,
|
||||
"net_displacement": round(net_disp, 6),
|
||||
"net_dx": round(dx_net, 6),
|
||||
"net_dy": round(dy_net, 6),
|
||||
"angular_direction_deg": round(angle_deg, 2),
|
||||
}
|
||||
|
||||
|
||||
def write_report(
|
||||
is_stable: bool,
|
||||
stability_details: List[Dict[str, Any]],
|
||||
avg_corrs: Dict[str, float],
|
||||
centers: List[Dict[str, Any]],
|
||||
annual_centers: List[Dict[str, Any]],
|
||||
drift: Dict[str, Any],
|
||||
output_path: str,
|
||||
chart_path: str,
|
||||
non_annual: List[str],
|
||||
) -> None:
|
||||
"""Write the SVD stability and drift report as Markdown."""
|
||||
"""Write the center drift report as Markdown."""
|
||||
lines: List[str] = []
|
||||
|
||||
lines.append("# SVD Center Drift & Axis Stability Report\n")
|
||||
|
||||
lines.append("## Axis Stability Validation\n")
|
||||
lines.append("# Center Drift Report (Procrustes-Aligned)\n")
|
||||
|
||||
lines.append("## Alignment Method\n")
|
||||
lines.append(
|
||||
f"**Stability threshold:** Spearman ρ ≥ {STABILITY_THRESHOLD} for both axes. "
|
||||
f"Maximum unstable pairs allowed: {MAX_UNSTABLE_PAIRS}.\n"
|
||||
"Party positions are Procrustes-aligned across all windows, then "
|
||||
"PCA-rotated to a common 2D reference frame. This ensures that axis "
|
||||
"orientation is consistent across time — no stability validation is "
|
||||
"needed because all positions live in the same coordinate system.\n"
|
||||
)
|
||||
lines.append(
|
||||
"This is the same alignment used by the Explorer UI compass and "
|
||||
"trajectories: 1) zero-padding vectors to max dimension across all "
|
||||
"windows, 2) chained Procrustes orthogonal rotation (each window to "
|
||||
"the previous aligned one), 3) global PCA on the stacked aligned "
|
||||
"matrix, 4) flip-correction per component using canonical left/right "
|
||||
"parties.\n"
|
||||
)
|
||||
|
||||
unstable_count = sum(1 for d in stability_details if d.get("unstable"))
|
||||
lines.append(
|
||||
f"**Result:** {unstable_count} unstable pair(s) out of "
|
||||
f"{len(stability_details)} consecutive window pairs.\n"
|
||||
)
|
||||
|
||||
if not is_stable:
|
||||
if non_annual:
|
||||
lines.append(
|
||||
"**CONCLUSION: SVD axes are too unstable for longitudinal comparison. "
|
||||
"Positions may reflect re-orientation rather than genuine drift. "
|
||||
"The following drift metrics and chart should be interpreted with extreme caution.**\n"
|
||||
f"**Note:** Non-annual windows excluded from drift analysis: "
|
||||
f"{', '.join(sorted(non_annual))}\n"
|
||||
)
|
||||
|
||||
lines.append(f"- Mean axis-1 correlation: {avg_corrs['mean_axis1_corr']:.4f}")
|
||||
lines.append(f"- Mean axis-2 correlation: {avg_corrs['mean_axis2_corr']:.4f}\n")
|
||||
|
||||
lines.append("### Per-Pair Stability Details\n")
|
||||
lines.append("| Window Pair | Axis 1 ρ | Axis 2 ρ | Unstable | Shared Parties |")
|
||||
lines.append("|---|---|---|---|---|")
|
||||
for d in stability_details:
|
||||
r1 = f"{d['axis1_corr']:.3f}" if d["axis1_corr"] is not None else "N/A"
|
||||
r2 = f"{d['axis2_corr']:.3f}" if d["axis2_corr"] is not None else "N/A"
|
||||
flag = "**YES**" if d.get("unstable") else "no"
|
||||
parties = ", ".join(d.get("shared_parties", []))
|
||||
lines.append(f"| {d['window_pair']} | {r1} | {r2} | {flag} | {parties} |")
|
||||
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Centrist Center of Gravity\n")
|
||||
lines.append(
|
||||
"| Window | Centrist Ax1 | Centrist Ax2 | Right Ax1 | Right Ax2 | "
|
||||
"Centrist Parties Present | Right Parties Present |"
|
||||
"Centrist Parties | Right Parties |"
|
||||
)
|
||||
lines.append("|---|---|---|---|---|---|---|")
|
||||
for c in centers:
|
||||
cent_a1 = f"{c['centrist_mean_axis1']:.4f}" if c["centrist_mean_axis1"] is not None else "N/A"
|
||||
cent_a2 = f"{c['centrist_mean_axis2']:.4f}" if c["centrist_mean_axis2"] is not None else "N/A"
|
||||
right_a1 = f"{c['right_mean_axis1']:.4f}" if c["right_mean_axis1"] is not None else "N/A"
|
||||
right_a2 = f"{c['right_mean_axis2']:.4f}" if c["right_mean_axis2"] is not None else "N/A"
|
||||
cent_a1 = (
|
||||
f"{c['centrist_mean_axis1']:.4f}"
|
||||
if c["centrist_mean_axis1"] is not None
|
||||
else "N/A"
|
||||
)
|
||||
cent_a2 = (
|
||||
f"{c['centrist_mean_axis2']:.4f}"
|
||||
if c["centrist_mean_axis2"] is not None
|
||||
else "N/A"
|
||||
)
|
||||
right_a1 = (
|
||||
f"{c['right_mean_axis1']:.4f}"
|
||||
if c["right_mean_axis1"] is not None
|
||||
else "N/A"
|
||||
)
|
||||
right_a2 = (
|
||||
f"{c['right_mean_axis2']:.4f}"
|
||||
if c["right_mean_axis2"] is not None
|
||||
else "N/A"
|
||||
)
|
||||
cent_parties = ", ".join(c["centrist_parties_present"])
|
||||
right_parties = ", ".join(c["right_parties_present"])
|
||||
lines.append(
|
||||
f"| {c['window_id']} | {cent_a1} | {cent_a2} | {right_a1} | {right_a2} "
|
||||
f"| {cent_parties} | {right_parties} |"
|
||||
f"| {c['window_id']} | {cent_a1} | {cent_a2} | "
|
||||
f"{right_a1} | {right_a2} | {cent_parties} | {right_parties} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
|
||||
if is_stable:
|
||||
lines.append("## Drift Metrics\n")
|
||||
lines.append(f"- **Net displacement (first → last):** {drift['net_displacement']}")
|
||||
# Drift metrics
|
||||
lines.append("## Drift Metrics (Annual Windows Only)\n")
|
||||
|
||||
if drift.get("net_displacement") is not None:
|
||||
lines.append(
|
||||
f"- **Net centrist displacement (first → last):** "
|
||||
f"{drift['net_displacement']}"
|
||||
)
|
||||
lines.append(f" - Δ axis-1: {drift['net_dx']}")
|
||||
lines.append(f" - Δ axis-2: {drift['net_dy']}")
|
||||
lines.append(f"- **Net direction:** {drift['angular_direction_deg']}° "
|
||||
f"(arctan2(Δy, Δx))")
|
||||
lines.append(
|
||||
f"- **Net direction:** {drift['angular_direction_deg']}° "
|
||||
f"(arctan2(Δy, Δx))"
|
||||
)
|
||||
lines.append(f" - Positive Δx = rightward on axis 1")
|
||||
lines.append(f" - Positive Δy = upward on axis 2\n")
|
||||
|
||||
if drift.get("right_net"):
|
||||
rn = drift["right_net"]
|
||||
lines.append("- **Right-wing net displacement (reference):**")
|
||||
lines.append(f" - Net displacement: {rn['net_displacement']}")
|
||||
lines.append(f" - Δ axis-1: {rn['net_dx']}")
|
||||
lines.append(f" - Δ axis-2: {rn['net_dy']}\n")
|
||||
|
||||
if drift.get("approach_to_right"):
|
||||
ar = drift["approach_to_right"]
|
||||
lines.append("- **Centrist–right distance:**")
|
||||
lines.append(f" - First window: {ar['first_distance']}")
|
||||
lines.append(f" - Last window: {ar['last_distance']}")
|
||||
lines.append(
|
||||
f" - Δ distance: {ar['delta_distance']} "
|
||||
f"(centrist center moving **{ar['direction']}**)\n"
|
||||
)
|
||||
|
||||
lines.append("### Year-over-Year Drift\n")
|
||||
lines.append("| Window Pair | Euclidean Distance | Δ Axis-1 | Δ Axis-2 |")
|
||||
lines.append("| Window Pair | Distance | Δ Axis-1 | Δ Axis-2 |")
|
||||
lines.append("|---|---|---|---|")
|
||||
total_dist = 0.0
|
||||
for step in drift["euclidean_steps"]:
|
||||
@@ -480,39 +448,30 @@ def write_report(
|
||||
)
|
||||
total_dist += step["distance"]
|
||||
lines.append(f"\n**Total path length:** {total_dist:.6f}\n")
|
||||
|
||||
else:
|
||||
lines.append("## Drift Metrics (UNRELIABLE — Axes Unstable)\n")
|
||||
lines.append(
|
||||
"Drift metrics were computed but are unreliable due to axis instability. "
|
||||
"Cross-window comparisons on unstable axes conflate positional change "
|
||||
"with axis re-orientation.\n"
|
||||
)
|
||||
lines.append("Insufficient annual windows for drift computation.\n")
|
||||
|
||||
lines.append(f"## Chart\n")
|
||||
lines.append(f"})\n")
|
||||
lines.append("## Chart\n")
|
||||
lines.append(f"})\n")
|
||||
|
||||
lines.append("## Interpretability Statement\n")
|
||||
if is_stable:
|
||||
lines.append(
|
||||
"The SVD axes show sufficient stability for cross-window comparison. "
|
||||
"The parliamentary center trajectory reflects genuine shifts in voting "
|
||||
"behavior rather than axis re-orientation artifact. The centrist center-of-gravity "
|
||||
"movement on the 2D compass can be interpreted as a measure of ideological drift.\n"
|
||||
)
|
||||
else:
|
||||
lines.append(
|
||||
"SVD axes are too unstable for longitudinal comparison. The trajectory "
|
||||
"plotted above may reflect axis re-orientation (each SVD window independently "
|
||||
"determines its principal axes) rather than genuine ideological drift. "
|
||||
"We recommend against drawing conclusions from this analysis.\n"
|
||||
)
|
||||
lines.append(
|
||||
"Party positions use Procrustes-aligned PCA axes that provide a "
|
||||
"common reference frame across all windows. Unlike raw per-window "
|
||||
"SVD axes — which may re-orient between windows and cause 9/10 "
|
||||
"consecutive window pairs to fail axis stability (Spearman ρ < 0.7) "
|
||||
"— this alignment ensures that positional changes reflect genuine "
|
||||
"shifts in voting behavior rather than axis re-orientation artifacts. "
|
||||
"The centrist center-of-gravity movement on the 2D compass can be "
|
||||
"interpreted as a measure of ideological drift.\n"
|
||||
)
|
||||
|
||||
lines.append("---\n")
|
||||
lines.append(
|
||||
"*Note: SVD axes reflect voting patterns, not semantic content. "
|
||||
"A shift means voting behavior changed, not that parties changed their rhetoric. "
|
||||
"See: docs/solutions/best-practices/svd-labels-voting-patterns-not-semantics.md*\n"
|
||||
"*Note: PCA axes reflect voting patterns, not semantic content. "
|
||||
"A shift means voting behavior changed, not that parties changed "
|
||||
"their rhetoric. See: docs/solutions/best-practices/"
|
||||
"svd-labels-voting-patterns-not-semantics.md*\n"
|
||||
)
|
||||
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
@@ -521,95 +480,84 @@ def write_report(
|
||||
logger.info("Report saved to %s", output_path)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
def main() -> Dict[str, Any]:
|
||||
os.makedirs(str(REPORTS_DIR), exist_ok=True)
|
||||
|
||||
con = duckdb.connect(database=DB_PATH, read_only=False)
|
||||
logger.info("Loading aligned party positions...")
|
||||
windows = get_uniform_dim_windows(DB_PATH)
|
||||
if not windows:
|
||||
logger.error("No uniform-dim windows found in database")
|
||||
return {"error": "No windows found", "windows_analyzed": 0}
|
||||
|
||||
try:
|
||||
windows = get_annual_windows(con)
|
||||
logger.info("Found %d annual windows: %s", len(windows), windows)
|
||||
scores = load_party_scores_all_windows_aligned(DB_PATH)
|
||||
if not scores:
|
||||
logger.error("No aligned party scores loaded")
|
||||
return {"error": "No scores loaded", "windows_analyzed": 0}
|
||||
|
||||
all_positions: Dict[str, Dict[str, Tuple[float, float]]] = {}
|
||||
for w in windows:
|
||||
pos = compute_party_positions(con, w)
|
||||
all_positions[w] = pos
|
||||
n_parties = len(pos)
|
||||
centrist_present = sum(1 for p in pos if p in CANONICAL_CENTRIST)
|
||||
right_present = sum(1 for p in pos if p in CANONICAL_RIGHT)
|
||||
logger.info(
|
||||
"Window %s: %d parties, %d centrist, %d right",
|
||||
w, n_parties, centrist_present, right_present,
|
||||
)
|
||||
logger.info("Found %d total windows: %s", len(windows), windows)
|
||||
logger.info(
|
||||
"Loaded scores for %d parties: %s",
|
||||
len(scores),
|
||||
sorted(scores.keys()),
|
||||
)
|
||||
|
||||
is_stable, stability_details, avg_corrs = validate_axis_stability(
|
||||
all_positions, windows
|
||||
)
|
||||
# Classify windows: annual (pure digit years) vs non-annual
|
||||
annual_indices: List[int] = []
|
||||
non_annual: List[str] = []
|
||||
for idx, w in enumerate(windows):
|
||||
if w.strip().isdigit():
|
||||
annual_indices.append(idx)
|
||||
else:
|
||||
non_annual.append(w)
|
||||
|
||||
unstable_count = sum(1 for d in stability_details if d.get("unstable"))
|
||||
annual_window_ids = [windows[i] for i in annual_indices]
|
||||
logger.info("Annual windows (%d): %s", len(annual_window_ids), annual_window_ids)
|
||||
if non_annual:
|
||||
logger.info(
|
||||
"Stability: %s (%d/%d unstable pairs), mean axis1 ρ=%.3f, mean axis2 ρ=%.3f",
|
||||
"STABLE" if is_stable else "UNSTABLE",
|
||||
unstable_count,
|
||||
len(stability_details),
|
||||
avg_corrs["mean_axis1_corr"],
|
||||
avg_corrs["mean_axis2_corr"],
|
||||
"Non-annual windows (excluded from drift): %s", sorted(non_annual)
|
||||
)
|
||||
|
||||
for d in stability_details:
|
||||
if d.get("unstable"):
|
||||
logger.warning(
|
||||
"Unstable pair %s: axis1=%.3f, axis2=%.3f, reason=%s",
|
||||
d["window_pair"],
|
||||
d["axis1_corr"] or 0,
|
||||
d["axis2_corr"] or 0,
|
||||
d.get("reason", ""),
|
||||
)
|
||||
# Compute centers for all windows
|
||||
centers = compute_aligned_centers(scores, windows, annual_indices)
|
||||
|
||||
centers = compute_centers(all_positions, windows)
|
||||
|
||||
stability_score = (
|
||||
avg_corrs["mean_axis1_corr"] + avg_corrs["mean_axis2_corr"]
|
||||
) / 2.0
|
||||
|
||||
for c_row in centers:
|
||||
c_row["stability_score"] = stability_score
|
||||
|
||||
create_table(con, centers, stability_score)
|
||||
|
||||
n_rows = con.execute("SELECT COUNT(*) FROM overton_svd_center").fetchone()[0]
|
||||
logger.info("Created overton_svd_center table with %d rows", n_rows)
|
||||
|
||||
chart_path = str(REPORTS_DIR / "svd_drift_chart.png")
|
||||
plot_trajectory(centers, stability_details, avg_corrs, chart_path)
|
||||
|
||||
drift = compute_drift_metrics(centers)
|
||||
|
||||
report_path = str(REPORTS_DIR / "svd_stability_report.md")
|
||||
write_report(
|
||||
is_stable, stability_details, avg_corrs, centers,
|
||||
drift, report_path, chart_path,
|
||||
for c in centers:
|
||||
logger.info(
|
||||
"Window %s: %d centrist, %d right (annual=%s)",
|
||||
c["window_id"],
|
||||
c["centrist_count"],
|
||||
c["right_count"],
|
||||
c["is_annual"],
|
||||
)
|
||||
|
||||
summary = {
|
||||
"stability_status": "STABLE" if is_stable else "UNSTABLE",
|
||||
"unstable_pairs": unstable_count,
|
||||
"total_pairs": len(stability_details),
|
||||
"mean_axis1_corr": round(avg_corrs["mean_axis1_corr"], 4),
|
||||
"mean_axis2_corr": round(avg_corrs["mean_axis2_corr"], 4),
|
||||
"windows": len(windows),
|
||||
"table_rows": n_rows,
|
||||
"net_displacement": drift.get("net_displacement"),
|
||||
"net_dx": drift.get("net_dx"),
|
||||
"net_dy": drift.get("net_dy"),
|
||||
"angular_direction_deg": drift.get("angular_direction_deg"),
|
||||
}
|
||||
# Filter to annual-only for drift and chart
|
||||
annual_centers = [c for c in centers if c["is_annual"]]
|
||||
|
||||
logger.info("Summary: %s", json.dumps(summary, indent=2))
|
||||
return summary
|
||||
drift = compute_drift_metrics(annual_centers)
|
||||
|
||||
finally:
|
||||
con.close()
|
||||
# Chart
|
||||
chart_path = str(REPORTS_DIR / "svd_drift_chart.png")
|
||||
plot_trajectory(annual_centers, chart_path)
|
||||
|
||||
# Report
|
||||
report_path = str(REPORTS_DIR / "svd_stability_report.md")
|
||||
write_report(centers, annual_centers, drift, report_path, chart_path, non_annual)
|
||||
|
||||
summary = {
|
||||
"method": "Procrustes-aligned PCA",
|
||||
"total_windows": len(windows),
|
||||
"annual_windows_analyzed": len(annual_centers),
|
||||
"non_annual_skipped": sorted(non_annual),
|
||||
"parties_loaded": len(scores),
|
||||
"windows": windows,
|
||||
"net_displacement": drift.get("net_displacement"),
|
||||
"net_dx": drift.get("net_dx"),
|
||||
"net_dy": drift.get("net_dy"),
|
||||
"angular_direction_deg": drift.get("angular_direction_deg"),
|
||||
"approach_to_right": drift.get("approach_to_right"),
|
||||
}
|
||||
|
||||
logger.info("Summary: %s", json.dumps(summary, indent=2))
|
||||
return summary
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user