motief/scripts/generate_extra_charts.py

"""Generate additional blog charts: controversy trend + party alignment heatmap."""

from __future__ import annotations
import os, sys

ROOT = os.path.dirname(os.path.abspath(__file__))
if ROOT not in sys.path:
    sys.path.insert(0, ROOT)

import duckdb
import plotly.graph_objects as go
import plotly.express as px
import numpy as np

DB = "data/motions.db"
OUT = "outputs/blog-charts"
os.makedirs(OUT, exist_ok=True)

con = duckdb.connect(DB, read_only=True)

# ─── 1. Controversy trend (bar chart, 2019-2026, quarterly) ──────────────────
rows = con.execute("""
    SELECT
        YEAR(date) || '-Q' || QUARTER(date) as wid,
        YEAR(date) as yr,
        QUARTER(date) as q,
        COUNT(*) as n,
        ROUND(AVG(controversy_score), 3) as avg_c,
        COUNT(*) FILTER (WHERE controversy_score >= 0.7) as high_c
    FROM motions
    WHERE controversy_score IS NOT NULL
      AND date >= '2019-01-01' AND date < '2026-04-01'
    GROUP BY wid, yr, q
    ORDER BY yr, q
""").fetchall()

windows = [r[0] for r in rows]
avg_c = [r[4] for r in rows]
high_pct = [round(100.0 * r[5] / r[3], 1) if r[3] else 0 for r in rows]

fig = go.Figure()
fig.add_trace(
    go.Bar(
        x=windows,
        y=high_pct,
        name="% highly contested (score ≥ 0.7)",
        marker_color="#00d9a3",
        opacity=0.85,
    )
)
fig.add_trace(
    go.Scatter(
        x=windows,
        y=[v * 100 for v in avg_c],
        name="avg controversy × 100",
        mode="lines+markers",
        line=dict(color="#e6edf3", width=2),
        marker=dict(size=4),
    )
)
fig.update_layout(
    title="Political controversy per quarter (Tweede Kamer, 2019–2026)",
    xaxis_title="Quarter",
    yaxis_title="% of motions",
    plot_bgcolor="#161b22",
    paper_bgcolor="#0d1117",
    font=dict(color="#e6edf3", family="Inter, system-ui"),
    legend=dict(bgcolor="rgba(0,0,0,0)", bordercolor="#30363d", borderwidth=1),
    xaxis=dict(tickangle=-45, gridcolor="#30363d"),
    yaxis=dict(gridcolor="#30363d", range=[0, 55]),
    bargap=0.15,
)
out1 = os.path.join(OUT, "controversy_trend.html")
fig.write_html(out1, include_plotlyjs="cdn", full_html=True)
print(f"Wrote {out1}")

# ─── 2. Party alignment heatmap ──────────────────────────────────────────────
# Only include major parties with sufficient data
MAJOR = [
    "VVD",
    "PVV",
    "D66",
    "CDA",
    "PvdA",
    "GroenLinks",
    "SP",
    "ChristenUnie",
    "SGP",
    "FVD",
    "BBB",
    "PvdD",
    "Volt",
    "GroenLinks-PvdA",
    "Nieuw Sociaal Contract",
    "DENK",
    "JA21",
]

rows = con.execute("""
    WITH pv AS (
        SELECT motion_id, party,
               CASE
                 WHEN SUM(CASE WHEN vote='voor' THEN 1 ELSE 0 END) > SUM(CASE WHEN vote='tegen' THEN 1 ELSE 0 END) THEN 'voor'
                 WHEN SUM(CASE WHEN vote='tegen' THEN 1 ELSE 0 END) > SUM(CASE WHEN vote='voor' THEN 1 ELSE 0 END) THEN 'tegen'
                 ELSE 'split'
               END as pv
        FROM mp_votes WHERE party IS NOT NULL AND vote IN ('voor','tegen')
        GROUP BY motion_id, party
    ),
    d AS (SELECT * FROM pv WHERE pv != 'split')
    SELECT a.party, b.party,
           COUNT(*) as shared,
           ROUND(100.0 * SUM(CASE WHEN a.pv = b.pv THEN 1 ELSE 0 END) / COUNT(*), 1) as pct
    FROM d a JOIN d b ON a.motion_id = b.motion_id AND a.party != b.party
    GROUP BY a.party, b.party
    HAVING COUNT(*) >= 100
""").fetchall()

# Build matrix
agree = {}
for a, b, _, pct in rows:
    agree[(a, b)] = pct

# Filter to parties that have data
present = set()
for a, b in agree:
    if a in MAJOR:
        present.add(a)
    if b in MAJOR:
        present.add(b)
parties = [p for p in MAJOR if p in present]

n = len(parties)
matrix = np.full((n, n), np.nan)
for i, a in enumerate(parties):
    matrix[i, i] = 100.0
    for j, b in enumerate(parties):
        if i != j and (a, b) in agree:
            matrix[i, j] = agree[(a, b)]

fig2 = go.Figure(
    data=go.Heatmap(
        z=matrix,
        x=parties,
        y=parties,
        colorscale=[[0, "#6e40c9"], [0.5, "#30363d"], [1, "#00d9a3"]],
        zmid=70,
        zmin=35,
        zmax=100,
        text=[[f"{v:.0f}%" if not np.isnan(v) else "" for v in row] for row in matrix],
        texttemplate="%{text}",
        textfont=dict(size=9),
        hoverongaps=False,
        showscale=True,
        colorbar=dict(title="Agreement %", tickfont=dict(color="#e6edf3")),
    )
)
fig2.update_layout(
    title="Cross-party vote alignment (all years combined)",
    plot_bgcolor="#161b22",
    paper_bgcolor="#0d1117",
    font=dict(color="#e6edf3", family="Inter, system-ui", size=11),
    xaxis=dict(tickangle=-45, side="bottom", gridcolor="#30363d"),
    yaxis=dict(autorange="reversed", gridcolor="#30363d"),
    height=600,
)
out2 = os.path.join(OUT, "party_alignment.html")
fig2.write_html(out2, include_plotlyjs="cdn", full_html=True)
print(f"Wrote {out2}")

con.close()
print("Done.")