feat: add StemAtlas Streamlit app, explorer, Docker deployment, blog charts

2026-03-22 22:38:17 +01:00
parent daa22c5e2b
commit 2891e9ee70
20 changed files with 2576 additions and 84 deletions
@@ -1,7 +1,7 @@
 """download_past_year.py — One-shot data download: parliamentary motions for a date range.

 Fetches Stemming records from the OData API in chunks (default 90-day windows),
-stores motions into data/motions.db using MotionDatabase.insert_motion().
+stores motions into data/motions.db using MotionDatabase.batch_insert_motions().

 Skips AI summarisation — this is a raw data fetch for the embedding pipeline.

@@ -105,11 +105,7 @@ def main():

            inserted = 0
            duplicates = 0
-            for m in motions:
-                if db.insert_motion(m):
-                    inserted += 1
-                else:
-                    duplicates += 1
+            inserted, duplicates = db.batch_insert_motions(motions)

            total_inserted += inserted
            total_duplicates += duplicates
@@ -0,0 +1,172 @@
+"""Generate additional blog charts: controversy trend + party alignment heatmap."""
+
+from __future__ import annotations
+import os, sys
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+if ROOT not in sys.path:
+    sys.path.insert(0, ROOT)
+
+import duckdb
+import plotly.graph_objects as go
+import plotly.express as px
+import numpy as np
+
+DB = "data/motions.db"
+OUT = "outputs/blog-charts"
+os.makedirs(OUT, exist_ok=True)
+
+con = duckdb.connect(DB, read_only=True)
+
+# ─── 1. Controversy trend (bar chart, 2019-2026, quarterly) ──────────────────
+rows = con.execute("""
+    SELECT
+        YEAR(date) || '-Q' || QUARTER(date) as wid,
+        YEAR(date) as yr,
+        QUARTER(date) as q,
+        COUNT(*) as n,
+        ROUND(AVG(controversy_score), 3) as avg_c,
+        COUNT(*) FILTER (WHERE controversy_score >= 0.7) as high_c
+    FROM motions
+    WHERE controversy_score IS NOT NULL
+      AND date >= '2019-01-01' AND date < '2026-04-01'
+    GROUP BY wid, yr, q
+    ORDER BY yr, q
+""").fetchall()
+
+windows = [r[0] for r in rows]
+avg_c = [r[4] for r in rows]
+high_pct = [round(100.0 * r[5] / r[3], 1) if r[3] else 0 for r in rows]
+
+fig = go.Figure()
+fig.add_trace(
+    go.Bar(
+        x=windows,
+        y=high_pct,
+        name="% highly contested (score ≥ 0.7)",
+        marker_color="#00d9a3",
+        opacity=0.85,
+    )
+)
+fig.add_trace(
+    go.Scatter(
+        x=windows,
+        y=[v * 100 for v in avg_c],
+        name="avg controversy × 100",
+        mode="lines+markers",
+        line=dict(color="#e6edf3", width=2),
+        marker=dict(size=4),
+    )
+)
+fig.update_layout(
+    title="Political controversy per quarter (Tweede Kamer, 2019–2026)",
+    xaxis_title="Quarter",
+    yaxis_title="% of motions",
+    plot_bgcolor="#161b22",
+    paper_bgcolor="#0d1117",
+    font=dict(color="#e6edf3", family="Inter, system-ui"),
+    legend=dict(bgcolor="rgba(0,0,0,0)", bordercolor="#30363d", borderwidth=1),
+    xaxis=dict(tickangle=-45, gridcolor="#30363d"),
+    yaxis=dict(gridcolor="#30363d", range=[0, 55]),
+    bargap=0.15,
+)
+out1 = os.path.join(OUT, "controversy_trend.html")
+fig.write_html(out1, include_plotlyjs="cdn", full_html=True)
+print(f"Wrote {out1}")
+
+# ─── 2. Party alignment heatmap ──────────────────────────────────────────────
+# Only include major parties with sufficient data
+MAJOR = [
+    "VVD",
+    "PVV",
+    "D66",
+    "CDA",
+    "PvdA",
+    "GroenLinks",
+    "SP",
+    "ChristenUnie",
+    "SGP",
+    "FVD",
+    "BBB",
+    "PvdD",
+    "Volt",
+    "GroenLinks-PvdA",
+    "Nieuw Sociaal Contract",
+    "DENK",
+    "JA21",
+]
+
+rows = con.execute("""
+    WITH pv AS (
+        SELECT motion_id, party,
+               CASE
+                 WHEN SUM(CASE WHEN vote='voor' THEN 1 ELSE 0 END) > SUM(CASE WHEN vote='tegen' THEN 1 ELSE 0 END) THEN 'voor'
+                 WHEN SUM(CASE WHEN vote='tegen' THEN 1 ELSE 0 END) > SUM(CASE WHEN vote='voor' THEN 1 ELSE 0 END) THEN 'tegen'
+                 ELSE 'split'
+               END as pv
+        FROM mp_votes WHERE party IS NOT NULL AND vote IN ('voor','tegen')
+        GROUP BY motion_id, party
+    ),
+    d AS (SELECT * FROM pv WHERE pv != 'split')
+    SELECT a.party, b.party,
+           COUNT(*) as shared,
+           ROUND(100.0 * SUM(CASE WHEN a.pv = b.pv THEN 1 ELSE 0 END) / COUNT(*), 1) as pct
+    FROM d a JOIN d b ON a.motion_id = b.motion_id AND a.party != b.party
+    GROUP BY a.party, b.party
+    HAVING COUNT(*) >= 100
+""").fetchall()
+
+# Build matrix
+agree = {}
+for a, b, _, pct in rows:
+    agree[(a, b)] = pct
+
+# Filter to parties that have data
+present = set()
+for a, b in agree:
+    if a in MAJOR:
+        present.add(a)
+    if b in MAJOR:
+        present.add(b)
+parties = [p for p in MAJOR if p in present]
+
+n = len(parties)
+matrix = np.full((n, n), np.nan)
+for i, a in enumerate(parties):
+    matrix[i, i] = 100.0
+    for j, b in enumerate(parties):
+        if i != j and (a, b) in agree:
+            matrix[i, j] = agree[(a, b)]
+
+fig2 = go.Figure(
+    data=go.Heatmap(
+        z=matrix,
+        x=parties,
+        y=parties,
+        colorscale=[[0, "#6e40c9"], [0.5, "#30363d"], [1, "#00d9a3"]],
+        zmid=70,
+        zmin=35,
+        zmax=100,
+        text=[[f"{v:.0f}%" if not np.isnan(v) else "" for v in row] for row in matrix],
+        texttemplate="%{text}",
+        textfont=dict(size=9),
+        hoverongaps=False,
+        showscale=True,
+        colorbar=dict(title="Agreement %", tickfont=dict(color="#e6edf3")),
+    )
+)
+fig2.update_layout(
+    title="Cross-party vote alignment (all years combined)",
+    plot_bgcolor="#161b22",
+    paper_bgcolor="#0d1117",
+    font=dict(color="#e6edf3", family="Inter, system-ui", size=11),
+    xaxis=dict(tickangle=-45, side="bottom", gridcolor="#30363d"),
+    yaxis=dict(autorange="reversed", gridcolor="#30363d"),
+    height=600,
+)
+out2 = os.path.join(OUT, "party_alignment.html")
+fig2.write_html(out2, include_plotlyjs="cdn", full_html=True)
+print(f"Wrote {out2}")
+
+con.close()
+print("Done.")