feat(overton): add category domain decomposition with interactive charts and TDD tests
Populated the right_wing_motions.category column (previously 100% NULL across 3,030 motions) via parallel subagent classification — 80 agents derived a 10-category taxonomy and classified all motions in minutes. Adds to the Overton QMD report: - Plotly dropdown filter on Chart 1 to toggle between policy categories - Chart 7: category delta bar chart (pre/post centrist support per domain) - Chart 8: quarterly domain trajectories for the 5 largest categories - Domain Decomposition narrative section Also fixes a Streamlit tab crash (m.text -> m.body_text) and adds TDD tests.
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
"""Validate category decomposition data for Overton report."""
|
||||
from __future__ import annotations
|
||||
|
||||
import duckdb
|
||||
import pytest
|
||||
|
||||
DB_PATH = "data/motions.db"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def con():
|
||||
c = duckdb.connect(DB_PATH)
|
||||
yield c
|
||||
c.close()
|
||||
|
||||
|
||||
def test_category_distribution(con):
|
||||
"""There are exactly 10 categories and all 3,030 motions are classified."""
|
||||
df = con.execute("""
|
||||
SELECT category, COUNT(*) as cnt
|
||||
FROM right_wing_motions
|
||||
WHERE classified = TRUE
|
||||
GROUP BY category
|
||||
ORDER BY cnt DESC
|
||||
""").fetchdf()
|
||||
assert len(df) == 10
|
||||
assert df["cnt"].sum() == 3030
|
||||
assert df[df["category"] == "overig"]["cnt"].values[0] >= 100
|
||||
|
||||
|
||||
def test_category_deltas(con):
|
||||
"""Pre/post CS deltas (2024 split) — ALL categories gained, energie/klimaat leads."""
|
||||
df = con.execute("""
|
||||
SELECT
|
||||
category,
|
||||
AVG(CASE WHEN year < 2024 THEN centrist_support_strict END) as pre_cs,
|
||||
AVG(CASE WHEN year >= 2024 THEN centrist_support_strict END) as post_cs,
|
||||
COUNT(*) as n
|
||||
FROM right_wing_motions
|
||||
WHERE classified = TRUE AND year >= 2017
|
||||
GROUP BY category
|
||||
""").fetchdf()
|
||||
|
||||
assert len(df) == 10
|
||||
df = df.copy()
|
||||
df["delta"] = df["post_cs"] - df["pre_cs"]
|
||||
top = df.sort_values("delta", ascending=False)
|
||||
assert top.iloc[0]["category"] == "energie/klimaat"
|
||||
assert 0.35 < abs(top.iloc[0]["delta"]) < 0.45
|
||||
|
||||
bottom = df.sort_values("delta", ascending=True)
|
||||
assert bottom.iloc[0]["category"] in ("veiligheid/justitie", "onderwijs/wetenschap")
|
||||
|
||||
assert all(df["delta"] > 0)
|
||||
|
||||
|
||||
def test_yearly_category_cs(con):
|
||||
"""Yearly CS per category returns data for every year-category combo."""
|
||||
df = con.execute("""
|
||||
SELECT year, category, AVG(centrist_support_strict) as cs, COUNT(*) as n
|
||||
FROM right_wing_motions
|
||||
WHERE classified = TRUE AND year >= 2017
|
||||
GROUP BY year, category
|
||||
ORDER BY year, category
|
||||
""").fetchdf()
|
||||
assert len(df) >= 50
|
||||
assert df["category"].nunique() == 10
|
||||
assert df["year"].nunique() >= 8
|
||||
|
||||
|
||||
def test_quarterly_category_data(con):
|
||||
"""Quarterly CS per key categories returns expected shape."""
|
||||
key_cats = ["asiel/vreemdelingen", "energie/klimaat", "buitenland/europa",
|
||||
"landbouw/natuur", "economie"]
|
||||
df = con.execute("""
|
||||
SELECT
|
||||
EXTRACT(YEAR FROM m.date) AS y,
|
||||
CEIL(EXTRACT(MONTH FROM m.date) / 3.0) AS q,
|
||||
r.category,
|
||||
AVG(r.centrist_support_strict) AS cs,
|
||||
COUNT(*) AS n
|
||||
FROM right_wing_motions r
|
||||
JOIN motions m ON r.motion_id = m.id
|
||||
WHERE r.classified = TRUE AND m.date IS NOT NULL
|
||||
AND r.category IN ({})
|
||||
GROUP BY y, q, r.category
|
||||
ORDER BY y, q
|
||||
""".format(",".join(f"'{c}'" for c in key_cats))).fetchdf()
|
||||
|
||||
assert df["category"].nunique() <= 5
|
||||
for cat in key_cats:
|
||||
assert cat in df["category"].values
|
||||
|
||||
|
||||
def test_qmd_has_domain_decomposition_section():
|
||||
"""QMD should have a Domain Decomposition heading after implementation."""
|
||||
qmd = open("reports/overton_window/overton_window.qmd").read()
|
||||
assert "## Domain Decomposition" in qmd
|
||||
|
||||
|
||||
def test_qmd_has_category_delta_chart():
|
||||
"""QMD should have a category delta bar chart cell."""
|
||||
qmd = open("reports/overton_window/overton_window.qmd").read()
|
||||
assert "chart-7-category-delta" in qmd
|
||||
|
||||
|
||||
def test_qmd_has_category_filter_dropdown():
|
||||
"""Chart 1 should have updatemenu for category filtering."""
|
||||
qmd = open("reports/overton_window/overton_window.qmd").read()
|
||||
assert "updatemenus" in qmd
|
||||
|
||||
|
||||
def test_qmd_has_domain_trajectories():
|
||||
"""Quarterly chart should have category trajectories."""
|
||||
qmd = open("reports/overton_window/overton_window.qmd").read()
|
||||
assert "chart-8-domain-trajectories" in qmd
|
||||
Reference in New Issue
Block a user