feat(extremity): two-dimensional rescoring with subagent pipeline

- Project-local skill .opencode/skills/score-extremity/ for subagent dispatch - Orchestrator extremity_rescore_2d.py with load_skill/sample/format/validate/store - 16 TDD tests covering all orchestrator functions - 117 motions scored by deepseek v4 flash subagents (12 parallel batches) - Pearson r=0.45 between stylistic and material dimensions — separable - Key finding: 36.8% of motions use restrained language for consequential policies - 2d_extremity_correlation_report.md documents distribution, divergence patterns, and implications for the Overton acceptance-without-conversion narrative
2026-05-24 23:13:42 +02:00
parent 10fc002ef9
commit bf37f84a8b
3 changed files with 834 additions and 0 deletions
@@ -0,0 +1,360 @@
+"""Tests for two-dimensional extremity rescoring orchestrator."""
+
+import json
+
+import duckdb
+import pytest
+
+pytest.importorskip("duckdb")
+
+
+# ── fixtures ────────────────────────────────────────────────────────────────
+
+@pytest.fixture
+def synthetic_motions():
+    """Return 103 synthetic motion dicts for testing batch formatting."""
+    motions = []
+    for i in range(103):
+        motions.append({
+            "motion_id": i + 1,
+            "title": f"Motion {i + 1}",
+            "text": f"Body text for motion {i + 1}",
+            "layman": f"Layman explanation {i + 1}",
+        })
+    return motions
+
+
+@pytest.fixture
+def prompt_template():
+    """Minimal prompt template with {title}, {text}, {layman} placeholders."""
+    return (
+        "Titel: {title}\n"
+        "Tekst: {text}\n"
+        "Uitleg: {layman}\n"
+    )
+
+
+@pytest.fixture
+def valid_single_result():
+    """A valid single-motion 2d result dict."""
+    return {
+        "stijl_extremiteit": 3,
+        "stijl_toelichting": "Neutraal taalgebruik",
+        "materiele_impact": 4,
+        "materiele_toelichting": "Beperkt rechten voor specifieke groep",
+    }
+
+
+# ── load_skill tests ────────────────────────────────────────────────────────
+
+class TestLoadSkill:
+    def test_returns_prompt_and_schema(self):
+        from analysis.right_wing.extremity_rescore_2d import load_skill
+
+        result = load_skill()
+        assert isinstance(result, dict)
+        assert "prompt_template" in result
+        assert "batch_schema" in result
+        assert "single_schema" in result
+        assert isinstance(result["prompt_template"], str)
+        assert len(result["prompt_template"]) > 0
+        assert "STIJL-EXTREMITEIT" in result["prompt_template"]
+        assert "MATERIELE IMPACT" in result["prompt_template"]
+        assert isinstance(result["batch_schema"], dict)
+        assert "motions" in result["batch_schema"]
+        assert isinstance(result["single_schema"], dict)
+
+    def test_missing_file_raises(self):
+        from analysis.right_wing.extremity_rescore_2d import load_skill
+
+        with pytest.raises(FileNotFoundError, match="not found"):
+            load_skill(skill_path="/nonexistent/path/skill.md")
+
+
+# ── format_batches tests ────────────────────────────────────────────────────
+
+class TestFormatBatches:
+    def test_splits_into_batches(self, synthetic_motions, prompt_template):
+        from analysis.right_wing.extremity_rescore_2d import format_batches
+
+        batches = format_batches(synthetic_motions[:100], prompt_template, batch_size=10)
+        assert isinstance(batches, list)
+        assert len(batches) == 10
+        for batch in batches:
+            assert isinstance(batch, list)
+            assert len(batch) == 10
+            for prompt_str in batch:
+                assert "Motion" in prompt_str
+
+    def test_uneven_batches(self, synthetic_motions, prompt_template):
+        from analysis.right_wing.extremity_rescore_2d import format_batches
+
+        batches = format_batches(synthetic_motions, prompt_template, batch_size=10)
+        assert len(batches) == 11
+        for batch in batches[:-1]:
+            assert len(batch) == 10
+        assert len(batches[-1]) == 3
+
+    def test_substitutes_placeholders(self, prompt_template):
+        from analysis.right_wing.extremity_rescore_2d import format_batches
+
+        motions = [{
+            "motion_id": 1,
+            "title": "Test Title",
+            "text": "Test Text",
+            "layman": "Test Layman",
+        }]
+        batches = format_batches(motions, prompt_template, batch_size=1)
+        prompt_str = batches[0][0]
+        assert "Test Title" in prompt_str
+        assert "Test Text" in prompt_str
+        assert "Test Layman" in prompt_str
+
+
+# ── validate_single_result tests ────────────────────────────────────────────
+
+class TestValidateSingleResult:
+    def test_valid_result(self, valid_single_result):
+        from analysis.right_wing.extremity_rescore_2d import validate_single_result
+
+        ok, err = validate_single_result(valid_single_result)
+        assert ok is True
+        assert err is None
+
+    def test_missing_field(self, valid_single_result):
+        from analysis.right_wing.extremity_rescore_2d import validate_single_result
+
+        invalid = dict(valid_single_result)
+        del invalid["materiele_impact"]
+        ok, err = validate_single_result(invalid)
+        assert ok is False
+        assert "materiele_impact" in err
+
+    def test_out_of_range_high(self, valid_single_result):
+        from analysis.right_wing.extremity_rescore_2d import validate_single_result
+
+        invalid = dict(valid_single_result)
+        invalid["stijl_extremiteit"] = 6
+        ok, err = validate_single_result(invalid)
+        assert ok is False
+        assert "stijl_extremiteit" in err
+
+    def test_out_of_range_low(self, valid_single_result):
+        from analysis.right_wing.extremity_rescore_2d import validate_single_result
+
+        invalid = dict(valid_single_result)
+        invalid["materiele_impact"] = 0
+        ok, err = validate_single_result(invalid)
+        assert ok is False
+        assert "materiele_impact" in err
+
+    def test_non_integer_score(self, valid_single_result):
+        from analysis.right_wing.extremity_rescore_2d import validate_single_result
+
+        invalid = dict(valid_single_result)
+        invalid["stijl_extremiteit"] = "3"
+        ok, err = validate_single_result(invalid)
+        assert ok is False
+        assert "stijl_extremiteit" in err
+
+
+# ── store_scores tests ──────────────────────────────────────────────────────
+
+class TestStoreScores:
+    def test_stores_and_returns_count(self, tmp_duckdb_path):
+        import duckdb
+        from analysis.right_wing.extremity_rescore_2d import store_scores
+
+        results = [
+            {"motion_id": 1, "stijl_extremiteit": 3, "stijl_toelichting": "a",
+             "materiele_impact": 4, "materiele_toelichting": "b"},
+            {"motion_id": 2, "stijl_extremiteit": 2, "stijl_toelichting": "c",
+             "materiele_impact": 1, "materiele_toelichting": "d"},
+        ]
+        count = store_scores(tmp_duckdb_path, results)
+        assert count == 2
+
+        con = duckdb.connect(tmp_duckdb_path)
+        try:
+            rows = con.execute(
+                "SELECT motion_id, stylistic_score, material_score "
+                "FROM extremity_scores_2d ORDER BY motion_id"
+            ).fetchall()
+            assert len(rows) == 2
+            assert rows[0] == (1, 3, 4)
+            assert rows[1] == (2, 2, 1)
+        finally:
+            con.close()
+
+    def test_replace_existing(self, tmp_duckdb_path):
+        import duckdb
+        from analysis.right_wing.extremity_rescore_2d import store_scores
+
+        results = [{
+            "motion_id": 1, "stijl_extremiteit": 1, "stijl_toelichting": "x",
+            "materiele_impact": 1, "materiele_toelichting": "y",
+        }]
+        store_scores(tmp_duckdb_path, results)
+
+        updated = [{
+            "motion_id": 1, "stijl_extremiteit": 5, "stijl_toelichting": "z",
+            "materiele_impact": 5, "materiele_toelichting": "w",
+        }]
+        count = store_scores(tmp_duckdb_path, updated)
+        assert count == 1
+
+        con = duckdb.connect(tmp_duckdb_path)
+        try:
+            rows = con.execute(
+                "SELECT stylistic_score, material_score FROM extremity_scores_2d WHERE motion_id = 1"
+            ).fetchall()
+            assert rows[0] == (5, 5)
+        finally:
+            con.close()
+
+
+# ── sample_motions tests ────────────────────────────────────────────────────
+
+class TestSampleMotions:
+    @pytest.fixture(autouse=True)
+    def setup_db(self, tmp_duckdb_path):
+        """Set up right_wing_motions and extremity_scores tables with synthetic data."""
+        con = duckdb.connect(tmp_duckdb_path)
+        try:
+            con.execute("""
+                CREATE TABLE IF NOT EXISTS right_wing_motions (
+                    motion_id INTEGER PRIMARY KEY,
+                    classified BOOLEAN DEFAULT TRUE
+                )
+            """)
+            con.execute("""
+                CREATE TABLE IF NOT EXISTS motions (
+                    id INTEGER PRIMARY KEY,
+                    title VARCHAR,
+                    body_text VARCHAR,
+                    layman_explanation VARCHAR
+                )
+            """)
+            con.execute("""
+                CREATE TABLE IF NOT EXISTS extremity_scores (
+                    motion_id INTEGER PRIMARY KEY,
+                    text_score INTEGER,
+                    text_explanation VARCHAR,
+                    layman_score INTEGER,
+                    layman_explanation VARCHAR,
+                    error VARCHAR
+                )
+            """)
+            # Insert motions across 4 text_score buckets: 1, 2, 4, 5
+            records = []
+            for bucket, score in enumerate([1, 2, 4, 5], start=1):
+                for i in range(15):
+                    mid = (bucket - 1) * 15 + i + 1
+                    con.execute(
+                        "INSERT INTO motions VALUES (?, ?, ?, ?)",
+                        (mid, f"Title {mid}", f"Text {mid}", f"Layman {mid}"),
+                    )
+                    con.execute(
+                        "INSERT INTO right_wing_motions VALUES (?, TRUE)",
+                        (mid,),
+                    )
+                    con.execute(
+                        "INSERT OR REPLACE INTO extremity_scores VALUES (?, ?, '', ?, '', NULL)",
+                        (mid, score, score),
+                    )
+            con.commit()
+        finally:
+            con.close()
+
+    def test_returns_stratified_sample(self, tmp_duckdb_path):
+        from analysis.right_wing.extremity_rescore_2d import sample_motions
+
+        result = sample_motions(tmp_duckdb_path, n_per_bucket=5, seed=42)
+        assert isinstance(result, list)
+        assert len(result) == 20  # 4 buckets * 5 each
+        for row in result:
+            assert "motion_id" in row
+            assert "title" in row
+            assert "text" in row
+            assert "layman" in row
+            assert "text_score" in row
+
+    def test_respects_seed(self, tmp_duckdb_path):
+        from analysis.right_wing.extremity_rescore_2d import sample_motions
+
+        result_a = sample_motions(tmp_duckdb_path, n_per_bucket=3, seed=99)
+        result_b = sample_motions(tmp_duckdb_path, n_per_bucket=3, seed=99)
+        ids_a = sorted(r["motion_id"] for r in result_a)
+        ids_b = sorted(r["motion_id"] for r in result_b)
+        assert ids_a == ids_b
+
+    def test_n_per_bucket_limits(self, tmp_duckdb_path):
+        from analysis.right_wing.extremity_rescore_2d import sample_motions
+
+        result = sample_motions(tmp_duckdb_path, n_per_bucket=2, seed=1)
+        assert len(result) == 8  # 4 buckets * 2
+
+
+# ── rescore_2d dry_run tests ────────────────────────────────────────────────
+
+class TestRescore2dDryRun:
+    @pytest.fixture(autouse=True)
+    def setup_db(self, tmp_duckdb_path):
+        """Set up minimal tables for dry_run test."""
+        con = duckdb.connect(tmp_duckdb_path)
+        try:
+            con.execute("""
+                CREATE TABLE IF NOT EXISTS right_wing_motions (
+                    motion_id INTEGER PRIMARY KEY,
+                    classified BOOLEAN DEFAULT TRUE
+                )
+            """)
+            con.execute("""
+                CREATE TABLE IF NOT EXISTS motions (
+                    id INTEGER PRIMARY KEY,
+                    title VARCHAR,
+                    body_text VARCHAR,
+                    layman_explanation VARCHAR
+                )
+            """)
+            con.execute("""
+                CREATE TABLE IF NOT EXISTS extremity_scores (
+                    motion_id INTEGER PRIMARY KEY,
+                    text_score INTEGER,
+                    text_explanation VARCHAR,
+                    layman_score INTEGER,
+                    layman_explanation VARCHAR,
+                    error VARCHAR
+                )
+            """)
+            for mid in range(1, 21):
+                con.execute(
+                    "INSERT INTO motions VALUES (?, ?, ?, ?)",
+                    (mid, f"Title {mid}", f"Text {mid}", f"Layman {mid}"),
+                )
+                con.execute(
+                    "INSERT INTO right_wing_motions VALUES (?, TRUE)",
+                    (mid,),
+                )
+                con.execute(
+                    "INSERT OR REPLACE INTO extremity_scores VALUES (?, ?, '', ?, '', NULL)",
+                    (mid, (mid % 5) + 1, (mid % 5) + 1),
+                )
+            con.commit()
+        finally:
+            con.close()
+
+    def test_dry_run_no_subagents(self, tmp_duckdb_path, caplog):
+        from analysis.right_wing.extremity_rescore_2d import rescore_2d
+
+        import logging
+        caplog.set_level(logging.INFO)
+
+        result = rescore_2d(tmp_duckdb_path, n_per_bucket=3, dry_run=True)
+        assert isinstance(result, dict)
+        assert result.get("dry_run") is True
+        assert "motions_count" in result
+        assert "batch_count" in result
+
+        combined = caplog.text.lower()
+        assert "dry run" in combined