feat(extremity): two-dimensional rescoring with subagent pipeline
- Project-local skill .opencode/skills/score-extremity/ for subagent dispatch - Orchestrator extremity_rescore_2d.py with load_skill/sample/format/validate/store - 16 TDD tests covering all orchestrator functions - 117 motions scored by deepseek v4 flash subagents (12 parallel batches) - Pearson r=0.45 between stylistic and material dimensions — separable - Key finding: 36.8% of motions use restrained language for consequential policies - 2d_extremity_correlation_report.md documents distribution, divergence patterns, and implications for the Overton acceptance-without-conversion narrative
This commit is contained in:
@@ -0,0 +1,362 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Two-dimensional extremity rescoring orchestrator.
|
||||
|
||||
Scores Dutch parliamentary motions on two independent dimensions:
|
||||
1. stijl_extremiteit (stylistic extremity, 1-5)
|
||||
2. materiele_impact (material impact, 1-5)
|
||||
|
||||
Usage:
|
||||
uv run python analysis/right_wing/extremity_rescore_2d.py --db data/motions.db
|
||||
uv run python analysis/right_wing/extremity_rescore_2d.py --db data/motions.db --dry-run
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import duckdb
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── prompt / schema loading ──────────────────────────────────────────────────
|
||||
|
||||
SKILL_MD_PATH = Path(__file__).parent.parent.parent / ".opencode" / "skills" / "score-extremity" / "SKILL.md"
|
||||
|
||||
|
||||
def load_skill(skill_path: str | None = None) -> dict[str, Any]:
|
||||
"""Read SKILL.md and extract prompt template and output schemas.
|
||||
|
||||
Returns:
|
||||
dict with keys "prompt_template", "single_schema", "batch_schema".
|
||||
"""
|
||||
path = Path(skill_path) if skill_path else SKILL_MD_PATH
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Skill file not found: {path}")
|
||||
|
||||
content = path.read_text(encoding="utf-8")
|
||||
|
||||
# Extract prompt template from ```text ... ``` block
|
||||
prompt_match = re.search(r"```text\n(.*?)```", content, re.DOTALL)
|
||||
prompt_template = prompt_match.group(1).strip() if prompt_match else ""
|
||||
|
||||
# Extract JSON schema blocks (first = single, second = batch)
|
||||
json_blocks = re.findall(r"```json\n(.*?)```", content, re.DOTALL)
|
||||
|
||||
single_schema: dict[str, Any] = {}
|
||||
batch_schema: dict[str, Any] = {}
|
||||
if len(json_blocks) >= 1:
|
||||
try:
|
||||
single_schema = json.loads(json_blocks[0].strip())
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Failed to parse single schema JSON block")
|
||||
if len(json_blocks) >= 2:
|
||||
try:
|
||||
batch_schema = json.loads(json_blocks[1].strip())
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Failed to parse batch schema JSON block")
|
||||
|
||||
return {
|
||||
"prompt_template": prompt_template,
|
||||
"single_schema": single_schema,
|
||||
"batch_schema": batch_schema,
|
||||
}
|
||||
|
||||
|
||||
# ── sampling ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def sample_motions(
|
||||
db_path: str,
|
||||
n_per_bucket: int = 25,
|
||||
seed: int = 42,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Stratified sample from right_wing_motions JOIN extremity_scores.
|
||||
|
||||
Samples n_per_bucket motions from each text_score bucket (1-5).
|
||||
|
||||
Returns:
|
||||
List of dicts with keys: motion_id, title, text, layman, text_score.
|
||||
"""
|
||||
con = duckdb.connect(db_path)
|
||||
try:
|
||||
# Ensure tables exist
|
||||
tables = {t[0] for t in con.execute("SHOW TABLES").fetchall()}
|
||||
required = {"right_wing_motions", "motions", "extremity_scores"}
|
||||
missing = required - tables
|
||||
if missing:
|
||||
logger.warning("Missing tables: %s, returning empty sample", missing)
|
||||
return []
|
||||
|
||||
# Apply seed for reproducibility
|
||||
con.execute(f"SELECT setseed({seed / 1000000.0})")
|
||||
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT m.id, m.title, m.body_text, m.layman_explanation, e.text_score
|
||||
FROM right_wing_motions r
|
||||
JOIN motions m ON r.motion_id = m.id
|
||||
JOIN extremity_scores e ON r.motion_id = e.motion_id
|
||||
WHERE r.classified = TRUE
|
||||
AND e.text_score IS NOT NULL
|
||||
AND e.error IS NULL
|
||||
ORDER BY RANDOM()
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
# Bucket by text_score
|
||||
buckets: dict[int, list[dict[str, Any]]] = {}
|
||||
for row in rows:
|
||||
mid, title, body_text, layman, text_score = row
|
||||
score_bucket = int(text_score)
|
||||
buckets.setdefault(score_bucket, []).append({
|
||||
"motion_id": mid,
|
||||
"title": title or "",
|
||||
"text": body_text or "",
|
||||
"layman": layman or "",
|
||||
"text_score": score_bucket,
|
||||
})
|
||||
|
||||
# Sample n_per_bucket from each bucket
|
||||
result: list[dict[str, Any]] = []
|
||||
for bucket_id in sorted(buckets.keys()):
|
||||
bucket = buckets[bucket_id]
|
||||
result.extend(bucket[:n_per_bucket])
|
||||
|
||||
logger.info(
|
||||
"Sampled %d motions from %d buckets (n_per_bucket=%d)",
|
||||
len(result), len(buckets), n_per_bucket,
|
||||
)
|
||||
return result
|
||||
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
# ── batch formatting ─────────────────────────────────────────────────────────
|
||||
|
||||
def format_batches(
|
||||
motions: list[dict[str, Any]],
|
||||
prompt_template: str,
|
||||
batch_size: int = 10,
|
||||
) -> list[list[str]]:
|
||||
"""Split motions into batches and fill prompt template for each motion.
|
||||
|
||||
Args:
|
||||
motions: List of dicts with keys title, text, layman.
|
||||
prompt_template: Template string with {title}, {text}, {layman} placeholders.
|
||||
batch_size: Number of motions per batch.
|
||||
|
||||
Returns:
|
||||
List of batches; each batch is a list of filled prompt strings, one per motion.
|
||||
"""
|
||||
batches: list[list[str]] = []
|
||||
for i in range(0, len(motions), batch_size):
|
||||
batch_motions = motions[i : i + batch_size]
|
||||
batch_prompts: list[str] = []
|
||||
for m in batch_motions:
|
||||
prompt = prompt_template.format(
|
||||
title=m.get("title", ""),
|
||||
text=m.get("text", ""),
|
||||
layman=m.get("layman", ""),
|
||||
)
|
||||
batch_prompts.append(prompt)
|
||||
batches.append(batch_prompts)
|
||||
return batches
|
||||
|
||||
|
||||
# ── validation ───────────────────────────────────────────────────────────────
|
||||
|
||||
EXPECTED_FIELDS = [
|
||||
"stijl_extremiteit",
|
||||
"stijl_toelichting",
|
||||
"materiele_impact",
|
||||
"materiele_toelichting",
|
||||
]
|
||||
|
||||
|
||||
def validate_single_result(result: dict[str, Any]) -> tuple[bool, str | None]:
|
||||
"""Validate a single motion 2d scoring result.
|
||||
|
||||
Returns:
|
||||
(True, None) if valid, (False, error_message) otherwise.
|
||||
"""
|
||||
# Check all required fields exist
|
||||
for field in EXPECTED_FIELDS:
|
||||
if field not in result:
|
||||
return False, f"missing field: {field}"
|
||||
|
||||
# Validate stijl_extremiteit (int, 1-5)
|
||||
se = result["stijl_extremiteit"]
|
||||
if not isinstance(se, int) or se < 1 or se > 5:
|
||||
return False, f"stijl_extremiteit out of range 1-5: {se}"
|
||||
|
||||
# Validate materiele_impact (int, 1-5)
|
||||
mi = result["materiele_impact"]
|
||||
if not isinstance(mi, int) or mi < 1 or mi > 5:
|
||||
return False, f"materiele_impact out of range 1-5: {mi}"
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
# ── storage ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def store_scores(db_path: str, results: list[dict[str, Any]]) -> int:
|
||||
"""Store validated 2d scores in the extremity_scores_2d table.
|
||||
|
||||
Creates the table if it doesn't exist.
|
||||
|
||||
Args:
|
||||
db_path: Path to DuckDB database.
|
||||
results: List of dicts with keys: motion_id, stijl_extremiteit,
|
||||
stijl_toelichting, materiele_impact, materiele_toelichting.
|
||||
|
||||
Returns:
|
||||
Number of rows inserted.
|
||||
"""
|
||||
con = duckdb.connect(db_path)
|
||||
try:
|
||||
con.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS extremity_scores_2d (
|
||||
motion_id INTEGER PRIMARY KEY,
|
||||
stylistic_score INTEGER NOT NULL,
|
||||
material_score INTEGER NOT NULL,
|
||||
stylistic_rationale TEXT,
|
||||
material_rationale TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
count = 0
|
||||
for r in results:
|
||||
con.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO extremity_scores_2d
|
||||
(motion_id, stylistic_score, material_score, stylistic_rationale, material_rationale)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
r["motion_id"],
|
||||
r["stijl_extremiteit"],
|
||||
r["materiele_impact"],
|
||||
r.get("stijl_toelichting"),
|
||||
r.get("materiele_toelichting"),
|
||||
),
|
||||
)
|
||||
count += 1
|
||||
|
||||
con.commit()
|
||||
logger.info("Stored %d scores in extremity_scores_2d", count)
|
||||
return count
|
||||
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
# ── orchestrator ─────────────────────────────────────────────────────────────
|
||||
|
||||
def rescore_2d(
|
||||
db_path: str,
|
||||
n_per_bucket: int = 25,
|
||||
batch_size: int = 10,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Two-dimensional extremity rescoring orchestrator.
|
||||
|
||||
Samples motions from right_wing_motions/extremity_scores, formats batches,
|
||||
and (in non-dry-run mode) dispatches subagents for scoring.
|
||||
|
||||
Args:
|
||||
db_path: Path to DuckDB database.
|
||||
n_per_bucket: Number of motions to sample per text_score bucket.
|
||||
batch_size: Motions per subagent batch.
|
||||
dry_run: If True, only print the plan without spawning subagents.
|
||||
|
||||
Returns:
|
||||
Dict with summary stats.
|
||||
"""
|
||||
skill = load_skill()
|
||||
prompt_template = skill["prompt_template"]
|
||||
|
||||
motions = sample_motions(db_path, n_per_bucket=n_per_bucket)
|
||||
|
||||
if not motions:
|
||||
logger.warning("No motions to rescore.")
|
||||
return {"motions_count": 0, "batch_count": 0, "dry_run": dry_run}
|
||||
|
||||
batches = format_batches(motions, prompt_template, batch_size=batch_size)
|
||||
|
||||
logger.info("Plan: %d motions in %d batches (batch_size=%d)", len(motions), len(batches), batch_size)
|
||||
|
||||
if dry_run:
|
||||
logger.info("DRY RUN — no subagents will be spawned.")
|
||||
return {
|
||||
"motions_count": len(motions),
|
||||
"batch_count": len(batches),
|
||||
"dry_run": True,
|
||||
}
|
||||
|
||||
# ── subagent dispatch (placeholder) ──────────────────────────────────
|
||||
# In production, each batch would be sent to a subagent via the `task` tool.
|
||||
# The subagent receives:
|
||||
# - The prompt_template filled with motion data
|
||||
# - Instruction to return JSON matching the batch_schema
|
||||
#
|
||||
# Example dispatch (not executed in script):
|
||||
# for batch_idx, batch_prompts in enumerate(batches):
|
||||
# combined_prompt = "\n\n---\n\n".join(batch_prompts)
|
||||
# result = task(
|
||||
# description=f"Score batch {batch_idx + 1}/{len(batches)}",
|
||||
# prompt=combined_prompt,
|
||||
# subagent_type="general",
|
||||
# )
|
||||
# validated_results = [r for r in json.loads(result)["motions"] if validate_single_result(r)[0]]
|
||||
# store_scores(db_path, validated_results)
|
||||
|
||||
logger.info(
|
||||
"Subagent dispatch placeholder: %d batches ready for scoring. "
|
||||
"Run via an agent context (e.g. opencode task) to execute.",
|
||||
len(batches),
|
||||
)
|
||||
|
||||
return {
|
||||
"motions_count": len(motions),
|
||||
"batch_count": len(batches),
|
||||
"dry_run": False,
|
||||
"subagents_spawned": 0,
|
||||
}
|
||||
|
||||
|
||||
# ── CLI ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Two-dimensional extremity rescoring orchestrator"
|
||||
)
|
||||
parser.add_argument("--db", default="data/motions.db", help="Path to DuckDB database")
|
||||
parser.add_argument("--n-per-bucket", type=int, default=25, help="Motions per text_score bucket")
|
||||
parser.add_argument("--batch-size", type=int, default=10, help="Motions per subagent batch")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print plan without spawning subagents")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = rescore_2d(
|
||||
db_path=args.db,
|
||||
n_per_bucket=args.n_per_bucket,
|
||||
batch_size=args.batch_size,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,112 @@
|
||||
# Two-Dimensional Extremity Correlation Report
|
||||
|
||||
**Date:** 2026-05-24
|
||||
**Motions scored:** 117 (stratified sample: ~25 per original extremity bucket)
|
||||
**Scoring model:** Deepseek v4 flash (subagents via project skill)
|
||||
|
||||
## Purpose
|
||||
|
||||
The original extremity score is a single 1–5 rating of policy radicalism. This conflates two potentially independent dimensions:
|
||||
- **Stylistic extremity (stijl-extremiteit):** How inflammatory, hostile, or polarizing the language is
|
||||
- **Material impact (materiële impact):** How much the proposed policy would substantively affect people's rights, institutions, or freedoms
|
||||
|
||||
This validation samples motions across the full extremity range and scores both dimensions independently to test whether they correlate strongly enough for a single score, or whether they should be tracked separately.
|
||||
|
||||
---
|
||||
|
||||
## Results
|
||||
|
||||
### Overall correlation
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| N | 117 |
|
||||
| Pearson r | **0.453** (moderate) |
|
||||
| Mean stylistic | 2.01 |
|
||||
| Mean material | 2.86 |
|
||||
| Mean absolute difference | 1.11 |
|
||||
| S ≤ 2 AND M ≥ 3 (masking) | 43 (36.8%) |
|
||||
|
||||
**r = 0.453 is moderate — the dimensions are partly correlated but clearly separable.** Stylistic extremism explains only ~20% of the variance in material impact (R² = 0.205). A motion can be inflammatory without being consequential, and vice versa.
|
||||
|
||||
### Joint distribution
|
||||
|
||||
| | M=1 | M=2 | M=3 | M=4 | M=5 |
|
||||
|---|---|---|---|---|---|
|
||||
| **S=1** | 11 | 17 | 10 | 5 | 1 |
|
||||
| **S=2** | 4 | 9 | 15 | 8 | 4 |
|
||||
| **S=3** | 2 | 4 | 9 | 4 | 5 |
|
||||
| **S=4** | 0 | 1 | 0 | 3 | 2 |
|
||||
| **S=5** | 0 | 0 | 0 | 1 | 2 |
|
||||
|
||||
### By original extremity bucket
|
||||
|
||||
| Bucket | N | Mean style | Mean material | Gap |
|
||||
|--------|---|-----------|--------------|-----|
|
||||
| 1–2 (mild) | 50 | 1.56 | 2.24 | +0.68 |
|
||||
| 2–3 (moderate) | 25 | 2.00 | 2.88 | +0.88 |
|
||||
| 3–4 (high) | 25 | 2.56 | 3.56 | +1.00 |
|
||||
| 4–5 (extreme) | 17 | 2.53 | 3.65 | +1.12 |
|
||||
|
||||
Material impact consistently rates higher than stylistic extremity across all buckets. The gap widens at higher original extremity levels — suggesting the original LLM scoring was more sensitive to language style, while subagents systematically identify greater material consequences in the same motions.
|
||||
|
||||
---
|
||||
|
||||
## Key findings
|
||||
|
||||
### 1. "Low style, high impact" is the dominant divergence pattern
|
||||
|
||||
**36.8% of motions (43 of 117)** use restrained language (S ≤ 2) for policies with substantial material impact (M ≥ 3). These are the motions most poorly captured by a single-dimensional score:
|
||||
|
||||
- **Motion 16227** (S=1, M=5): "Verzoekt de regering kennis te geven van het voornemen tot uittreding uit de Europese Unie conform artikel 50 VWEU." Neutral, procedural language invoking an EU treaty article — but the policy is fundamental dissolution of the entire Dutch-EU legal framework.
|
||||
|
||||
- **Motion 7713** (S=1, M=4): "Verzoekt de regering per direct te stoppen met arbeidsmigratie." Restrained, single-sentence motion with no inflammatory language — but it would suspend free movement of persons, a fundamental EU treaty right.
|
||||
|
||||
- **Motion 16704** (S=1, M=3): Formal Raad van State advice and technical amendment text. No political rhetoric — but a concrete law change with measurable employment and investment effects.
|
||||
|
||||
- **Motion 687** (S=1, M=3): Technical-juridical language about the scope of "emissiegegevens" in the EU environmental information directive — but would significantly restrict public transparency about agricultural emissions.
|
||||
|
||||
### 2. Material impact averages significantly higher
|
||||
|
||||
Across all buckets, material impact scores are 0.68–1.12 points higher than stylistic scores. This suggests:
|
||||
- Parliamentarians write motions using formal, restrained language even when proposing consequential policies
|
||||
- The original LLM scoring (which showed mean extremity = 2.19 overall) likely understates how radical these policies are in material terms
|
||||
- Dutch parliamentary language norms mask policy radicalism
|
||||
|
||||
### 3. "High style" motions are rare and concentrated
|
||||
|
||||
Only 3 motions scored S=5 (the most inflammatory end), and all had M=4 or M=5. Explicitly discriminatory or hostile language — when it occurs — is paired with substantively extreme policies. But the vast majority of consequential right-wing motions use parliamentary language:
|
||||
|
||||
- **Motion 11956** (S=4, M=5): Explicitly hostile language ("à la Turkije," "vreemdelingen die we hier niet willen hebben") paired with fundamental rights violation (forced deportation without country-of-origin consent)
|
||||
- **Motion 18064** (S=5, M=4): Explicit ethnic targeting ("niet-westerse allochtonen" as COVID rulebreakers) — discriminatory state action
|
||||
|
||||
### 4. The original LLM audit gap is partially explained
|
||||
|
||||
The manual audit found 75% agreement with the original LLM scores and noted "systematic overrating of anti-institutional language." The two-dimensional data clarifies this: the original LLM was more sensitive to *stylistic* extremity (inflammatory language) than to *material* policy impact. The 25% disagreement likely occurred on "low style, high impact" motions where the single-dimensional score was anchored to language rather than substance.
|
||||
|
||||
---
|
||||
|
||||
## Implications for Overton analysis
|
||||
|
||||
### For the current findings
|
||||
|
||||
The "no content extremity increase" (d = −0.09) finding in the Overton report relied on single-dimensional LLM scores. The two-dimensional data suggests this may be an **artifact of the language-focused scoring**: if right-wing motions became more consequential while maintaining or softening their language, the single score would miss the shift entirely.
|
||||
|
||||
The "acceptance without conversion" interpretation — centrists vote more with right-wing despite spatial divergence — is **strengthened** by these findings. It is consistent with right-wing motions becoming *substantively* consequential (high material impact) while maintaining procedural language norms, making them harder for centrists to vote against without appearing obstructionist.
|
||||
|
||||
### Recommendations
|
||||
|
||||
1. **Re-score all 2,986 motions with two-dimensional scoring.** The moderate r = 0.453 confirms the dimensions are separable. A single score obscures the most important category: motions with low stylistic extremism but high material impact.
|
||||
|
||||
2. **Re-run the extremity-stratified centrist support analysis with material impact buckets.** The critical question: did centrist support for *high material impact* motions increase after 2024? If low-language, high-impact motions are the ones gaining centrist tolerance, that is stronger Overton evidence than the current analysis captures.
|
||||
|
||||
3. **For mechanism analysis (U4):** Score mechanisms specifically for *material impact* rather than general extremity. The question is not "how extreme is this motion?" but "what specific rights, institutions, or groups does this motion affect, and how much?"
|
||||
|
||||
---
|
||||
|
||||
## Data
|
||||
|
||||
- **Full results:** `data/motions.db` → `extremity_scores_2d` (117 rows)
|
||||
- **Raw JSON:** `/tmp/extremity_2d_results.json`
|
||||
- **Scoring skill:** `.opencode/skills/score-extremity/SKILL.md`
|
||||
- **Orchestrator:** `analysis/right_wing/extremity_rescore_2d.py`
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Tests for two-dimensional extremity rescoring orchestrator."""
|
||||
|
||||
import json
|
||||
|
||||
import duckdb
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("duckdb")
|
||||
|
||||
|
||||
# ── fixtures ────────────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def synthetic_motions():
|
||||
"""Return 103 synthetic motion dicts for testing batch formatting."""
|
||||
motions = []
|
||||
for i in range(103):
|
||||
motions.append({
|
||||
"motion_id": i + 1,
|
||||
"title": f"Motion {i + 1}",
|
||||
"text": f"Body text for motion {i + 1}",
|
||||
"layman": f"Layman explanation {i + 1}",
|
||||
})
|
||||
return motions
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prompt_template():
|
||||
"""Minimal prompt template with {title}, {text}, {layman} placeholders."""
|
||||
return (
|
||||
"Titel: {title}\n"
|
||||
"Tekst: {text}\n"
|
||||
"Uitleg: {layman}\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def valid_single_result():
|
||||
"""A valid single-motion 2d result dict."""
|
||||
return {
|
||||
"stijl_extremiteit": 3,
|
||||
"stijl_toelichting": "Neutraal taalgebruik",
|
||||
"materiele_impact": 4,
|
||||
"materiele_toelichting": "Beperkt rechten voor specifieke groep",
|
||||
}
|
||||
|
||||
|
||||
# ── load_skill tests ────────────────────────────────────────────────────────
|
||||
|
||||
class TestLoadSkill:
|
||||
def test_returns_prompt_and_schema(self):
|
||||
from analysis.right_wing.extremity_rescore_2d import load_skill
|
||||
|
||||
result = load_skill()
|
||||
assert isinstance(result, dict)
|
||||
assert "prompt_template" in result
|
||||
assert "batch_schema" in result
|
||||
assert "single_schema" in result
|
||||
assert isinstance(result["prompt_template"], str)
|
||||
assert len(result["prompt_template"]) > 0
|
||||
assert "STIJL-EXTREMITEIT" in result["prompt_template"]
|
||||
assert "MATERIELE IMPACT" in result["prompt_template"]
|
||||
assert isinstance(result["batch_schema"], dict)
|
||||
assert "motions" in result["batch_schema"]
|
||||
assert isinstance(result["single_schema"], dict)
|
||||
|
||||
def test_missing_file_raises(self):
|
||||
from analysis.right_wing.extremity_rescore_2d import load_skill
|
||||
|
||||
with pytest.raises(FileNotFoundError, match="not found"):
|
||||
load_skill(skill_path="/nonexistent/path/skill.md")
|
||||
|
||||
|
||||
# ── format_batches tests ────────────────────────────────────────────────────
|
||||
|
||||
class TestFormatBatches:
|
||||
def test_splits_into_batches(self, synthetic_motions, prompt_template):
|
||||
from analysis.right_wing.extremity_rescore_2d import format_batches
|
||||
|
||||
batches = format_batches(synthetic_motions[:100], prompt_template, batch_size=10)
|
||||
assert isinstance(batches, list)
|
||||
assert len(batches) == 10
|
||||
for batch in batches:
|
||||
assert isinstance(batch, list)
|
||||
assert len(batch) == 10
|
||||
for prompt_str in batch:
|
||||
assert "Motion" in prompt_str
|
||||
|
||||
def test_uneven_batches(self, synthetic_motions, prompt_template):
|
||||
from analysis.right_wing.extremity_rescore_2d import format_batches
|
||||
|
||||
batches = format_batches(synthetic_motions, prompt_template, batch_size=10)
|
||||
assert len(batches) == 11
|
||||
for batch in batches[:-1]:
|
||||
assert len(batch) == 10
|
||||
assert len(batches[-1]) == 3
|
||||
|
||||
def test_substitutes_placeholders(self, prompt_template):
|
||||
from analysis.right_wing.extremity_rescore_2d import format_batches
|
||||
|
||||
motions = [{
|
||||
"motion_id": 1,
|
||||
"title": "Test Title",
|
||||
"text": "Test Text",
|
||||
"layman": "Test Layman",
|
||||
}]
|
||||
batches = format_batches(motions, prompt_template, batch_size=1)
|
||||
prompt_str = batches[0][0]
|
||||
assert "Test Title" in prompt_str
|
||||
assert "Test Text" in prompt_str
|
||||
assert "Test Layman" in prompt_str
|
||||
|
||||
|
||||
# ── validate_single_result tests ────────────────────────────────────────────
|
||||
|
||||
class TestValidateSingleResult:
|
||||
def test_valid_result(self, valid_single_result):
|
||||
from analysis.right_wing.extremity_rescore_2d import validate_single_result
|
||||
|
||||
ok, err = validate_single_result(valid_single_result)
|
||||
assert ok is True
|
||||
assert err is None
|
||||
|
||||
def test_missing_field(self, valid_single_result):
|
||||
from analysis.right_wing.extremity_rescore_2d import validate_single_result
|
||||
|
||||
invalid = dict(valid_single_result)
|
||||
del invalid["materiele_impact"]
|
||||
ok, err = validate_single_result(invalid)
|
||||
assert ok is False
|
||||
assert "materiele_impact" in err
|
||||
|
||||
def test_out_of_range_high(self, valid_single_result):
|
||||
from analysis.right_wing.extremity_rescore_2d import validate_single_result
|
||||
|
||||
invalid = dict(valid_single_result)
|
||||
invalid["stijl_extremiteit"] = 6
|
||||
ok, err = validate_single_result(invalid)
|
||||
assert ok is False
|
||||
assert "stijl_extremiteit" in err
|
||||
|
||||
def test_out_of_range_low(self, valid_single_result):
|
||||
from analysis.right_wing.extremity_rescore_2d import validate_single_result
|
||||
|
||||
invalid = dict(valid_single_result)
|
||||
invalid["materiele_impact"] = 0
|
||||
ok, err = validate_single_result(invalid)
|
||||
assert ok is False
|
||||
assert "materiele_impact" in err
|
||||
|
||||
def test_non_integer_score(self, valid_single_result):
|
||||
from analysis.right_wing.extremity_rescore_2d import validate_single_result
|
||||
|
||||
invalid = dict(valid_single_result)
|
||||
invalid["stijl_extremiteit"] = "3"
|
||||
ok, err = validate_single_result(invalid)
|
||||
assert ok is False
|
||||
assert "stijl_extremiteit" in err
|
||||
|
||||
|
||||
# ── store_scores tests ──────────────────────────────────────────────────────
|
||||
|
||||
class TestStoreScores:
|
||||
def test_stores_and_returns_count(self, tmp_duckdb_path):
|
||||
import duckdb
|
||||
from analysis.right_wing.extremity_rescore_2d import store_scores
|
||||
|
||||
results = [
|
||||
{"motion_id": 1, "stijl_extremiteit": 3, "stijl_toelichting": "a",
|
||||
"materiele_impact": 4, "materiele_toelichting": "b"},
|
||||
{"motion_id": 2, "stijl_extremiteit": 2, "stijl_toelichting": "c",
|
||||
"materiele_impact": 1, "materiele_toelichting": "d"},
|
||||
]
|
||||
count = store_scores(tmp_duckdb_path, results)
|
||||
assert count == 2
|
||||
|
||||
con = duckdb.connect(tmp_duckdb_path)
|
||||
try:
|
||||
rows = con.execute(
|
||||
"SELECT motion_id, stylistic_score, material_score "
|
||||
"FROM extremity_scores_2d ORDER BY motion_id"
|
||||
).fetchall()
|
||||
assert len(rows) == 2
|
||||
assert rows[0] == (1, 3, 4)
|
||||
assert rows[1] == (2, 2, 1)
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
def test_replace_existing(self, tmp_duckdb_path):
|
||||
import duckdb
|
||||
from analysis.right_wing.extremity_rescore_2d import store_scores
|
||||
|
||||
results = [{
|
||||
"motion_id": 1, "stijl_extremiteit": 1, "stijl_toelichting": "x",
|
||||
"materiele_impact": 1, "materiele_toelichting": "y",
|
||||
}]
|
||||
store_scores(tmp_duckdb_path, results)
|
||||
|
||||
updated = [{
|
||||
"motion_id": 1, "stijl_extremiteit": 5, "stijl_toelichting": "z",
|
||||
"materiele_impact": 5, "materiele_toelichting": "w",
|
||||
}]
|
||||
count = store_scores(tmp_duckdb_path, updated)
|
||||
assert count == 1
|
||||
|
||||
con = duckdb.connect(tmp_duckdb_path)
|
||||
try:
|
||||
rows = con.execute(
|
||||
"SELECT stylistic_score, material_score FROM extremity_scores_2d WHERE motion_id = 1"
|
||||
).fetchall()
|
||||
assert rows[0] == (5, 5)
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
# ── sample_motions tests ────────────────────────────────────────────────────
|
||||
|
||||
class TestSampleMotions:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_db(self, tmp_duckdb_path):
|
||||
"""Set up right_wing_motions and extremity_scores tables with synthetic data."""
|
||||
con = duckdb.connect(tmp_duckdb_path)
|
||||
try:
|
||||
con.execute("""
|
||||
CREATE TABLE IF NOT EXISTS right_wing_motions (
|
||||
motion_id INTEGER PRIMARY KEY,
|
||||
classified BOOLEAN DEFAULT TRUE
|
||||
)
|
||||
""")
|
||||
con.execute("""
|
||||
CREATE TABLE IF NOT EXISTS motions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
title VARCHAR,
|
||||
body_text VARCHAR,
|
||||
layman_explanation VARCHAR
|
||||
)
|
||||
""")
|
||||
con.execute("""
|
||||
CREATE TABLE IF NOT EXISTS extremity_scores (
|
||||
motion_id INTEGER PRIMARY KEY,
|
||||
text_score INTEGER,
|
||||
text_explanation VARCHAR,
|
||||
layman_score INTEGER,
|
||||
layman_explanation VARCHAR,
|
||||
error VARCHAR
|
||||
)
|
||||
""")
|
||||
# Insert motions across 4 text_score buckets: 1, 2, 4, 5
|
||||
records = []
|
||||
for bucket, score in enumerate([1, 2, 4, 5], start=1):
|
||||
for i in range(15):
|
||||
mid = (bucket - 1) * 15 + i + 1
|
||||
con.execute(
|
||||
"INSERT INTO motions VALUES (?, ?, ?, ?)",
|
||||
(mid, f"Title {mid}", f"Text {mid}", f"Layman {mid}"),
|
||||
)
|
||||
con.execute(
|
||||
"INSERT INTO right_wing_motions VALUES (?, TRUE)",
|
||||
(mid,),
|
||||
)
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO extremity_scores VALUES (?, ?, '', ?, '', NULL)",
|
||||
(mid, score, score),
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
def test_returns_stratified_sample(self, tmp_duckdb_path):
|
||||
from analysis.right_wing.extremity_rescore_2d import sample_motions
|
||||
|
||||
result = sample_motions(tmp_duckdb_path, n_per_bucket=5, seed=42)
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 20 # 4 buckets * 5 each
|
||||
for row in result:
|
||||
assert "motion_id" in row
|
||||
assert "title" in row
|
||||
assert "text" in row
|
||||
assert "layman" in row
|
||||
assert "text_score" in row
|
||||
|
||||
def test_respects_seed(self, tmp_duckdb_path):
|
||||
from analysis.right_wing.extremity_rescore_2d import sample_motions
|
||||
|
||||
result_a = sample_motions(tmp_duckdb_path, n_per_bucket=3, seed=99)
|
||||
result_b = sample_motions(tmp_duckdb_path, n_per_bucket=3, seed=99)
|
||||
ids_a = sorted(r["motion_id"] for r in result_a)
|
||||
ids_b = sorted(r["motion_id"] for r in result_b)
|
||||
assert ids_a == ids_b
|
||||
|
||||
def test_n_per_bucket_limits(self, tmp_duckdb_path):
|
||||
from analysis.right_wing.extremity_rescore_2d import sample_motions
|
||||
|
||||
result = sample_motions(tmp_duckdb_path, n_per_bucket=2, seed=1)
|
||||
assert len(result) == 8 # 4 buckets * 2
|
||||
|
||||
|
||||
# ── rescore_2d dry_run tests ────────────────────────────────────────────────
|
||||
|
||||
class TestRescore2dDryRun:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_db(self, tmp_duckdb_path):
|
||||
"""Set up minimal tables for dry_run test."""
|
||||
con = duckdb.connect(tmp_duckdb_path)
|
||||
try:
|
||||
con.execute("""
|
||||
CREATE TABLE IF NOT EXISTS right_wing_motions (
|
||||
motion_id INTEGER PRIMARY KEY,
|
||||
classified BOOLEAN DEFAULT TRUE
|
||||
)
|
||||
""")
|
||||
con.execute("""
|
||||
CREATE TABLE IF NOT EXISTS motions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
title VARCHAR,
|
||||
body_text VARCHAR,
|
||||
layman_explanation VARCHAR
|
||||
)
|
||||
""")
|
||||
con.execute("""
|
||||
CREATE TABLE IF NOT EXISTS extremity_scores (
|
||||
motion_id INTEGER PRIMARY KEY,
|
||||
text_score INTEGER,
|
||||
text_explanation VARCHAR,
|
||||
layman_score INTEGER,
|
||||
layman_explanation VARCHAR,
|
||||
error VARCHAR
|
||||
)
|
||||
""")
|
||||
for mid in range(1, 21):
|
||||
con.execute(
|
||||
"INSERT INTO motions VALUES (?, ?, ?, ?)",
|
||||
(mid, f"Title {mid}", f"Text {mid}", f"Layman {mid}"),
|
||||
)
|
||||
con.execute(
|
||||
"INSERT INTO right_wing_motions VALUES (?, TRUE)",
|
||||
(mid,),
|
||||
)
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO extremity_scores VALUES (?, ?, '', ?, '', NULL)",
|
||||
(mid, (mid % 5) + 1, (mid % 5) + 1),
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
def test_dry_run_no_subagents(self, tmp_duckdb_path, caplog):
|
||||
from analysis.right_wing.extremity_rescore_2d import rescore_2d
|
||||
|
||||
import logging
|
||||
caplog.set_level(logging.INFO)
|
||||
|
||||
result = rescore_2d(tmp_duckdb_path, n_per_bucket=3, dry_run=True)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("dry_run") is True
|
||||
assert "motions_count" in result
|
||||
assert "batch_count" in result
|
||||
|
||||
combined = caplog.text.lower()
|
||||
assert "dry run" in combined
|
||||
Reference in New Issue
Block a user