diff --git a/ai_provider.py b/ai_provider.py index aaae765..1accc19 100644 --- a/ai_provider.py +++ b/ai_provider.py @@ -9,6 +9,7 @@ from __future__ import annotations import os import time import random +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from email.utils import parsedate_to_datetime from typing import Any @@ -55,7 +56,7 @@ def _post_with_retries( backoff = 0.5 for attempt in range(1, retries + 1): try: - resp = requests.post(url, json=json, headers=headers, timeout=10) + resp = requests.post(url, json=json, headers=headers, timeout=60) except requests.ConnectionError as exc: if attempt == retries: raise ProviderError( @@ -287,3 +288,88 @@ def chat_completion(messages: list[dict], model: str | None = None) -> str: ) from exc return str(content) + + +def chat_completion_json( + messages: list[dict], + model: str | None = None, + json_schema: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Return parsed JSON from a chat completion request using JSON mode. + + Some OpenRouter models (e.g., Google Gemma 4) support native JSON output via + the OpenAI-compatible response_format field. We request type='json_object' and + optionally supply a JSON schema in the top-level json_schema key. + """ + if not isinstance(messages, list): + raise ProviderError("messages must be a list of dicts") + + if model is None: + model = ( + os.environ.get("QWEN_MODEL") + or os.environ.get("CHAT_MODEL") + or "qwen/qwen-3.2" + ) + + payload: dict[str, Any] = {"model": model, "messages": messages} + + # Prefer explicit JSON schema (supported by some providers/OpenAI spec) + if json_schema is not None: + payload["response_format"] = { + "type": "json_schema", + "json_schema": json_schema, + } + else: + # Fallback: simple JSON object mode + payload["response_format"] = {"type": "json_object"} + + resp = _post_with_retries("/chat/completions", json=payload) + + try: + data = resp.json() + except Exception as exc: + raise ProviderError(f"Invalid JSON response from provider: {exc}") from exc + + try: + content = data["choices"][0]["message"]["content"] + except Exception as exc: + raise ProviderError( + f"Unexpected chat completion response shape: {data}" + ) from exc + + import json as _json + + try: + parsed = _json.loads(content) + except Exception as exc: + raise ProviderError(f"Model returned invalid JSON: {exc}") from exc + + if not isinstance(parsed, dict): + raise ProviderError(f"Expected JSON object, got {type(parsed).__name__}") + + return parsed + + +def chat_completion_json_parallel( + message_batches: list[list[dict]], + model: str | None = None, + json_schema: dict[str, Any] | None = None, + max_workers: int = 3, +) -> list[dict[str, Any]]: + """Send multiple chat completion requests in parallel and return parsed JSON for each. + + Useful for saturating the API when the provider supports concurrent requests. + Each item in message_batches is a separate conversation (list of messages). + Returns a list of parsed JSON dicts in the same order as the input batches. + """ + if not message_batches: + return [] + + def _fetch_one(messages: list[dict]) -> dict[str, Any]: + return chat_completion_json(messages, model=model, json_schema=json_schema) + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(_fetch_one, batch) for batch in message_batches] + results = [f.result() for f in futures] + + return results diff --git a/analysis/right_wing/classify_motions.py b/analysis/right_wing/classify_motions.py index 10946f0..fe54bca 100644 --- a/analysis/right_wing/classify_motions.py +++ b/analysis/right_wing/classify_motions.py @@ -130,11 +130,10 @@ def classify_motions( con = duckdb.connect(str(db)) try: - # Create output table - con.execute("DROP TABLE IF EXISTS right_wing_motions") + # Create output table (idempotent — does not drop existing columns) con.execute( """ - CREATE TABLE right_wing_motions ( + CREATE TABLE IF NOT EXISTS right_wing_motions ( motion_id INTEGER PRIMARY KEY, year INTEGER, title VARCHAR, diff --git a/reports/overton_window/temporal_trajectory.md b/reports/overton_window/temporal_trajectory.md index 43414a3..b779e68 100644 --- a/reports/overton_window/temporal_trajectory.md +++ b/reports/overton_window/temporal_trajectory.md @@ -14,10 +14,10 @@ showing the exact timing and shape of the Overton window shift. **Inflection point:** 2024-Q2 (first quarter where centrist_support > 0.4) **Pre-inflection mean:** 0.336 (n=25 quarters) -**Post-inflection mean:** 0.516 (n=8 quarters) +**Post-inflection mean:** 0.517 (n=9 quarters) **Peak support:** 0.648 in 2024-Q4 **Post-inflection slope:** +0.075 per quarter -**Last quarter (2026-Q1):** 0.334 +**Last quarter (2026-Q2):** 0.523 **Interpretation:** - The inflection point (2024-Q2) is the @@ -26,9 +26,9 @@ showing the exact timing and shape of the Overton window shift. a one-quarter increase of +0.18. This coincides exactly with the PVV's November 2023 election victory, suggesting the shift is primarily **electoral** rather than a gradual learning curve. -- Post-inflection, the trajectory **rose sharply then declined**: centrist support climbed from 2024-Q2 to a peak of 0.648 in 2024-Q4 (slope from inflection to peak: +0.075/quarter), then fell to 0.334 in 2026-Q1. +- Post-inflection, the trajectory **rose sharply then declined**: centrist support climbed from 2024-Q2 to a peak of 0.648 in 2024-Q4 (slope from inflection to peak: +0.075/quarter), then fell to 0.523 in 2026-Q2. -- The most recent quarter (2026-Q1) shows centrist support at 0.334, **below the post-inflection average** of 0.516, suggesting possible reversion. +- The most recent quarter (2026-Q2) shows centrist support at 0.523, consistent with the post-inflection trend. --- @@ -79,33 +79,34 @@ the new political reality, not as a response to coalition dynamics. | 2018-Q4 | 4 | 1.000 | N/A | N/A | 0 | N/A | 0 | N/A | 4 | 1.000 | 0.938 | | 2019-Q1 | 1 | 0.000 | N/A | N/A | 0 | N/A | 0 | N/A | 1 | 0.000 | 0.833 | | 2019-Q2 | 4 | 0.500 | N/A | N/A | 2 | 0.000 | 0 | N/A | 4 | 0.500 | 0.667 | -| 2019-Q3 | 25 | 0.300 | 0.160 | 0.460 | 17 | 0.176 | 2 | 0.000 | 23 | 0.326 | 0.317 | -| 2019-Q4 | 165 | 0.391 | 0.330 | 0.458 | 86 | 0.181 | 14 | 0.179 | 151 | 0.411 | 0.382 | -| 2020-Q1 | 79 | 0.278 | 0.196 | 0.373 | 45 | 0.100 | 12 | 0.000 | 67 | 0.328 | 0.350 | -| 2020-Q2 | 130 | 0.258 | 0.196 | 0.323 | 87 | 0.086 | 13 | 0.231 | 117 | 0.261 | 0.321 | -| 2020-Q3 | 78 | 0.167 | 0.096 | 0.237 | 57 | 0.088 | 4 | 0.000 | 74 | 0.176 | 0.239 | -| 2020-Q4 | 182 | 0.396 | 0.332 | 0.462 | 98 | 0.204 | 18 | 0.250 | 164 | 0.412 | 0.304 | -| 2021-Q1 | 90 | 0.150 | 0.083 | 0.222 | 65 | 0.015 | 1 | 0.000 | 89 | 0.152 | 0.281 | -| 2021-Q2 | 104 | 0.139 | 0.087 | 0.197 | 84 | 0.065 | 9 | 0.000 | 95 | 0.153 | 0.266 | -| 2021-Q3 | 68 | 0.167 | 0.103 | 0.230 | 54 | 0.127 | 9 | 0.167 | 59 | 0.167 | 0.150 | -| 2021-Q4 | 163 | 0.215 | 0.160 | 0.270 | 119 | 0.155 | 12 | 0.083 | 151 | 0.225 | 0.182 | +| 2019-Q3 | 25 | 0.300 | 0.160 | 0.460 | 17 | 0.176 | 0 | N/A | 25 | 0.300 | 0.317 | +| 2019-Q4 | 165 | 0.391 | 0.333 | 0.455 | 86 | 0.181 | 0 | N/A | 165 | 0.391 | 0.382 | +| 2020-Q1 | 79 | 0.278 | 0.190 | 0.367 | 45 | 0.100 | 0 | N/A | 79 | 0.278 | 0.350 | +| 2020-Q2 | 130 | 0.258 | 0.188 | 0.323 | 87 | 0.086 | 0 | N/A | 130 | 0.258 | 0.321 | +| 2020-Q3 | 78 | 0.167 | 0.102 | 0.237 | 57 | 0.088 | 0 | N/A | 78 | 0.167 | 0.239 | +| 2020-Q4 | 182 | 0.396 | 0.338 | 0.462 | 98 | 0.204 | 0 | N/A | 182 | 0.396 | 0.304 | +| 2021-Q1 | 90 | 0.150 | 0.083 | 0.222 | 65 | 0.015 | 0 | N/A | 90 | 0.150 | 0.281 | +| 2021-Q2 | 104 | 0.139 | 0.091 | 0.197 | 84 | 0.065 | 0 | N/A | 104 | 0.139 | 0.266 | +| 2021-Q3 | 68 | 0.167 | 0.105 | 0.228 | 54 | 0.127 | 0 | N/A | 68 | 0.167 | 0.150 | +| 2021-Q4 | 163 | 0.215 | 0.163 | 0.273 | 119 | 0.155 | 0 | N/A | 163 | 0.215 | 0.182 | | 2022-Q1 | 15 | 0.067 | 0.000 | 0.167 | 13 | 0.038 | 0 | N/A | 15 | 0.067 | 0.193 | -| 2022-Q2 | 119 | 0.214 | 0.151 | 0.282 | 84 | 0.077 | 23 | 0.043 | 96 | 0.255 | 0.207 | -| 2022-Q3 | 83 | 0.133 | 0.072 | 0.199 | 71 | 0.063 | 24 | 0.083 | 59 | 0.153 | 0.173 | -| 2022-Q4 | 229 | 0.227 | 0.186 | 0.273 | 159 | 0.148 | 28 | 0.304 | 201 | 0.216 | 0.205 | -| 2023-Q1 | 77 | 0.148 | 0.091 | 0.213 | 56 | 0.107 | 9 | 0.056 | 68 | 0.160 | 0.191 | -| 2023-Q2 | 90 | 0.306 | 0.233 | 0.389 | 58 | 0.190 | 8 | 0.375 | 82 | 0.299 | 0.230 | -| 2023-Q3 | 68 | 0.184 | 0.110 | 0.257 | 53 | 0.104 | 15 | 0.167 | 53 | 0.189 | 0.219 | -| 2023-Q4 | 130 | 0.321 | 0.262 | 0.381 | 87 | 0.262 | 32 | 0.177 | 98 | 0.367 | 0.284 | -| 2024-Q1 | 98 | 0.501 | 0.423 | 0.576 | 40 | 0.358 | 9 | 0.370 | 89 | 0.514 | 0.349 | -| 2024-Q2 | 124 | 0.573 | 0.505 | 0.637 | 45 | 0.504 | 16 | 0.396 | 108 | 0.599 | 0.460 | -| 2024-Q3 | 17 | 0.588 | 0.431 | 0.765 | 7 | 0.476 | 3 | 0.778 | 14 | 0.548 | 0.544 | -| 2024-Q4 | 230 | 0.648 | 0.603 | 0.695 | 89 | 0.509 | 30 | 0.389 | 200 | 0.686 | 0.620 | -| 2025-Q1 | 29 | 0.598 | 0.437 | 0.747 | 12 | 0.778 | 0 | N/A | 29 | 0.598 | 0.639 | -| 2025-Q2 | 165 | 0.503 | 0.440 | 0.564 | 60 | 0.483 | 28 | 0.357 | 137 | 0.533 | 0.588 | -| 2025-Q3 | 155 | 0.437 | 0.376 | 0.499 | 48 | 0.333 | 46 | 0.319 | 109 | 0.486 | 0.481 | -| 2025-Q4 | 106 | 0.450 | 0.372 | 0.533 | 35 | 0.416 | 12 | 0.395 | 94 | 0.456 | 0.466 | -| 2026-Q1 | 151 | 0.334 | 0.265 | 0.404 | 69 | 0.325 | 27 | 0.333 | 124 | 0.334 | 0.402 | +| 2022-Q2 | 119 | 0.214 | 0.147 | 0.282 | 84 | 0.077 | 0 | N/A | 119 | 0.214 | 0.207 | +| 2022-Q3 | 83 | 0.133 | 0.072 | 0.199 | 71 | 0.063 | 0 | N/A | 83 | 0.133 | 0.173 | +| 2022-Q4 | 229 | 0.227 | 0.183 | 0.273 | 159 | 0.148 | 0 | N/A | 229 | 0.227 | 0.205 | +| 2023-Q1 | 77 | 0.148 | 0.091 | 0.213 | 56 | 0.107 | 0 | N/A | 77 | 0.148 | 0.191 | +| 2023-Q2 | 90 | 0.306 | 0.233 | 0.389 | 58 | 0.190 | 0 | N/A | 90 | 0.306 | 0.230 | +| 2023-Q3 | 68 | 0.184 | 0.110 | 0.257 | 53 | 0.104 | 0 | N/A | 68 | 0.184 | 0.219 | +| 2023-Q4 | 130 | 0.321 | 0.260 | 0.383 | 87 | 0.262 | 0 | N/A | 130 | 0.321 | 0.284 | +| 2024-Q1 | 98 | 0.501 | 0.429 | 0.571 | 40 | 0.358 | 0 | N/A | 98 | 0.501 | 0.349 | +| 2024-Q2 | 124 | 0.573 | 0.505 | 0.640 | 45 | 0.504 | 0 | N/A | 124 | 0.573 | 0.460 | +| 2024-Q3 | 17 | 0.588 | 0.412 | 0.765 | 7 | 0.476 | 0 | N/A | 17 | 0.588 | 0.544 | +| 2024-Q4 | 230 | 0.648 | 0.601 | 0.695 | 89 | 0.509 | 0 | N/A | 230 | 0.648 | 0.620 | +| 2025-Q1 | 29 | 0.598 | 0.448 | 0.747 | 12 | 0.778 | 0 | N/A | 29 | 0.598 | 0.639 | +| 2025-Q2 | 165 | 0.503 | 0.442 | 0.562 | 60 | 0.483 | 0 | N/A | 165 | 0.503 | 0.588 | +| 2025-Q3 | 155 | 0.437 | 0.370 | 0.503 | 48 | 0.333 | 0 | N/A | 155 | 0.437 | 0.481 | +| 2025-Q4 | 106 | 0.450 | 0.373 | 0.532 | 35 | 0.416 | 0 | N/A | 106 | 0.450 | 0.466 | +| 2026-Q1 | 151 | 0.334 | 0.265 | 0.400 | 69 | 0.325 | 0 | N/A | 151 | 0.334 | 0.402 | +| 2026-Q2 | 44 | 0.523 | 0.386 | 0.670 | 0 | N/A | 0 | N/A | 44 | 0.523 | 0.402 | > **Note:** CI intervals use 1000-iteration bootstrap resampling. > Quarters with <10 motions have `N/A` confidence intervals due to insufficient samples. diff --git a/reports/overton_window/temporal_trajectory_figure.png b/reports/overton_window/temporal_trajectory_figure.png index f8d4af0..79b9d2a 100644 Binary files a/reports/overton_window/temporal_trajectory_figure.png and b/reports/overton_window/temporal_trajectory_figure.png differ