sgeboers/motief
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mp-quiz): add MP quiz tab and DB helpers; add design and plan docs

Browse Source
This commit is contained in:
sgeboers
2026-03-24 19:59:25 +01:00
parent b09e580f65
commit eb73275f32
17 changed files with 2879 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/rerun_embeddings.py
+16 -3
View File
@@ -88,7 +88,10 @@ def _clear_embeddings(db_path: str) -> int:
def rerun_embeddings(
db_path: str, model: str = None, retry_missing: bool = False
db_path: str,
model: str = None,
retry_missing: bool = False,
growth_factor: float = 1.5,
) -> dict:
"""Full rerun: clear → embed → fuse → similarity for all windows.
@@ -105,7 +108,9 @@ def rerun_embeddings(
# (stored, skipped_existing, skipped_no_text, errors) or a 5-tuple that
# includes failed_ids as the fifth element. Support both shapes for
# backward-compatibility.
result = text_pipeline.ensure_text_embeddings(db_path=db_path, model=model)
result = text_pipeline.ensure_text_embeddings(
db_path=db_path, model=model, growth_factor=growth_factor
)
if isinstance(result, tuple) and len(result) == 5:
stored, skipped_existing, skipped_no_text, emb_errors, failed_ids = result
elif isinstance(result, tuple) and len(result) == 4:
@@ -207,8 +212,16 @@ def _main():
default=None,
help="Embedding model name (default: text_pipeline default)",
)
parser.add_argument(
"--growth-factor",
type=float,
default=1.5,
help="AIMD growth factor for batch-size tuning (default: 1.5)",
)
args = parser.parse_args()
summary = rerun_embeddings(args.db_path, model=args.model)
summary = rerun_embeddings(
args.db_path, model=args.model, growth_factor=args.growth_factor
)
print(f"cleared_rows: {summary['cleared_rows']}")
print(f"embeddings_stored: {summary['embeddings_stored']}")
print(f"embeddings_skipped_no_text: {summary['embeddings_skipped_no_text']}")
scripts/sync_motion_content.py
+139 -49
View File
@@ -252,43 +252,67 @@ def walk_syncfeed(
# ---------------------------------------------------------------------------
def _extract_motion_text(html: str) -> Optional[str]:
"""Extract clean motion text from Overheid.nl HTML.
Targets <div id="broodtekst"> which contains the actual kamerstuk body.
Falls back to <div id="content"> if broodtekst is absent.
Returns plain text with normalised whitespace, capped at 32 000 chars.
"""
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "lxml")
# Primary target: the kamerstuk body div
node = soup.find("div", id="broodtekst")
if node is None:
# Fallback: main content area
node = soup.find("div", id="content")
if node is None:
# Last resort: whole <article> if present
node = soup.find("article")
if node is None:
# Final fallback: strip all tags from the full body
node = soup.body or soup
text = node.get_text(separator=" ")
# Collapse whitespace
text = re.sub(r"\s+", " ", text).strip()
return text[:32_000] if text else None
def _fetch_body_text(
ext_id: str, session: requests.Session, retries: int = 3
) -> Optional[str]:
"""Fetch plain text body from officielebekendmakingen.nl for ext_id.
"""Fetch plain motion text from officielebekendmakingen.nl for ext_id.
Retries on network errors and on HTTP 5xx or 429 responses using
exponential backoff starting at 0.5s. On permanent failure returns None
and records an audit event via database.db.append_audit_event(...).
Uses BeautifulSoup to extract only the <div id="broodtekst"> element,
avoiding JavaScript, navigation, and cookie-banner noise.
Retries on network errors and HTTP 5xx / 429 with exponential backoff
starting at 0.5 s. Returns None on permanent failure.
"""
import time
import re
from requests import exceptions as req_exceptions
import database
url = BODY_TEXT_BASE.format(ext_id=ext_id)
attempt = 0
backoff = 0.5
last_exc = None
last_exc: Optional[Exception] = None
while attempt < retries:
attempt += 1
try:
resp = session.get(url, timeout=30)
# treat 5xx and 429 as transient
status = getattr(resp, "status_code", None)
if status == 429 or (status is not None and 500 <= status < 600):
last_exc = Exception(f"HTTP {status}")
raise req_exceptions.RequestException(f"HTTP {status}")
resp.raise_for_status()
# Very simple text extraction: strip tags
text = re.sub(r"<[^>]+>", " ", resp.text)
text = re.sub(r"\s+", " ", text).strip()
return text[:32_000] if text else None
text = _extract_motion_text(resp.text)
return text
except req_exceptions.RequestException as exc:
last_exc = exc
# retry for transient errors unless we've exhausted attempts
if attempt < retries:
_logger.info(
"Transient body fetch error for %s (attempt %d/%d): %s; retrying in %.1fs",
@@ -298,51 +322,32 @@ def _fetch_body_text(
exc,
backoff,
)
try:
time.sleep(backoff)
except Exception:
pass
time.sleep(backoff)
backoff *= 2
continue
# exhausted retries => permanent failure
_logger.warning(
"Body text fetch permanently failed for %s: %s", ext_id, exc
)
metadata = {"attempts": attempt, "error": str(exc)}
try:
# MotionDatabase.append_audit_event signature: (actor_id, action, ...)
import database
database.db.append_audit_event(
None,
"body_fetch_failed",
target_type="document",
target_id=ext_id,
metadata=metadata,
metadata={"attempts": attempt, "error": str(exc)},
)
except Exception:
_logger.exception(
"Failed to write audit event for body fetch failure %s", ext_id
)
pass
return None
except Exception as exc: # pragma: no cover - unexpected errors
except Exception as exc: # pragma: no cover
_logger.exception(
"Unexpected error fetching body text for %s: %s", ext_id, exc
)
last_exc = exc
break
# If we fall through here, ensure audit event is recorded
try:
database.db.append_audit_event(
None,
"body_fetch_failed",
target_type="document",
target_id=ext_id,
metadata={"attempts": retries, "error": str(last_exc)},
)
except Exception:
_logger.exception(
"Failed to write audit event for body fetch failure %s", ext_id
)
return None
@@ -572,6 +577,86 @@ def sync_motion_content(db_path: str, skip_body_text: bool = False) -> Dict:
}
# ---------------------------------------------------------------------------
# Body-only re-scrape (uses stored externe_identifier; no SyncFeed walk needed)
# ---------------------------------------------------------------------------
def rescrape_body_texts(
db_path: str,
max_workers: int = MAX_BODY_WORKERS,
batch_size: int = 500,
) -> Dict:
"""Re-fetch and overwrite body_text for every motion that has an externe_identifier.
Reads externe_identifier directly from the DB — no SyncFeed walk needed.
Fetches in parallel (max_workers threads) and commits in batches of batch_size
to limit memory use and provide progress checkpoints.
Returns summary dict with counts.
"""
conn = duckdb.connect(db_path, read_only=True)
rows = conn.execute(
"SELECT id, externe_identifier FROM motions WHERE externe_identifier IS NOT NULL"
).fetchall()
conn.close()
total = len(rows)
_logger.info(
"Re-scraping body_text for %d motions (workers=%d) ...", total, max_workers
)
session = requests.Session()
session.headers["User-Agent"] = "stemwijzer-scraper/1.0"
fetched = 0
failed = 0
committed = 0
# Process in batches so we can commit progress and log along the way
for batch_start in range(0, total, batch_size):
batch = rows[batch_start : batch_start + batch_size]
with ThreadPoolExecutor(max_workers=max_workers) as pool:
future_to_row = {
pool.submit(_fetch_body_text, ext_id, session): (mid, ext_id)
for mid, ext_id in batch
}
updates: List[Tuple[int, Optional[str], Optional[str], Optional[str]]] = []
for future in as_completed(future_to_row):
mid, ext_id = future_to_row[future]
try:
text = future.result()
except Exception as exc:
_logger.warning(
"Future failed for motion %d (%s): %s", mid, ext_id, exc
)
text = None
if text:
fetched += 1
updates.append((mid, None, text, None))
else:
failed += 1
_update_motions(db_path, updates)
committed += len(updates)
done = batch_start + len(batch)
_logger.info(
"Progress: %d/%d done — %d fetched, %d failed, %d committed",
done,
total,
fetched,
failed,
committed,
)
_logger.info(
"Re-scrape complete. fetched=%d, failed=%d, total=%d", fetched, failed, total
)
return {"total": total, "fetched": fetched, "failed": failed}
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
@@ -582,7 +667,6 @@ def _main():
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
# allow overriding MAX_BODY_WORKERS from CLI
parser = argparse.ArgumentParser(description="Sync motion content from SyncFeed")
parser.add_argument("--db-path", required=True, help="Path to motions.db")
parser.add_argument(
@@ -590,22 +674,28 @@ def _main():
action="store_true",
help="Skip fetching body text from officielebekendmakingen.nl",
)
parser.add_argument(
"--body-only",
action="store_true",
help=(
"Skip SyncFeed walk; re-fetch and overwrite body_text for all motions "
"that already have an externe_identifier in the DB."
),
)
parser.add_argument(
"--max-body-workers",
type=int,
default=MAX_BODY_WORKERS,
help=f"Maximum concurrent workers for fetching body text (default: {MAX_BODY_WORKERS})",
)
# Use a local copy for the default to avoid referencing the name after assignment
args = parser.parse_args()
# Set module-level MAX_BODY_WORKERS based on CLI
try:
MAX_BODY_WORKERS = (
int(args.max_body_workers) if args.max_body_workers else MAX_BODY_WORKERS
)
except Exception:
pass
summary = sync_motion_content(args.db_path, skip_body_text=args.skip_body_text)
max_workers = args.max_body_workers or MAX_BODY_WORKERS
if args.body_only:
summary = rescrape_body_texts(args.db_path, max_workers=max_workers)
else:
summary = sync_motion_content(args.db_path, skip_body_text=args.skip_body_text)
for k, v in summary.items():
print(f" {k}: {v}")