feat: motion content enrichment pipeline hardening

- ai_provider_wrapper: retry/fallback with exponential backoff, None sentinel for failed items
- text_pipeline: use wrapper, return 5-tuple (stored, skipped_existing, skipped_no_text, errors, failed_ids)
- similarity/compute: filter trivial 1.0 matches on identical short titles (<12 chars)
- rerun_embeddings: --retry-missing mode, calls ensure_text_embeddings_for_ids on failed ids
- sync_motion_content: per-ext_id retries, HTTPAdapter pool, --max-body-workers CLI flag, audit on failure
- qa_similarity script: samples motions, writes JSON ledger to thoughts/ledgers/
- All tests green: 61 passed, 2 skipped

tests/test_qa_similarity.py

@@ -0,0 +1,51 @@
+import json
+from pathlib import Path
+
+
+def test_qa_similarity_creates_ledger(tmp_path, monkeypatch):
+    # Prepare monkeypatched database.db
+    class DummyDB:
+        def sample_motions(self, sample_size):
+            assert sample_size == 2
+            return [1, 2]
+
+        def get_cached_similarities(self, motion_id, top_k):
+            # return deterministic neighbors
+            return [
+                {"id": motion_id * 10 + i, "score": 1.0 - i * 0.1} for i in range(top_k)
+            ]
+
+    dummy = DummyDB()
+
+    # Monkeypatch the database module to provide .db — use monkeypatch.setitem
+    # so the override is active for this test and auto-reverts after.
+    import types
+
+    fake_db_module = types.SimpleNamespace(db=dummy)
+
+    import sys
+
+    monkeypatch.setitem(sys.modules, "database", fake_db_module)
+
+    # Ensure thoughts/ledgers inside tmp_path
+    base = tmp_path
+    (base / "thoughts" / "ledgers").mkdir(parents=True)
+
+    # Monkeypatch cwd so ledger writes to tmp_path/thoughts
+    monkeypatch.chdir(base)
+
+    from scripts.qa_similarity import main
+
+    summary = main(db_path=":memory:", sample_size=2, top_k=3)
+
+    assert summary["sample_size"] == 2
+    assert summary["top_k"] == 3
+    assert 1 in summary["motions"]
+    assert 2 in summary["motions"]
+
+    ledger_path = Path(summary["ledger_path"])
+    assert ledger_path.exists()
+
+    data = json.loads(ledger_path.read_text(encoding="utf-8"))
+    assert "motions" in data
+    assert len(data["motions"]) == 2

tests/test_rerun_embeddings.py

@@ -0,0 +1,84 @@
+"""Tests for scripts/rerun_embeddings.py.
+
+Monkeypatches pipeline functions directly on their bound module references
+inside rerun_embeddings. Import at module level so the real 'database' module
+is in sys.modules before any test-local sys.modules.setdefault calls run.
+"""
+
+from unittest.mock import MagicMock
+
+import scripts.rerun_embeddings as rer
+
+
+def test_rerun_embeddings_calls_pipeline_steps(monkeypatch, tmp_path):
+    db_file = str(tmp_path / "motions.db")
+    fake_windows = ["2022-Q3", "2023-Q1", "2024-Q2"]
+
+    called = {"ensure": False, "fuse_windows": [], "sim_windows": []}
+
+    # Patch duckdb.connect used in _clear_embeddings and _get_all_windows
+    fake_conn = MagicMock()
+    fake_conn.execute.return_value.rowcount = 0
+    fake_conn.execute.return_value.fetchall.return_value = [(w,) for w in fake_windows]
+    fake_duckdb = MagicMock()
+    fake_duckdb.connect.return_value = fake_conn
+    monkeypatch.setattr(rer, "duckdb", fake_duckdb)
+
+    # ensure_text_embeddings now returns a 5-tuple:
+    # (stored, skipped_existing, skipped_no_text, errors, failed_ids)
+    def fake_ensure(db_path=None, model=None, batch_size=50, **kwargs):
+        called["ensure"] = True
+        return (5, 0, 2, 0, [])
+
+    def fake_fuse(window_id, db_path=None):
+        called["fuse_windows"].append(window_id)
+        return {
+            "inserted": 1,
+            "skipped_missing_text": 0,
+            "skipped_missing_svd": 0,
+            "errors": 0,
+        }
+
+    def fake_sim(vector_type="fused", window_id=None, db_path=None, top_k=10, **kwargs):
+        called["sim_windows"].append(window_id)
+        return 10
+
+    monkeypatch.setattr(rer.text_pipeline, "ensure_text_embeddings", fake_ensure)
+    monkeypatch.setattr(rer.fusion_pipeline, "fuse_for_window", fake_fuse)
+    monkeypatch.setattr(rer.similarity_compute, "compute_similarities", fake_sim)
+
+    summary = rer.rerun_embeddings(db_file)
+
+    assert called["ensure"] is True
+    assert called["fuse_windows"] == fake_windows
+    assert called["sim_windows"] == fake_windows
+    assert summary["windows_processed"] == len(fake_windows)
+    assert summary["embeddings_stored"] == 5
+    assert summary["embeddings_skipped_no_text"] == 2
+    assert summary["embeddings_failed_ids"] == []
+
+
+def test_rerun_retries_when_retry_missing_and_failed_ids(monkeypatch, tmp_path):
+    """When retry_missing=True and first pass returns failed_ids, retry is triggered."""
+    db_file = str(tmp_path / "motions.db")
+
+    monkeypatch.setattr(rer, "_clear_embeddings", lambda db_path: 0)
+    monkeypatch.setattr(rer, "_get_all_windows", lambda db_path: [])
+
+    retry_called = {"ids": None}
+
+    def fake_ensure(db_path=None, model=None, batch_size=50, **kwargs):
+        return (3, 0, 0, 2, [201, 202])
+
+    def fake_retry(db_path=None, ids=None, model=None, batch_size=10, **kwargs):
+        retry_called["ids"] = ids
+        return (2, 0, 0, 0, [])
+
+    monkeypatch.setattr(rer.text_pipeline, "ensure_text_embeddings", fake_ensure)
+    monkeypatch.setattr(rer.text_pipeline, "ensure_text_embeddings_for_ids", fake_retry)
+
+    summary = rer.rerun_embeddings(db_file, retry_missing=True)
+
+    assert retry_called["ids"] is not None, "retry was not called"
+    assert set(retry_called["ids"]) == {201, 202}
+    assert summary["embeddings_failed_ids"] == [201, 202]

tests/test_sync_motion_content.py

@@ -0,0 +1,97 @@
+"""Tests for scripts/sync_motion_content.py.
+
+Tests retry logic in _fetch_body_text and permanent-failure audit recording.
+"""
+
+import requests
+from unittest.mock import MagicMock, patch
+
+import scripts.sync_motion_content as s
+
+
+def test_parse_besluit_simple():
+    xml = '<Besluit id="b1' + '">\n<Zaak ref="z1"/>\n</Besluit>'
+    parsed = s.parse_besluit(xml)
+    assert parsed["id"] == "b1"
+
+
+def test_fetch_body_text_retries_on_transient_error(monkeypatch):
+    """_fetch_body_text retries after a ConnectionError and returns text on success."""
+    session = MagicMock()
+    call_count = {"n": 0}
+
+    def fake_get(url, timeout=30):
+        call_count["n"] += 1
+        if call_count["n"] == 1:
+            raise requests.exceptions.ConnectionError("timeout")
+        # Second attempt succeeds
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.text = "<p>body text here</p>"
+        resp.raise_for_status.return_value = None
+        return resp
+
+    session.get.side_effect = fake_get
+
+    # Patch time.sleep to avoid delays in tests
+    monkeypatch.setattr("time.sleep", lambda s: None)
+
+    result = s._fetch_body_text("ext123", session, retries=3)
+
+    assert result is not None
+    assert "body text here" in result
+    assert call_count["n"] == 2  # failed once, succeeded on second
+
+
+def test_fetch_body_text_permanent_failure_records_audit(monkeypatch):
+    """When all retries are exhausted, audit_event is recorded via database.db."""
+    session = MagicMock()
+    session.get.side_effect = requests.exceptions.ConnectionError("always fails")
+
+    monkeypatch.setattr("time.sleep", lambda s: None)
+
+    # Capture the audit event call
+    audit_calls = []
+
+    import database
+
+    monkeypatch.setattr(
+        database.db,
+        "append_audit_event",
+        lambda actor_id, action, **kwargs: (
+            audit_calls.append({"action": action, **kwargs}) or True
+        ),
+    )
+
+    result = s._fetch_body_text("ext_fail", session, retries=3)
+
+    assert result is None
+    assert len(audit_calls) >= 1
+    assert audit_calls[0]["action"] == "body_fetch_failed"
+    assert audit_calls[0]["target_id"] == "ext_fail"
+
+
+def test_fetch_body_text_retries_on_5xx(monkeypatch):
+    """5xx responses are treated as transient; retried before giving up."""
+    session = MagicMock()
+    call_count = {"n": 0}
+
+    def fake_get(url, timeout=30):
+        call_count["n"] += 1
+        resp = MagicMock()
+        if call_count["n"] < 3:
+            resp.status_code = 503
+            resp.raise_for_status.return_value = None
+        else:
+            resp.status_code = 200
+            resp.text = "clean text"
+            resp.raise_for_status.return_value = None
+        return resp
+
+    session.get.side_effect = fake_get
+    monkeypatch.setattr("time.sleep", lambda s: None)
+
+    result = s._fetch_body_text("ext_5xx", session, retries=3)
+
+    assert result is not None
+    assert call_count["n"] == 3

tests/test_text_pipeline.py

@@ -58,8 +58,8 @@ def test_ensure_text_embeddings_monkeypatch(tmp_path, monkeypatch):
     # run ensure_text_embeddings
     from pipeline.text_pipeline import ensure_text_embeddings
 
-    stored, skipped_existing, skipped_no_text, errors = ensure_text_embeddings(
-        db_path=db_path, model="test-model"
+    stored, skipped_existing, skipped_no_text, errors, failed_ids = (
+        ensure_text_embeddings(db_path=db_path, model="test-model")
     )
 
     assert stored == 2

tests/test_text_pipeline_retry.py

@@ -0,0 +1,122 @@
+"""Tests for pipeline/text_pipeline.py retry behaviour.
+
+Uses monkeypatching to stub get_embeddings_with_retry and store_embedding
+so no real DB or network is needed.
+"""
+
+import pipeline.text_pipeline as tp
+import pipeline.ai_provider_wrapper as ai_wrapper
+
+
+def _make_fake_db(store_results=None):
+    """Return a minimal fake db object for text_pipeline tests."""
+    store_results = store_results or {}
+    call_log = {"stored": []}
+
+    class FakeDB:
+        db_path = ":memory:"
+
+        def store_embedding(self, motion_id, model, vec):
+            call_log["stored"].append(motion_id)
+            return store_results.get(motion_id, 1)
+
+    return FakeDB(), call_log
+
+
+def _stub_select_text(monkeypatch, rows):
+    """Patch _select_text to return predetermined (motion_id, text) rows."""
+    monkeypatch.setattr(tp, "_select_text", lambda db, model: rows)
+
+
+def _stub_counts(monkeypatch, total=10, existing=0):
+    """Patch the duckdb connection used for count queries."""
+    import types
+    from unittest.mock import MagicMock
+
+    fake_conn = MagicMock()
+    # fetchone()[0] is used twice: total_motions and existing count
+    fake_conn.execute.return_value.fetchone.side_effect = [(total,), (existing,)]
+    fake_duckdb = MagicMock()
+    fake_duckdb.connect.return_value = fake_conn
+    monkeypatch.setattr(tp, "duckdb", fake_duckdb)
+
+
+def test_all_embeddings_stored(monkeypatch):
+    """When wrapper returns an embedding for every text, stored count matches."""
+    rows = [(1, "tekst een"), (2, "tekst twee"), (3, "tekst drie")]
+    _stub_select_text(monkeypatch, rows)
+    _stub_counts(monkeypatch, total=3, existing=0)
+
+    fake_db, call_log = _make_fake_db()
+
+    def fake_wrapper(texts, motion_ids=None, model=None, batch_size=50, **kwargs):
+        return [[0.1, 0.2, 0.3] for _ in texts]
+
+    monkeypatch.setattr(ai_wrapper, "get_embeddings_with_retry", fake_wrapper)
+
+    stored, skipped_existing, skipped_no_text, errors, failed_ids = (
+        tp.ensure_text_embeddings(db=fake_db, model="test-model")
+    )
+
+    assert stored == 3
+    assert errors == 0
+    assert failed_ids == []
+    assert skipped_no_text == 0
+    assert set(call_log["stored"]) == {1, 2, 3}
+
+
+def test_partial_failure_populates_failed_ids(monkeypatch):
+    """When wrapper returns None for some items, those ids appear in failed_ids."""
+    rows = [(10, "text a"), (11, "text b"), (12, "text c")]
+    _stub_select_text(monkeypatch, rows)
+    _stub_counts(monkeypatch, total=3, existing=0)
+
+    fake_db, call_log = _make_fake_db()
+
+    def fake_wrapper(texts, motion_ids=None, model=None, batch_size=50, **kwargs):
+        # Return embedding for first, None for second, embedding for third
+        return (
+            [[0.1] for _ in range(len(texts))]
+            if len(texts) != 3
+            else [
+                [0.1, 0.2],
+                None,  # motion_id=11 fails
+                [0.3, 0.4],
+            ]
+        )
+
+    monkeypatch.setattr(ai_wrapper, "get_embeddings_with_retry", fake_wrapper)
+
+    stored, skipped_existing, skipped_no_text, errors, failed_ids = (
+        tp.ensure_text_embeddings(db=fake_db, model="test-model")
+    )
+
+    assert stored == 2
+    assert errors == 1
+    assert 11 in failed_ids
+    assert 10 not in failed_ids
+    assert 12 not in failed_ids
+
+
+def test_no_text_motions_skipped(monkeypatch):
+    """Motions with empty text are counted as skipped_no_text, not sent to wrapper."""
+    rows = [(20, "has text"), (21, ""), (22, None)]
+    _stub_select_text(monkeypatch, rows)
+    _stub_counts(monkeypatch, total=3, existing=0)
+
+    fake_db, call_log = _make_fake_db()
+    wrapper_calls = {"count": 0}
+
+    def fake_wrapper(texts, motion_ids=None, model=None, batch_size=50, **kwargs):
+        wrapper_calls["count"] += len(texts)
+        return [[0.1] for _ in texts]
+
+    monkeypatch.setattr(ai_wrapper, "get_embeddings_with_retry", fake_wrapper)
+
+    stored, _, skipped_no_text, errors, failed_ids = tp.ensure_text_embeddings(
+        db=fake_db, model="test-model"
+    )
+
+    assert skipped_no_text == 2  # motions 21 and 22 have no text
+    assert stored == 1  # only motion 20 was stored
+    assert wrapper_calls["count"] == 1  # wrapper only received 1 text