feat(pipeline): implement parliamentary embedding pipeline MVP

- Add 4 migration files: mp_votes, mp_metadata, svd_vectors, fused_embeddings
- Extend database.py with 5 new helper methods and table init
- Add pipeline/ package: extract_mp_votes, fetch_mp_metadata, text_pipeline,
  svd_pipeline (with Procrustes alignment), fusion
- Add full test suite (17 tests) covering all pipeline modules and migrations
- Fix Procrustes alignment bug: scipy scale is a norm value, not a multiplier
- Fix DuckDB date type handling in test assertions (datetime.date vs string)
- Remove duckdb.py shim; tests now run against real duckdb + scipy via uv

Ref: thoughts/shared/plans/2026-03-21-parliamentary-embedding-pipeline-plan.md
This commit is contained in:
2026-03-21 22:31:22 +01:00
parent c498c3467e
commit a36e6cba4e
68 changed files with 6822 additions and 0 deletions
+35
View File
@@ -0,0 +1,35 @@
"""Motion-related simple types and JSON helpers.
Decision: MotionId is an alias for str for simplicity.
"""
from dataclasses import dataclass, asdict
from typing import List
import json
MotionId = str
Embedding = List[float]
@dataclass
class SimilarityNeighbor:
motion_id: MotionId
score: float
def to_json(neighbors: List[SimilarityNeighbor]) -> str:
"""Serialize a list of SimilarityNeighbor to a JSON string.
The format is a JSON list of objects with keys 'motion_id' and 'score'.
"""
list_of_dicts = [asdict(n) for n in neighbors]
return json.dumps(list_of_dicts)
def from_json(json_str: str) -> List[SimilarityNeighbor]:
"""Deserialize a JSON string (list of dicts) into SimilarityNeighbor list."""
parsed = json.loads(json_str)
return [
SimilarityNeighbor(motion_id=item["motion_id"], score=float(item["score"]))
for item in parsed
]