feat: complete parliamentary embedding pipeline with full historical coverage
- Add fused (SVD + text) embedding pipeline for annual windows 2016-2026 - Fix store_fused_embedding duplicate bug: DELETE before INSERT (idempotent) - Add --text-batch-size CLI flag to run_pipeline.py (default 200) - Add explicit --start-date/--end-date to download_past_year.py - Backfill mp_votes for all motions (party-level votes, 111k new rows) - Add similarity cache recompute: 212k rows across 9 annual windows - Improve ai_provider retry logic, text_pipeline batching - Improve analysis/political_axis PCA handling and visualizations - Add diagnostic/utility scripts: compare_svd, generate_compass, inspect_axis, etc. - Untrack data/motions.db (3.6GB binary), add to .gitignore with outputs/ - Update continuity ledger with full session state
This commit is contained in:
@@ -53,14 +53,14 @@ def test_pipeline_end_to_end(tmp_path, monkeypatch):
|
||||
|
||||
conn.close()
|
||||
|
||||
# monkeypatch ai_provider.get_embedding to deterministic vector
|
||||
# monkeypatch ai_provider.get_embeddings_batch to deterministic vectors
|
||||
import ai_provider
|
||||
|
||||
def fake_get_embedding(text, model=None):
|
||||
# produce a deterministic vector based on seeded numpy
|
||||
return list(np.random.rand(16))
|
||||
def fake_get_embeddings_batch(texts, model=None, batch_size=50):
|
||||
# produce a deterministic vector per text based on seeded numpy
|
||||
return [list(np.random.rand(16)) for _ in texts]
|
||||
|
||||
monkeypatch.setattr("ai_provider.get_embedding", fake_get_embedding)
|
||||
monkeypatch.setattr("ai_provider.get_embeddings_batch", fake_get_embeddings_batch)
|
||||
|
||||
# run ensure_text_embeddings
|
||||
from pipeline.text_pipeline import ensure_text_embeddings
|
||||
|
||||
@@ -49,11 +49,11 @@ def test_ensure_text_embeddings_monkeypatch(tmp_path, monkeypatch):
|
||||
|
||||
conn.close()
|
||||
|
||||
# monkeypatch ai_provider.get_embedding
|
||||
def fake_get_embedding(text, model=None):
|
||||
return [0.1] * 16
|
||||
# monkeypatch ai_provider.get_embeddings_batch (used by batched pipeline)
|
||||
def fake_get_embeddings_batch(texts, model=None, batch_size=50):
|
||||
return [[0.1] * 16 for _ in texts]
|
||||
|
||||
monkeypatch.setattr("ai_provider.get_embedding", fake_get_embedding)
|
||||
monkeypatch.setattr("ai_provider.get_embeddings_batch", fake_get_embeddings_batch)
|
||||
|
||||
# run ensure_text_embeddings
|
||||
from pipeline.text_pipeline import ensure_text_embeddings
|
||||
|
||||
Reference in New Issue
Block a user