feat(pipeline): implement parliamentary embedding pipeline MVP
- Add 4 migration files: mp_votes, mp_metadata, svd_vectors, fused_embeddings - Extend database.py with 5 new helper methods and table init - Add pipeline/ package: extract_mp_votes, fetch_mp_metadata, text_pipeline, svd_pipeline (with Procrustes alignment), fusion - Add full test suite (17 tests) covering all pipeline modules and migrations - Fix Procrustes alignment bug: scipy scale is a norm value, not a multiplier - Fix DuckDB date type handling in test assertions (datetime.date vs string) - Remove duckdb.py shim; tests now run against real duckdb + scipy via uv Ref: thoughts/shared/plans/2026-03-21-parliamentary-embedding-pipeline-plan.md
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
Test helper fixtures for database migrations.
|
||||
|
||||
Provides a pytest fixture `test_db` that inspects the environment variable
|
||||
`TEST_DB_URL` to decide what to yield:
|
||||
|
||||
- If `TEST_DB_URL` is not set, the fixture yields None. This allows tests to
|
||||
be skipped or operate in a no-database mode in CI or local runs where a
|
||||
test database is not available.
|
||||
- If `TEST_DB_URL` is set and starts with "sqlite", an sqlite3 connection is
|
||||
created via `sqlite3.connect` and yielded. The connection is closed after
|
||||
the test completes.
|
||||
|
||||
Decision: keep this fixture lightweight and focused on sqlite for local
|
||||
smoke-testing. If other database backends are needed later, expand this
|
||||
fixture accordingly.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_db():
|
||||
"""Yield a test database connection or None.
|
||||
|
||||
Behavior:
|
||||
- If TEST_DB_URL is not set in the environment, yield None.
|
||||
- If TEST_DB_URL is set and begins with 'sqlite', open an sqlite3
|
||||
connection and yield it. The connection will be closed when the test
|
||||
finishes.
|
||||
"""
|
||||
url = os.environ.get("TEST_DB_URL")
|
||||
if not url:
|
||||
yield None
|
||||
return
|
||||
|
||||
# Only support sqlite URLs in this lightweight fixture.
|
||||
if url.startswith("sqlite"):
|
||||
# For sqlite URLs, accept either a bare file path or a file:// style
|
||||
# URL. sqlite3.connect handles file paths; if a file:// prefix is
|
||||
# present, strip it.
|
||||
path = url
|
||||
if path.startswith("sqlite:///"):
|
||||
# sqlite:///path => /path
|
||||
path = path[len("sqlite:///") :]
|
||||
elif path.startswith("sqlite://"):
|
||||
path = path[len("sqlite://") :]
|
||||
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
# Best-effort close; tests shouldn't fail on close errors.
|
||||
pass
|
||||
return
|
||||
|
||||
# Unknown or unsupported TEST_DB_URL scheme — yield None to keep tests
|
||||
# tolerant in environments where the fixture can't create a connection.
|
||||
yield None
|
||||
Reference in New Issue
Block a user