feat: add StemAtlas Streamlit app, explorer, Docker deployment, blog charts
This commit is contained in:
+54
-19
@@ -29,6 +29,7 @@ import argparse
|
||||
import calendar
|
||||
import logging
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date, timedelta
|
||||
from typing import List, Tuple
|
||||
|
||||
@@ -143,27 +144,55 @@ def run(args: argparse.Namespace) -> int:
|
||||
# ── Phase 3: SVD per window ──────────────────────────────────────────────
|
||||
if not args.skip_svd:
|
||||
windows = _generate_windows(start_date, end_date, args.window_size)
|
||||
_logger.info("Phase 3: SVD for %d windows (k=%d)", len(windows), args.svd_k)
|
||||
from pipeline.svd_pipeline import run_svd_for_window
|
||||
_logger.info(
|
||||
"Phase 3: SVD for %d windows (k=%d, parallel)", len(windows), args.svd_k
|
||||
)
|
||||
from pipeline.svd_pipeline import compute_svd_for_window
|
||||
|
||||
for window_id, w_start, w_end in windows:
|
||||
_logger.info(" window %s: %s → %s", window_id, w_start, w_end)
|
||||
if not dry_run:
|
||||
result = run_svd_for_window(
|
||||
db=db,
|
||||
window_id=window_id,
|
||||
start_date=w_start,
|
||||
end_date=w_end,
|
||||
k=args.svd_k,
|
||||
)
|
||||
_logger.info(
|
||||
" k_used=%d stored_mp=%d stored_motion=%d",
|
||||
result["k_used"],
|
||||
result["stored_mp"],
|
||||
result["stored_motion"],
|
||||
)
|
||||
else:
|
||||
if dry_run:
|
||||
for window_id, w_start, w_end in windows:
|
||||
_logger.info(" [dry-run] would run SVD for window %s", window_id)
|
||||
else:
|
||||
# Compute all windows in parallel (numpy/scipy SVD releases the GIL).
|
||||
# IMPORTANT: collect ALL results before writing — DuckDB rejects mixing
|
||||
# read-only and read-write connections in the same process.
|
||||
# The `with` block waits for all threads to finish before we exit it,
|
||||
# ensuring all read-only connections are closed before writes begin.
|
||||
futures = {}
|
||||
max_workers = min(len(windows), (args.svd_workers or 4))
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
for window_id, w_start, w_end in windows:
|
||||
fut = pool.submit(
|
||||
compute_svd_for_window,
|
||||
db.db_path,
|
||||
window_id,
|
||||
w_start,
|
||||
w_end,
|
||||
args.svd_k,
|
||||
)
|
||||
futures[fut] = window_id
|
||||
# All threads are done here — all read-only connections are closed.
|
||||
# Now write results sequentially.
|
||||
for fut, window_id in futures.items():
|
||||
try:
|
||||
result = fut.result()
|
||||
except Exception as exc:
|
||||
_logger.error(" window %s raised: %s", window_id, exc)
|
||||
continue
|
||||
|
||||
if result["k_used"] == 0:
|
||||
_logger.info(" window %s: no data, skipped", window_id)
|
||||
continue
|
||||
|
||||
rows = result["mp_rows"] + result["motion_rows"]
|
||||
db.batch_store_svd_vectors(window_id, rows)
|
||||
_logger.info(
|
||||
" window %s: k_used=%d stored_mp=%d stored_motion=%d",
|
||||
window_id,
|
||||
result["k_used"],
|
||||
len(result["mp_rows"]),
|
||||
len(result["motion_rows"]),
|
||||
)
|
||||
else:
|
||||
_logger.info("Phase 3: skipped (--skip-svd)")
|
||||
|
||||
@@ -235,6 +264,12 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="Time window granularity",
|
||||
)
|
||||
parser.add_argument("--svd-k", type=int, default=50, help="SVD dimensions")
|
||||
parser.add_argument(
|
||||
"--svd-workers",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Parallel workers for SVD (default: min(windows, 4))",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text-model",
|
||||
default=None,
|
||||
|
||||
+99
-41
@@ -150,6 +150,96 @@ def _procrustes_align(
|
||||
return current_anchor
|
||||
|
||||
|
||||
def compute_svd_for_window(
|
||||
db_path: str,
|
||||
window_id: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
k: int = 50,
|
||||
) -> Dict:
|
||||
"""Pure-compute SVD for a window. Safe to run in a subprocess.
|
||||
|
||||
Opens the DB in read-only mode (allows concurrent parallel workers).
|
||||
Does NOT write to the DB — caller is responsible for persisting results.
|
||||
|
||||
Returns dict with keys:
|
||||
window_id, k_used, mp_rows, motion_rows
|
||||
where *_rows are List[Tuple[entity_type, entity_id, vector, model]]
|
||||
"""
|
||||
empty = {"window_id": window_id, "k_used": 0, "mp_rows": [], "motion_rows": []}
|
||||
|
||||
# Read vote matrix using a read-only connection — safe to run in parallel.
|
||||
conn = duckdb.connect(db_path, read_only=True)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT motion_id, mp_name, vote FROM mp_votes WHERE date BETWEEN ? AND ?",
|
||||
(start_date, end_date),
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if not rows:
|
||||
return empty
|
||||
|
||||
motion_ids = sorted({int(r[0]) for r in rows})
|
||||
mp_names = sorted({r[1] for r in rows})
|
||||
|
||||
m_count = len(mp_names)
|
||||
n_count = len(motion_ids)
|
||||
mat = np.zeros((m_count, n_count), dtype=float)
|
||||
|
||||
mp_index = {name: i for i, name in enumerate(mp_names)}
|
||||
motion_index = {mid: j for j, mid in enumerate(motion_ids)}
|
||||
|
||||
for motion_id, mp_name, vote in rows:
|
||||
i = mp_index[mp_name]
|
||||
j = motion_index[int(motion_id)]
|
||||
val = VOTE_MAP.get(
|
||||
vote, VOTE_MAP.get(vote.strip() if isinstance(vote, str) else vote, 0.0)
|
||||
)
|
||||
try:
|
||||
mat[i, j] = float(val)
|
||||
except Exception:
|
||||
mat[i, j] = 0.0
|
||||
|
||||
if mat.size == 0 or mat.shape[0] == 0 or mat.shape[1] == 0:
|
||||
return empty
|
||||
|
||||
k_used = _safe_k(mat, k)
|
||||
if k_used <= 0:
|
||||
return empty
|
||||
|
||||
try:
|
||||
A = csr_matrix(mat)
|
||||
U, s, Vt = svds(A, k=k_used)
|
||||
idx = np.argsort(s)[::-1]
|
||||
s = s[idx]
|
||||
U = U[:, idx]
|
||||
Vt = Vt[idx, :]
|
||||
|
||||
mp_vecs = (U * s.reshape(1, -1)).tolist()
|
||||
motion_vecs = (Vt.T * s.reshape(1, -1)).tolist()
|
||||
|
||||
mp_rows = [
|
||||
("mp", mp_name, mp_vecs[i], None) for i, mp_name in enumerate(mp_names)
|
||||
]
|
||||
motion_rows = [
|
||||
("motion", str(mid), motion_vecs[j], None)
|
||||
for j, mid in enumerate(motion_ids)
|
||||
]
|
||||
|
||||
return {
|
||||
"window_id": window_id,
|
||||
"k_used": k_used,
|
||||
"mp_rows": mp_rows,
|
||||
"motion_rows": motion_rows,
|
||||
}
|
||||
|
||||
except Exception:
|
||||
_logger.exception("SVD failed for window %s", window_id)
|
||||
return empty
|
||||
|
||||
|
||||
def run_svd_for_window(
|
||||
db: MotionDatabase,
|
||||
window_id: str,
|
||||
@@ -161,46 +251,14 @@ def run_svd_for_window(
|
||||
|
||||
Returns metadata dict with keys: k_used, stored_mp, stored_motion
|
||||
"""
|
||||
mat, mp_names, motion_ids = _build_vote_matrix(db, start_date, end_date)
|
||||
|
||||
if mat.size == 0 or mat.shape[0] == 0 or mat.shape[1] == 0:
|
||||
result = compute_svd_for_window(db.db_path, window_id, start_date, end_date, k)
|
||||
if result["k_used"] == 0:
|
||||
return {"k_used": 0, "stored_mp": 0, "stored_motion": 0}
|
||||
|
||||
k_used = _safe_k(mat, k)
|
||||
|
||||
if k_used <= 0:
|
||||
return {"k_used": 0, "stored_mp": 0, "stored_motion": 0}
|
||||
|
||||
# use sparse svds for efficiency
|
||||
try:
|
||||
A = csr_matrix(mat)
|
||||
U, s, Vt = svds(A, k=k_used)
|
||||
# svds does not guarantee ordering of singular values; sort descending
|
||||
idx = np.argsort(s)[::-1]
|
||||
s = s[idx]
|
||||
U = U[:, idx]
|
||||
Vt = Vt[idx, :]
|
||||
|
||||
# weight by singular values
|
||||
mp_vecs = (U * s.reshape(1, -1)).tolist() # m x k
|
||||
motion_vecs = (Vt.T * s.reshape(1, -1)).tolist() # n x k
|
||||
|
||||
stored_mp = 0
|
||||
stored_motion = 0
|
||||
for i, mp_name in enumerate(mp_names):
|
||||
db.store_svd_vector(window_id, "mp", mp_name, mp_vecs[i])
|
||||
stored_mp += 1
|
||||
|
||||
for j, motion_id in enumerate(motion_ids):
|
||||
db.store_svd_vector(window_id, "motion", str(motion_id), motion_vecs[j])
|
||||
stored_motion += 1
|
||||
|
||||
return {
|
||||
"k_used": k_used,
|
||||
"stored_mp": stored_mp,
|
||||
"stored_motion": stored_motion,
|
||||
}
|
||||
|
||||
except Exception:
|
||||
_logger.exception("SVD failed for window")
|
||||
return {"k_used": 0, "stored_mp": 0, "stored_motion": 0}
|
||||
rows = result["mp_rows"] + result["motion_rows"]
|
||||
stored = db.batch_store_svd_vectors(window_id, rows)
|
||||
return {
|
||||
"k_used": result["k_used"],
|
||||
"stored_mp": len(result["mp_rows"]),
|
||||
"stored_motion": len(result["motion_rows"]),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user