chore: confirm deletion of stale files
This commit is contained in:
@@ -1,43 +0,0 @@
|
||||
# Known anti-patterns and recommended remediation (Phase 1 findings)
|
||||
|
||||
anti_patterns:
|
||||
- id: broad_except_swallows_errors
|
||||
description: "Wide except: clauses that swallow exceptions without logging or re-raising."
|
||||
examples:
|
||||
- path: multiple
|
||||
note: "Observed in various pipeline and ingestion spots where except Exception: returns a default without context."
|
||||
remediation:
|
||||
- "Replace broad except with specific exceptions."
|
||||
- "When broad except is absolutely needed, call logger.exception(...) and re-raise or convert to a typed domain error."
|
||||
- "Add unit tests to ensure critical errors are visible in CI logs."
|
||||
|
||||
- id: mixed_print_and_logging
|
||||
description: "Mixing print() and logging() for errors and info messages."
|
||||
examples:
|
||||
- path: api_client.py
|
||||
excerpt: |
|
||||
```python
|
||||
print(f"Fetched {len(voting_records)} voting records from API")
|
||||
...
|
||||
except Exception as e:
|
||||
print(f"Error fetching motions from API: {e}")
|
||||
```
|
||||
remediation:
|
||||
- "Use logging.getLogger(__name__) and logger.info/warning/exception consistently."
|
||||
- "Add a top-level logging configuration for Streamlit and scripts."
|
||||
|
||||
- id: no_lockfile
|
||||
description: "No lockfile present -> unreproducible installs and CI unpredictability."
|
||||
remediation:
|
||||
- "Add a lockfile (poetry.lock, requirements.txt produced by pip-tools) and pin versions in CI."
|
||||
- "Make CI use the lockfile for reproducible builds."
|
||||
|
||||
- id: declared_but_unused_dependency
|
||||
description: "Dependency declared but unused (openai in pyproject)."
|
||||
remediation:
|
||||
- "Either remove the dependency or add clear adapter code/tests that exercise it. Keep pyproject tidy."
|
||||
|
||||
- id: brittle_identity_heuristics
|
||||
description: "Heuristics for MP identity (comma-based parsing) are brittle."
|
||||
remediation:
|
||||
- "Add robust parsing rules and unit tests; prefer canonical identifiers (persoon_id) where available."
|
||||
@@ -1,35 +0,0 @@
|
||||
# Architecture overview and confidence levels
|
||||
|
||||
layers:
|
||||
- name: ui
|
||||
description: "Streamlit pages and app entrypoints (Home.py, pages/*)."
|
||||
confidence: high
|
||||
- name: ingestion
|
||||
description: "API client and scrapers (api_client.py, scraper.py)."
|
||||
confidence: high
|
||||
- name: processing
|
||||
description: "Pipelines for embeddings, SVD, fusion (pipeline/*, similarity/*)."
|
||||
confidence: high
|
||||
- name: storage
|
||||
description: "DuckDB primary store; JSON fallback used in tests when duckdb missing."
|
||||
confidence: high
|
||||
- name: ai_provider
|
||||
description: "Lightweight HTTP wrapper around OpenRouter/OpenAI-style backends in ai_provider.py."
|
||||
confidence: medium
|
||||
- name: orchestration
|
||||
description: "Script-based orchestration (scripts/*.py), rerun_embeddings, scheduler."
|
||||
confidence: medium
|
||||
|
||||
organization:
|
||||
- Keep UI code separated from heavy compute — Streamlit runs should avoid heavy compute inline (use subprocess or schedule).
|
||||
- Pipelines are implemented as re-entrant functions returning summary dicts to facilitate testing and subprocess usage (seen in svd_pipeline.compute_svd_for_window).
|
||||
- DB access is centralised via MotionDatabase helper (database.py) with convenience methods (store_fused_embedding, append_audit_event).
|
||||
|
||||
design_decisions:
|
||||
- Use DuckDB for local fast analytics storage; read_only connections used in compute stages to allow parallel workers.
|
||||
- Embeddings and similarity cache are stored as JSON in DuckDB tables (vector columns).
|
||||
- The ai_provider uses requests with retry/backoff rather than a heavy SDK to keep testing simple.
|
||||
|
||||
confidence_summary:
|
||||
overall_confidence: high
|
||||
notes: "Phase 1 input inspected files across the repo; design mapping is consistent with code samples."
|
||||
@@ -1,32 +0,0 @@
|
||||
# Coding conventions cheat-sheet (extracted from Phase 1)
|
||||
|
||||
naming:
|
||||
module_files: snake_case (e.g., text_pipeline.py, ai_provider.py)
|
||||
functions: snake_case
|
||||
classes: PascalCase
|
||||
constants: UPPER_SNAKE_CASE
|
||||
module_singletons: module-level instances, named lower_snake (e.g., db = MotionDatabase())
|
||||
|
||||
imports:
|
||||
order:
|
||||
- stdlib
|
||||
- third-party
|
||||
- local application imports
|
||||
style:
|
||||
- group imports with a blank line between groups
|
||||
- prefer "from x import y" only when needed to avoid circular imports
|
||||
|
||||
types_and_dataclasses:
|
||||
- Use type hints broadly (functions, public APIs)
|
||||
- config should be a dataclass in config.py
|
||||
- Module-level singletons are allowed (but follow lifecycle rules in db_connection constraints)
|
||||
|
||||
tests:
|
||||
- pytest
|
||||
- tests/ directory, files named test_*.py
|
||||
- Use fixtures in tests/fixtures and conftest.py
|
||||
- Tests expect raises(...) for invalid input or ProviderError
|
||||
|
||||
error_handling:
|
||||
- Prefer explicit exceptions (ValueError, ProviderError)
|
||||
- Avoid overly-broad except: clauses (see anti-patterns)
|
||||
@@ -1,55 +0,0 @@
|
||||
# Dependencies map and recommended extras (Phase 1 authoritative)
|
||||
declared:
|
||||
- streamlit
|
||||
- duckdb
|
||||
- ibis-framework[duckdb]
|
||||
- plotly
|
||||
- scikit-learn
|
||||
- scipy
|
||||
- umap-learn
|
||||
- openai # note: declared but not observed imported; review usage
|
||||
- requests
|
||||
|
||||
observed:
|
||||
- requests
|
||||
- duckdb (used but sometimes import guarded)
|
||||
- numpy
|
||||
- pytest
|
||||
|
||||
grouped:
|
||||
core:
|
||||
- python >=3.13
|
||||
- streamlit
|
||||
- duckdb
|
||||
- ibis-framework[duckdb]
|
||||
- requests
|
||||
ml:
|
||||
- scikit-learn
|
||||
- scipy
|
||||
- umap-learn
|
||||
- numpy
|
||||
viz:
|
||||
- plotly
|
||||
testing:
|
||||
- pytest
|
||||
|
||||
recommended_extras:
|
||||
reproducibility:
|
||||
- poetry (poetry.lock) or pip-tools (requirements.txt + requirements.in)
|
||||
- pipx or virtualenv usage documented
|
||||
linting_and_formatting:
|
||||
- black
|
||||
- ruff
|
||||
- isort
|
||||
- mypy
|
||||
logging_and_monitoring:
|
||||
- structlog (optional)
|
||||
containerization:
|
||||
- docker (already used)
|
||||
heavy_analytics (optional):
|
||||
- pandas
|
||||
- altair
|
||||
- dash (if more interactive dashboards are needed)
|
||||
notes:
|
||||
- Because no lockfile was present during Phase 1, adding one is high priority for reproducible CI builds.
|
||||
- openai is declared but not imported anywhere in Phase 1 files; prefer to either remove or add an explicit adapter usage and tests.
|
||||
@@ -1,37 +0,0 @@
|
||||
# Domain glossary (core concepts from Phase 1)
|
||||
|
||||
terms:
|
||||
Motion:
|
||||
short: "A parliamentary motion/decision"
|
||||
keys: [id, title, description, date, body_text, url]
|
||||
motie:
|
||||
short: "Dutch: motion (motie). Equivalent to Motion in code comments and UI."
|
||||
MP:
|
||||
short: "Member of Parliament (kamerlid)"
|
||||
keys: [mp_name, party, van, tot_en_met, persoon_id]
|
||||
mp_votes:
|
||||
short: "Raw voting rows: motion_id, mp_name, vote, date"
|
||||
mp_metadata:
|
||||
short: "Per-MP metadata table and fields"
|
||||
user_sessions:
|
||||
short: "Streamlit user quiz session state (session_id, user_votes, completed_motions...)"
|
||||
embeddings:
|
||||
short: "Raw text embeddings stored per motion (embeddings table)"
|
||||
svd_vectors:
|
||||
short: "SVD-derived vectors from the vote matrix (svd_vectors table)"
|
||||
fused_embeddings:
|
||||
short: "Concatenation of SVD and text embeddings (fused_embeddings table)"
|
||||
similarity_cache:
|
||||
short: "Precomputed nearest neighbors for each motion"
|
||||
window_id:
|
||||
short: "Processing window identifier used for SVD/fusion runs"
|
||||
controversy_score:
|
||||
short: "Numeric measure stored in motions table"
|
||||
winning_margin:
|
||||
short: "Numeric field indicating margin of win in a vote"
|
||||
Politiek_Kompas:
|
||||
short: "Political compass; also appears in UI features"
|
||||
MP_quiz:
|
||||
short: "Interactive quiz derived from motions and mp_votes"
|
||||
notes:
|
||||
- Use these canonical terms in docs, tests, variable names and DB schemas.
|
||||
@@ -1,33 +0,0 @@
|
||||
# Tech stack (Phase 1 authoritative)
|
||||
|
||||
language:
|
||||
name: python
|
||||
version: ">=3.13"
|
||||
|
||||
frameworks:
|
||||
- streamlit: ">=1.48.0" # UI: Home.py, pages/..., app.py
|
||||
|
||||
database:
|
||||
primary: duckdb
|
||||
orm_or_adapter: ibis-framework[duckdb] # used for some parts
|
||||
|
||||
visualization:
|
||||
- plotly
|
||||
|
||||
ml:
|
||||
- scikit-learn
|
||||
- scipy
|
||||
- umap-learn
|
||||
|
||||
ai:
|
||||
declared_dependency: openai # declared in pyproject but not observed imported; ai_provider uses requests
|
||||
runtime_adapter: custom requests-based wrapper (ai_provider.py)
|
||||
|
||||
container:
|
||||
- docker: Dockerfile FROM python:3.13-slim, EXPOSE 8501, CMD streamlit run Home.py
|
||||
|
||||
testing:
|
||||
- pytest
|
||||
|
||||
ci:
|
||||
- drone: .drone.yml present
|
||||
Reference in New Issue
Block a user