sgeboers/motief
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix update_existing_motions: single write connection and module-level duckdb import

Browse Source 
Use one DuckDB write connection for the entire update loop instead of
opening/closing per row, wrapped in try/finally for proper cleanup.
Move 'import duckdb' to module level with other imports.
This commit is contained in:
sgeboers
2026-03-29 23:28:40 +02:00
parent be8887f6f8
commit 88110b0aaa
1 changed files with 50 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/download_past_year.py
+50 -44
View File
@@ -18,6 +18,8 @@ from datetime import datetime, timedelta
from typing import Optional, Tuple from typing import Optional, Tuple
from urllib.parse import urlparse from urllib.parse import urlparse
import duckdb
sys.path.insert(0, ".") # run from project root sys.path.insert(0, ".") # run from project root
from api_client import TweedeKamerAPI from api_client import TweedeKamerAPI
@@ -71,8 +73,6 @@ def update_existing_motions(
Returns: Returns:
(updated_count, skipped_count) tuple. (updated_count, skipped_count) tuple.
""" """
import duckdb
# Read motions with missing body_text # Read motions with missing body_text
conn_read = duckdb.connect(db_path, read_only=True) conn_read = duckdb.connect(db_path, read_only=True)
rows = conn_read.execute( rows = conn_read.execute(
@@ -89,57 +89,63 @@ def update_existing_motions(
updated = 0 updated = 0
skipped = 0 skipped = 0
for row in rows: conn_write = duckdb.connect(db_path, read_only=False)
motion_id, url, title, description = row try:
for row in rows:
motion_id, url, title, description = row
besluit_id = extract_besluit_id(url or "") besluit_id = extract_besluit_id(url or "")
if not besluit_id: if not besluit_id:
print(f" Skipping motion {motion_id}: cannot extract besluit_id from URL") print(
skipped += 1 f" Skipping motion {motion_id}: cannot extract besluit_id from URL"
continue )
skipped += 1
continue
print(f" Fetching details for motion {motion_id} (besluit_id={besluit_id})...") print(
details = api._get_motion_details(besluit_id) f" Fetching details for motion {motion_id} (besluit_id={besluit_id})..."
)
details = api._get_motion_details(besluit_id)
if not details or not details.get("body_text"): if not details or not details.get("body_text"):
print(f" Skipping motion {motion_id}: no body_text returned") print(f" Skipping motion {motion_id}: no body_text returned")
skipped += 1 skipped += 1
continue continue
# Build update: always set body_text; also update title/description if # Build update: always set body_text; also update title/description if
# they were placeholder values (e.g. "Motion abc12345" or "No description available") # they were placeholder values (e.g. "Motion abc12345" or "No description available")
new_body = details["body_text"] new_body = details["body_text"]
new_title = title new_title = title
new_desc = description new_desc = description
if title and (title.startswith("Motion ") or title.startswith("Besluit ")): if title and (title.startswith("Motion ") or title.startswith("Besluit ")):
new_title = details.get("title") or title new_title = details.get("title") or title
if description in ( if description in (
None, None,
"", "",
"No description available", "No description available",
"Geen beschrijving beschikbaar", "Geen beschrijving beschikbaar",
): ):
new_desc = details.get("description") or description new_desc = details.get("description") or description
conn_write = duckdb.connect(db_path, read_only=False) conn_write.execute(
conn_write.execute( """
""" UPDATE motions
UPDATE motions SET body_text = ?, title = ?, description = ?
SET body_text = ?, title = ?, description = ? WHERE id = ?
WHERE id = ? """,
""", (new_body, new_title, new_desc, motion_id),
(new_body, new_title, new_desc, motion_id), )
)
updated += 1
print(f" Updated motion {motion_id}")
if delay > 0 and updated + skipped < len(rows):
time.sleep(delay)
finally:
conn_write.close() conn_write.close()
updated += 1
print(f" Updated motion {motion_id}")
if delay > 0 and updated + skipped < len(rows):
time.sleep(delay)
return updated, skipped return updated, skipped