content_div = soup.find("div", class_="content")
if not content_div:
# Fallback
content_div = soup.find("div", class_="article-content") or soup.find(id="zoom")
content_html = str(content_div) if content_div else ""
if not content_html:
log(f" WARNING: no content for {url}")
return None, date_str, title
return content_html, date_str, title
def fetch_detail_batch(urls):
results = {}
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:
fut_map = {ex.submit(fetch_detail, url): url for url in urls}
for fut in as_completed(fut_map):
url = fut_map[fut]
try:
results[url] = fut.result()
except Exception:
results[url] = (None, None, None)
return results
def save_to_db(items):
if not items:
return 0
import sqlite3
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
inserted = 0
for url, title, date_str, content_html in items:
try:
c.execute(
"""INSERT OR REPLACE INTO gov_raw
(site_name, page_url, title, publish_date, content, category)
VALUES (?, ?, ?, ?, ?, ?)""",
(SITE_NAME, url, title, date_str, content_html, COLUMN_NAME),
)
inserted += 1
except Exception as e:
log(f" DB error for {url}: {e}")
conn.commit()
conn.close()
return inserted
def rebuild_fts():
try:
import sqlite3
conn = sqlite3.connect(DB_PATH)
conn.execute(
"""INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
SELECT rowid, title, site_name,
CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
FROM gov_raw WHERE category = ?""",
(COLUMN_NAME,),
)
conn.commit()
conn.close()
log("FTS index updated")
except Exception as e:
log(f"FTS note: {e}")
def main():
incremental = len(sys.argv) > 1 and sys.argv[1] == "1"
three_years_ago = (datetime.now() - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
log(f"3-year cutoff: {three_years_ago}")
# Phase 1: collect articles
all_articles = []
total_pages = 16 # From createPageHTML(16, ...)
for page in range(1, total_pages + 1):
items = fetch_list_page(page)
if not items:
log(f"Page {page}: no articles found")
continue
log(f"Page {page}: {len(items)} articles")
filtered = [(u, t, d) for u, t, d in items if d and d >= three_years_ago]
all_articles.extend(filtered)
if incremental:
log("Incremental: page 1 only")
break
if filtered and len(filtered) < len(items):
log(f"Page {page}: some articles before cutoff, stopping")
break
log(f"Total articles to process: {len(all_articles)}")
if not all_articles:
log("No articles within 3 years")
return
# Phase 2: details
results = []
batch_size = MAX_WORKERS * 3
for batch_start in range(0, len(all_articles), batch_size):
batch = all_articles[batch_start:batch_start + batch_size]
batch_urls = [u for u, _, _ in batch]
log(f" Details [{batch_start+1}-{batch_start+len(batch)}/{len(all_articles)}]...")
detail_map = fetch_detail_batch(batch_urls)
for url, title, date_str in batch:
content_html, detail_date, detail_title = detail_map.get(url, (None, None, None))
final_title = detail_title or title
final_date = detail_date or date_str
results.append((url, final_title, final_date, content_html or ""))
# Save
saved = save_to_db(results)
log(f"Saved: {saved}/{len(results)}")
if results:
dates = sorted([r[2] for r in results if r[2]])
log(f"Date range: {dates[0] if dates else 'N/A'} ~ {dates[-1] if dates else 'N/A'}")
rebuild_fts()
log("Done!")
if __name__ == "__main__":
main()