content_html = ""
div = soup.find("div", class_=lambda c: c and "main" in c and "ovh" in c)
if not div:
div = soup.find("div", class_="main")
if div:
content_html = str(div)
# Date:
date_str = ""
meta = soup.find("meta", attrs={"name": "PubDate"})
if meta and meta.get("content"):
m = re.match(r"(\d{4}-\d{2}-\d{2})", meta["content"].strip())
if m:
date_str = m.group(1)
if not content_html:
log(f" WARNING: no content for {url}")
return None, date_str
return content_html, date_str
def fetch_detail_batch(urls):
"""Fetch multiple detail pages in parallel"""
results = {}
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:
fut_map = {ex.submit(fetch_detail, url): url for url in urls}
for fut in as_completed(fut_map):
url = fut_map[fut]
try:
results[url] = fut.result()
except Exception as e:
log(f" ERROR in thread for {url}: {e}")
results[url] = (None, None)
return results
def save_to_db(items):
import sqlite3
if not items:
return 0
conn = sqlite3.connect(DB_PATH, timeout=60)
c = conn.cursor()
inserted = 0
for url, title, date_str, content_html in items:
try:
c.execute(
"""INSERT OR REPLACE INTO gov_raw (site_name, page_url, title, publish_date, content, category, script_name) VALUES (?, ?, ?, ?, ?, ?, 'crawl_guanxian.py')""",
(SITE_NAME, url, title, date_str, content_html, SOURCE),
)
inserted += 1
except Exception as e:
log(f" DB error for {url}: {e}")
conn.commit()
conn.close()
return inserted
def main():
incremental = len(sys.argv) > 1 and sys.argv[1] == "1"
three_years_ago = (datetime.now() - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
log(f"3-year cutoff: {three_years_ago}")
# Phase 1: Scan list pages
all_articles = []
for page in range(1, MAX_PAGES + 1):
articles, has_data = fetch_list_page(page)
if not has_data:
log(f"Page {page}: no data, stopping")
break
# Filter 3-year
filtered = [(u, t, d) for u, t, d in articles if d and d >= three_years_ago]
all_articles.extend(filtered)
if incremental:
log(f"Incremental: page 1, {len(filtered)} articles")
break
# Stop when all past cutoff
if articles and not filtered:
oldest_on_page = max([d for _, _, d in articles if d] or [""])
if oldest_on_page and oldest_on_page < three_years_ago:
log(f"Page {page}: all before cutoff, stopping")
break
if page % 50 == 0:
log(f" Page {page}: {len(all_articles)} articles so far")
log(f"Total articles: {len(all_articles)}")
# Phase 2: Fetch details (parallel batches)
results = []
batch_size = MAX_WORKERS * 3 # 60 per batch for smooth progress
for batch_start in range(0, len(all_articles), batch_size):
batch = all_articles[batch_start:batch_start + batch_size]
batch_urls = [u for u, _, _ in batch]
log(f" Fetching details [{batch_start+1}-{batch_start+len(batch)}/{len(all_articles)}]...")
detail_map = fetch_detail_batch(batch_urls)
for url, title, date_str in batch:
content_html, detail_date = detail_map.get(url, (None, None))
final_date = detail_date or date_str
results.append((url, title, final_date, content_html or ""))
# Phase 3: Save
saved = save_to_db(results)
# Summary
log(f"\n{'='*50}")
log(f"Done. Total: {len(results)} articles (saved: {saved})")
if results:
dates = sorted([r[2] for r in results if r[2]])
log(f"Date range: {dates[0]} ~ {dates[-1]}")
# Rebuild FTS
try:
import sqlite3
conn = sqlite3.connect(DB_PATH, timeout=60)
conn.execute("""
INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
SELECT rowid, title, site_name,
CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
FROM gov_raw WHERE category = ?
""", (SOURCE,))
conn.commit()
conn.close()
log("FTS index rebuilt")
except Exception as e:
log(f"FTS rebuild note: {e}")
if __name__ == "__main__":
main()