pattern = re.compile(
r']*>.*?\s*([^<]+)',
re.DOTALL,
)
for m in pattern.finditer(html):
url = urljoin(BASE, m.group(1))
title = m.group(2).strip()
date_str = m.group(3).strip()
items.append({"url": url, "title": title, "date": date_str})
return items
def fetch_page(session, url):
try:
r = session.get(url, headers=HEADERS, timeout=30)
r.encoding = "utf-8"
return r.text
except:
return None
def parse_detail(html):
"""Extract title, date, content from detail page."""
title = ""
m = re.search(r"]*>(.+?)", html, re.DOTALL)
if m:
title = re.sub(r"<[^>]+>", "", m.group(1)).strip()
date_str = ""
dm = re.search(
r']*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})', html
)
if not dm:
dm = re.search(r"发布时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})", html)
if dm:
date_str = dm.group(1)
# Extract content from
content = ""
cm = re.search(r']*>(.*?)
\s*', html, re.DOTALL)
if cm:
content = cm.group(1).strip()
else:
# Fallback: try body
bm = re.search(r"]*>(.*?)
", html, re.DOTALL)
if bm:
content = bm.group(1).strip()
# Clean up styles
content = re.sub(r'\s*style="[^"]*"', "", content)
# Clean TRS comments
content = re.sub(r"", "", content)
content = content.strip()
return title, date_str, content
def push_to_db(db, records):
"""Bulk insert records into gov_raw."""
inserted = 0
skipped = 0
for item in records:
try:
db.execute(
"""INSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags)
VALUES (?,?,?,?,?,?,?,?,?,?)""",
(
item["site_name"][:200],
item["source_url"],
item["page_url"],
item["title"],
item["publish_date"],
item["summary"],
item["content"],
item["status"],
item["category"],
item["tags"],
),
)
if db.total_changes > 0:
inserted += 1
else:
skipped += 1
except:
skipped += 1
return inserted, skipped
def run(max_pages=None, incremental=False):
if max_pages is None:
max_pages = MAX_PAGES
db = sqlite3.connect(SEARCH_DB, timeout=60)
db.execute("PRAGMA journal_mode=WAL")
# Get known URLs
known = set(
r[0]
for r in db.execute(
"SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)
).fetchall()
)
print(f" Known in DB: {len(known)}")
session = requests.Session()
session.headers.update(HEADERS)
# First, get page 1 to find total count
print(f" Fetching page 1...", end=" ")
html, total = fetch_list(session, 1)
if not html:
print("FAIL - API returned empty")
return
items = parse_list(html)
print(f"OK - {len(items)} items (total: {total})")
# Calculate how many pages to fetch
total_pages = min(max_pages, (total + 19) // 20 if total else max_pages)
print(f" Total pages: {total_pages}")
# If incremental, check if the newest item is already known
if incremental and known:
newest_url = items[0]["url"] if items else ""
if newest_url in known:
print(f" No new data (newest URL already in DB)")
db.close()
return
print(f" Newest item not in DB, will scan all pages")
# Collect all items across pages
all_items = list(items)
for page in range(2, total_pages + 1):
print(f" Fetching page {page}...", end=" ")
html, _ = fetch_list(session, page)
if not html:
print("FAIL")
break
page_items = parse_list(html)
if not page_items:
print("empty, done")
break
print(f"OK - {len(page_items)} items")
# Incremental: if all items on this page are known, stop
if incremental:
all_known = all(item["url"] in known for item in page_items)
if all_known:
print(f" All items on page {page} already in DB, stopping")
break
# Filter out known items
new_items = [i for i in page_items if i["url"] not in known]
if not new_items:
break
all_items.extend(page_items)
else:
all_items.extend(page_items)
# Filter by date (3 year cutoff)
all_items = [i for i in all_items if i["date"] >= CUTOFF or not i["date"]]
if not all_items:
print(" No items to process")
db.close()
return
print(f" Total items to fetch details: {len(all_items)}")
# Fetch details
results = []
for i, item in enumerate(all_items):
html = fetch_page(session, item["url"])
if not html:
print(f" [{i+1}/{len(all_items)}] SKIP (fetch failed): {item['url']}")
continue
title, date_str, content = parse_detail(html)
results.append(
{
"site_name": SITE_NAME,
"source_url": item["url"][:500],
"page_url": item["url"],
"title": (title or item["title"])[:500],
"publish_date": (date_str or item["date"])[:10],
"summary": (title or item["title"])[:500],
"content": content,
"status": "active",
"category": "",
"tags": "",
}
)
if (i + 1) % 10 == 0:
print(f" [{i+1}/{len(all_items)}]")
# Write to DB
inserted, skipped = push_to_db(db, results)
db.commit()
# Update FTS index
db.execute(
"""INSERT INTO gov_search(rowid,title,site_name,summary)
SELECT r.id, r.title, r.site_name, r.summary
FROM gov_raw r
WHERE r.id NOT IN (SELECT rowid FROM gov_search)
AND r.site_name=?""",
(SITE_NAME,),
)
db.commit()
db.close()
print(f" Done: new={inserted}, skip={skipped}")
if __name__ == "__main__":
incremental = "--incremental" in sys.argv
mp = None
for a in sys.argv[1:]:
if a.isdigit():
mp = int(a)
break
if a == "--test":
mp = 1
break
run(max_pages=mp, incremental=incremental)