(.*?)
0:
summary = re.sub(r"<[^>]+>", "", content)
summary = html_lib.unescape(summary)
summary = re.sub(r"\s+", " ", summary).strip()[:200]
conn.execute(
"INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
(cur.lastrowid, title, SITE_NAME, summary),
)
return True
return False
except sqlite3.OperationalError as e:
if "locked" in str(e):
time.sleep(3)
return store_record(conn, url, title, content, date, has_table)
raise
def main():
import requests
s = requests.Session()
s.headers.update(HEADERS)
conn = sqlite3.connect(DB_PATH, timeout=60)
conn.execute("PRAGMA busy_timeout=60000")
conn.execute("PRAGMA journal_mode=WAL")
new_count = 0
skip_count = 0
seen_urls = set()
# 首页
list_url = LIST_URL
r = fetch(list_url, s)
if not r:
print("新增: 0")
return
html_text = r.text
for page in range(1, _PAGES + 1):
if page > 1:
# 第2页起: index_{page-1}.html (paging.js 0-based)
page_url = f"{LIST_URL}index_{page-1}.html"
r = fetch(page_url, s)
if not r:
continue
html_text = r.text
else:
page_url = list_url
items = parse_list(html_text, page_url)
if not items:
continue
for url, title, date in items:
if url in seen_urls:
continue
seen_urls.add(url)
try:
dr = fetch(url, s)
except Exception as e:
print(f" [ERR] {url}: {e}")
skip_count += 1
continue
if not dr:
skip_count += 1
continue
d_title, d_date, d_html = parse_detail(dr.text, url)
if not d_title:
d_title = title
if not d_date:
d_date = date
content, has_table, attachments = html_to_text(d_html, url)
if not content:
skip_count += 1
continue
if store_record(conn, url, d_title, content, d_date, has_table):
new_count += 1
else:
skip_count += 1
conn.commit()
conn.close()
print(f"新增: {new_count}")
if __name__ == "__main__":
main()