', content_html) if te: content_html = content_html[te.start():] return title, date, content_html return title, date, "" def store_record(conn, url, title, content, date, has_table): cur = conn.execute( "SELECT id FROM gov_raw WHERE page_url=?", (url,), ) if cur.fetchone(): return False try: cur = conn.execute( "INSERT OR IGNORE INTO gov_raw " "(page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) " "VALUES (?,?,?,?,?,?,?,?,?,?)", (url, url, title, content, date, SITE_NAME, GROUP_NAME, SCRIPT_NAME, has_table, "other"), ) if cur.rowcount > 0: # sync FTS summary = re.sub(r"<[^>]+>", "", content) summary = html_lib.unescape(summary) summary = re.sub(r"\s+", " ", summary).strip()[:200] conn.execute( "INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)", (cur.lastrowid, title, SITE_NAME, summary), ) return True return False except sqlite3.OperationalError as e: if "locked" in str(e): time.sleep(3) return store_record(conn, url, title, content, date, has_table) raise def main(): import requests s = requests.Session() s.headers.update(HEADERS) conn = sqlite3.connect(DB_PATH, timeout=60) conn.execute("PRAGMA busy_timeout=60000") conn.execute("PRAGMA journal_mode=WAL") new_count = 0 skip_count = 0 seen_urls = set() for page in range(1, _PAGES + 1): if page == 1: url = LIST_URL else: url = f"{LIST_URL}index_{page-1}.html" try: r = fetch(url, session=s) html_text = r.content.decode("utf-8", errors="replace") except Exception as e: print(f"Page {page} fetch error: {e}", flush=True) break items = parse_list(html_text, url) print(f"Page {page}: found {len(items)} items", flush=True) if not items and page > 1: print("Empty page, stop pagination", flush=True) break for abs_url, title, date in items: if abs_url in seen_urls: continue seen_urls.add(abs_url) try: r = fetch(abs_url, session=s) dhtml = r.content.decode("utf-8", errors="replace") except Exception as e: print(f" detail error {abs_url}: {e}", flush=True) skip_count += 1 continue dtitle, ddate, content_html = parse_detail(dhtml, abs_url) if not dtitle: dtitle = title if not ddate: ddate = date content, has_table, _ = html_to_text(content_html, abs_url) # empty content check plain_len = len(re.sub(r"<[^>]+>", "", content).strip()) if plain_len < 10: print(f" skip empty: {dtitle[:30]}", flush=True) skip_count += 1 continue ok = store_record(conn, abs_url, dtitle, content, ddate, has_table) if ok: new_count += 1 print(f" + {dtitle[:40]} ({ddate})", flush=True) else: skip_count += 1 time.sleep(0.3) time.sleep(0.5) conn.commit() conn.close() print(f"新增: {new_count}", flush=True) print(f"跳过: {skip_count}", flush=True) print(f"=== {SITE_NAME} done ===", flush=True) if __name__ == "__main__": main()