内容加载失败
" return title, content_html, date_str def article_exists(page_url): c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url = ?", (page_url,)) return c.fetchone()[0] > 0 def sync_fts(row_id, title, site_name): try: # 2026-09-22: 先提交 gov_raw —— 下面手动写 FTS 会因触发器已写过同一 # rowid 而 IntegrityError，若不先 commit，这条记录会被一并回滚（静默丢数据） conn.commit() c.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)", (row_id, title, site_name, "")) except: pass def insert_article(title, page_url, content_html, date_str): display_url = page_url.replace("http://", "").replace("https://", "") summary = "" c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,)) row = c.fetchone() existing_id = row[0] if row else None try: c.execute("""INSERT OR REPLACE INTO gov_raw (page_url, title, content, publish_date, site_name, source_url, summary, category, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'crawl_daye.py')""", (page_url, title, content_html, date_str, SITE_NAME, display_url, summary, "")) row_id = existing_id or c.lastrowid if row_id: sync_fts(row_id, title, SITE_NAME) return True except Exception as e: print(f" [DB ERROR] {e}") return False def get_page_url(page_num): if page_num == 0: return BASE_URL + "index.shtml" return f"{BASE_URL}index_{page_num}.shtml" def main(): total_new = 0 total_updated = 0 total_skipped = 0 # QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等) c.execute("DELETE FROM gov_raw WHERE site_name = ?", (SITE_NAME,)) conn.commit() print(f" Pages to crawl: 0-{MAX_PAGES-1}") for page in range(0, MAX_PAGES): url = get_page_url(page) print(f" List page {page}/{MAX_PAGES-1}") r = None for attempt in range(3): try: r = requests.get(url, headers=HEADERS, timeout=20) r.encoding = "utf-8" break except Exception as e: if attempt < 2: print(f" Retry {attempt+1}...") time.sleep(2) if r is None: print(f" [SKIP] page {page} failed") continue items = parse_list_page(r.text) if not items: print(f" No items found") continue print(f" Found {len(items)} items") for title, item_url, list_date in items: full_url = urljoin(BASE_URL, item_url) if list_date: try: item_date = datetime.strptime(list_date, "%Y-%m-%d") if item_date < cutoff_date.replace(tzinfo=None): total_skipped += 1 continue except: pass exists = article_exists(full_url) if INCREMENTAL and exists: total_skipped += 1 continue print(f" Fetching: {title[:35]}...", end=" ") d_title, content_html, detail_date = fetch_detail(full_url) if not d_title: d_title = title if not detail_date and list_date: detail_date = list_date if insert_article(d_title, full_url, content_html, detail_date): if exists: total_updated += 1 print("updated") else: total_new += 1 print("inserted") else: print("FAILED") time.sleep(0.3) conn.commit() print(f" [COMMIT] page {page} done") conn.close() print(f"\n ✅ 完成: 新增 {total_new}, 更新 {total_updated}, 跳过 {total_skipped}") if __name__ == "__main__": main()