',
re.DOTALL
)
for m in pattern.finditer(html):
href = m.group(1).strip()
title = m.group(2).strip()
date_str = m.group(3)
if href and title:
items.append((title, href, date_str))
return items
def fetch_detail(url):
"""获取详情页的正文内容"""
full_url = BASE + url if url.startswith('/') else url
html = fetch_page(full_url, encoding='utf-8')
if not html:
return None, None, None
# 发布日期
pub_date = ""
m = re.search(r'发布日期[：:]\s*(\d{4}-\d{2}-\d{2})', html)
if m:
pub_date = m.group(1)
# 正文 —
content = ""
m = re.search(r']*>(.*?)
\s*', html, re.DOTALL)
if m:
content = m.group(1).strip()
if not content:
# fallback: 正文区域
m = re.search(r']*>(.*?)', html, re.DOTALL)
if m:
content = m.group(1).strip()
if content:
# 移除无用标签
content = re.sub(r']*>.*?', '', content, flags=re.DOTALL|re.I)
content = re.sub(r']*>.*?', '', content, flags=re.DOTALL|re.I)
# 保留结构化 HTML，仅清理 style/class
content = clean_html(content)
# 提取附件链接
attachments = []
for att_href, att_text in re.findall(
r']*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar))"[^>]*>\s*([^<]+)\s*',
html, re.IGNORECASE
):
att_full = BASE + att_href if att_href.startswith('/') else att_href
attachments.append(f'{att_text}')
# API-gateway 下载链接
for dl_href, dl_text in re.findall(
r']*href="(/api-gateway[^"]*download[^"]*)"[^>]*>\s*([^<]+)\s*', html, re.IGNORECASE
):
att_full = BASE + dl_href if dl_href.startswith('/') else dl_href
attachments.append(f'{dl_text}')
if attachments:
content += '\n
附件：
\n' + '\n'.join(f'
{a}
' for a in attachments)
# 标题
title = ""
m = re.search(r']*>(.*?)', html, re.DOTALL)
if not m:
m = re.search(r']*>(.*?)', html, re.DOTALL)
if m:
title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
return title or None, content or None, pub_date or None
def insert_to_db(items):
if not items:
print(" ⏭ 无数据")
return
db = sqlite3.connect(SEARCH_DB, timeout=60)
db.execute("PRAGMA journal_mode=WAL")
db.execute("PRAGMA synchronous=NORMAL")
ok, skip = 0, 0
for item in items:
try:
db.execute("INSERT OR IGNORE INTO gov_raw "
"(site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
item.get("site_name","")[:200],
item.get("url",""),
item.get("url",""),
(item.get("title") or "")[:500],
(item.get("pub_date") or "")[:10],
(item.get("summary") or "")[:500],
item.get("content",""),
"active", "", (item.get("tags") or "")[:100],
))
if db.total_changes > 0:
ok += 1
else:
skip += 1
except:
skip += 1
db.commit()
db.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
"SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
"WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (SITE_NAME,))
db.commit()
db.close()
print(f" 💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")
def main():
print(f"\n{'='*50}")
print(f"🏠 {SITE_NAME} - 已批准项目公告")
print(f"{'='*50}")
all_items = []
seen_urls = set()
# 只爬最近3页（已批准项目公告通常最相关的在最新几页）
# 如需全量，改 range(1, TOTAL_PAGES+1)
MAX_PAGES = 3 # 增量用，每天只跑前3页
if len(sys.argv) > 1 and sys.argv[1] == "full":
MAX_PAGES = TOTAL_PAGES
print("📋 全量模式：122页")
for page_no in range(1, MAX_PAGES + 1):
print(f" 📄 第 {page_no}/{MAX_PAGES} 页...", end=" ", flush=True)
html = fetch_list_page(page_no)
if not html:
print("❌ 无响应")
break
items = parse_list(html)
if not items:
print("0 条（可能已到底）")
break
print(f"✅ {len(items)} 条")
for title, href, date_str in items:
full_url = BASE + href if href.startswith('/') else href
if full_url in seen_urls:
continue
seen_urls.add(full_url)
if date_str and date_str < THREE_YEARS_AGO:
continue
print(f" {title[:55]}...", end=" ", flush=True)
dt, content, pub_date = fetch_detail(href)
if dt:
title = dt
if pub_date:
date_str = pub_date
summary = re.sub(r'<[^>]+>', ' ', content or '').strip()[:300]
summary = re.sub(r'\s+', ' ', summary)
all_items.append({
"site_name": SITE_NAME,
"title": title,
"url": full_url,
"content": content or "",
"pub_date": date_str or "",
"summary": summary,
"tags": "环评审批",
})
print(f"✅ {date_str}")
time.sleep(0.3)
if len(items) < PAGE_SIZE:
print(" (末页)")
break
time.sleep(0.5)
if all_items:
insert_to_db(all_items)
print(f"\n✅ 完成! 共 {len(all_items)} 条")
if __name__ == "__main__":
t0 = time.time()
main()
print(f"⏱ 耗时: {time.time()-t0:.1f}s")