.*?
\s*([0-9]{4}-[0-9]{2}-[0-9]{2})\s*
', html, re.DOTALL)
if dm:
date = dm.group(1).strip()
summary = re.sub(r'<[^>]+>', '', content)[:300] if content else ""
summary = re.sub(r'页面内容', '', summary).strip()
return {"title": title, "content": content, "date": date, "summary": summary}
def get_total_pages(html):
m = re.search(r'maxPage\s*=\s*(\d+)', html)
if m:
return int(m.group(1))
return 1
def crawl_all():
print("爬取: 石家庄炼化公示信息")
html = http_get(LIST_URL)
if not html:
print("无法获取首页")
return
total_pages = get_total_pages(html)
print(f"总页数: {total_pages}")
all_items = []
items = parse_list(html)
all_items.extend(items)
print(f" 第1页: {len(items)} 条")
for page in range(2, total_pages + 1):
html = http_get(f"{LIST_URL}?page={page}")
if not html:
continue
items = parse_list(html)
all_items.extend(items)
print(f" 第{page}页: {len(items)} 条")
print(f"\n列表共 {len(all_items)} 条")
for item in all_items:
print(f" 详情: {item['title'][:30]}...", end=" ")
detail = fetch_detail(item["url"])
if detail:
item["content"] = detail["content"]
item["summary"] = detail["summary"]
if detail["date"]:
item["date"] = detail["date"]
if detail["title"]:
item["title"] = detail["title"]
print("OK")
else:
print("FAIL")
db = sqlite3.connect(QUALITY_DB, timeout=60)
db.execute("CREATE TABLE IF NOT EXISTS quality_results (id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT, domain TEXT, title TEXT, url TEXT, content TEXT, publish_date TEXT, summary TEXT, created_at TEXT DEFAULT (datetime('now','localtime')))")
insert = 0
for item in all_items:
try:
db.execute("INSERT OR IGNORE INTO quality_results (site_name,domain,title,url,content,publish_date,summary) VALUES (?,?,?,?,?,?,?)",
(SITE_NAME, DOMAIN, item["title"], item["url"], item.get("content",""), item["date"], item.get("summary","")))
if db.total_changes > 0:
insert += 1
except:
pass
db.commit()
total = db.execute("SELECT COUNT(*) FROM quality_results WHERE site_name=?", (SITE_NAME,)).fetchone()[0]
db.close()
print(f"\n入库: 新增{insert}, 累计{total}")
sync_to_server(all_items)
return all_items
def sync_to_server(items):
if not items:
return
print("\n同步到服务器...")
# 写SQL到临时文件，SCP到服务器执行
sql_lines = []
for item in items:
t = item["title"].replace("'", "''")
u = item["url"].replace("'", "''")
c = (item.get("content","") or "").replace("'", "''")
s = (item.get("summary","") or "").replace("'", "''")
d = item["date"]
sql_lines.append(f"INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary, date_rank) VALUES ('{SITE_NAME}','{u}','{t}','{c}','{d}','{s}','{d}');")
sql = "BEGIN;" + "".join(sql_lines) + "COMMIT;"
tmp = "/tmp/sync_sjlh.sql"
with open(tmp, "w") as f:
f.write(sql)
# SCP到服务器
r = subprocess.run(["scp", "-o", "ConnectTimeout=10", tmp, f"{SERVER_SSH}:{tmp}"],
capture_output=True, text=True, timeout=60)
if r.returncode != 0:
print(f" SCP失败: {r.stderr}")
return
# 在服务器执行
r = subprocess.run(["ssh", "-o", "ConnectTimeout=10", SERVER_SSH, f"sqlite3 /root/search.db < {tmp} && rm {tmp}"],
capture_output=True, text=True, timeout=120)
if r.returncode == 0:
print(" 同步OK")
else:
print(f" 同步失败: {r.stderr[:200]}")
os.unlink(tmp)
# 更新FTS
subprocess.run(["ssh", "-o", "ConnectTimeout=10", SERVER_SSH,
"sqlite3 /root/search.db \"INSERT OR REPLACE INTO gov_search(gov_raw) SELECT rowid FROM gov_raw WHERE rowid NOT IN (SELECT rowid FROM gov_search);\""],
capture_output=True, text=True, timeout=60)
print(" FTS更新完成")
if __name__ == "__main__":
if "--test" in sys.argv:
html = http_get(LIST_URL)
if html:
items = parse_list(html)[:1]
for item in items:
print(f"标题: {item['title']}")
d = fetch_detail(item["url"])
if d:
print(f"日期: {d['date']}")
print(f"内容长度: {len(d['content'])}")
print(f"摘要: {d['summary'][:100]}")
else:
crawl_all()