parts = []
for line in seg.split("\n"):
line = line.strip()
if not line:
continue
line = re.sub(r"[ \t\u00a0]+", " ", line)
if not line:
continue
# 段内含表格/图片/链接 → 直接
]*>)?\s*(\d{4})-(\d{1,2})-(\d{1,2})", html_src)
publish = ""
if m:
publish = "%s-%02d-%02d" % (m.group(1), int(m.group(2)), int(m.group(3)))
# 正文: td#Zoom 深度计数到闭合
body_html = ""
zi = html_src.find('id="Zoom"')
if zi < 0:
zi = html_src.find("id='Zoom'")
if zi >= 0:
seg = html_src[zi:]
# 跳过 td 开标签
start = seg.find(">")
if start >= 0:
start += 1
depth = 1 # td 本身
pos = start
while pos < len(seg):
# 找下一个
td_open = seg.find("", pos)
if td_close < 0:
break
if td_open >= 0 and td_open < td_close:
depth += 1
pos = td_open + 3
else:
depth -= 1
if depth == 0:
body_html = seg[start:td_close].strip()
# 附件区: Zoom td 闭合后 | 附件名 |
att_seg = seg[td_close:td_close + 2000]
for m in re.finditer(
r']*>(.*?)',
att_seg, re.S | re.I):
ahref = m.group(1)
atxt = clean_title(re.sub(r"<[^>]+>", "", m.group(2)))
if atxt:
abs_a = urllib.parse.urljoin(BASE + "/hpgs/", ahref)
body_html += f'\n{atxt} '
break
pos = td_close + 5
return title or None, publish or None, body_html, []
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--pages", type=int, default=1, help="抓取页数")
ap.add_argument("--db", default="/mnt/data/search.db", help="SQLite 数据库路径")
ap.add_argument("--script-name", default=SCRIPT_NAME)
args = ap.parse_args()
db_path = args.db
script_name = args.script_name
if not db_path.startswith("/mnt/"):
db_path = os.path.expanduser(db_path)
conn = sqlite3.connect(db_path, timeout=60)
cur = conn.cursor()
new_count = 0
dup_count = 0
err_count = 0
session = requests.Session()
for page in range(1, args.pages + 1):
if page == 1:
url = f"{BASE}/hpgs/"
else:
url = f"{BASE}/hpgs/index_{page}.html"
try:
r = session.get(url, headers=HEADERS, timeout=TIMEOUT)
r.encoding = "utf-8"
except Exception as e:
print(f"[{page}] 请求失败 {url}: {e}")
err_count += 1
continue
items = parse_list(r.text)
print(f"[{page}] 列表 {len(items)} 条 {url}")
if not items:
continue
for title, art_url, pub_date in items:
try:
cur.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=? AND script_name=?", (art_url, script_name))
if cur.fetchone()[0] > 0:
dup_count += 1
continue
dr = session.get(art_url, headers=HEADERS, timeout=TIMEOUT)
dr.encoding = "utf-8"
d_title, d_date, content_html, links = parse_detail(dr.text, art_url)
if not content_html.strip():
print(f" 跳过空正文: {title[:30]}")
dup_count += 1
continue
if d_title:
title = d_title
if d_date:
pub_date = d_date
content, links_out = html_to_text(content_html)
att_txt = "\n\n".join(a[2] for a in links_out) if links_out else ""
summary = content[:500] if content else ""
cur.execute(
"INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, status, category, visits, content, tags, industry, attachments, group_name, has_table, script_name) "
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
(SITE_NAME, art_url, art_url, title, pub_date, 0, summary, "", "", 0, content, "",
"other", att_txt, GROUP_NAME, 1 if " |