]*>', '', raw)
raw = re.sub(r'
\s* \s*
', '', raw)
raw = re.sub(r'\s*
', '', raw)
if '' not in raw and ''
body = raw.strip()
if len(body) >= 20:
break
if not body or len(body) < 20:
body = ""
return title, pub_date, body
except Exception as e:
print(f" [ERR] detail {url}: {e}", flush=True)
return "", "", ""
def main():
max_pages = parse_args() or MAX_PAGES
print(f"[JXFZ-HBJ-GGGS] SITE={SITE_NAME} max_pages={max_pages}", flush=True)
session = requests.Session()
session.headers.update({"User-Agent": UA})
all_items = []
seen = set()
for pg in range(1, max_pages + 1):
items = fetch_list(session, pg)
if not items:
print(f" [END] page {pg} empty", flush=True)
break
new_count = 0
for it in items:
if it["href"] in seen:
continue
seen.add(it["href"])
all_items.append(it)
new_count += 1
print(f" Page {pg}: {len(items)}条(新{new_count}) 累计{len(all_items)}", flush=True)
time.sleep(0.4)
print(f"列表完成: {len(all_items)} 条, 抓详情...", flush=True)
results = []
for i, it in enumerate(all_items):
d_title, d_date, body = fetch_detail(session, it["href"])
title = d_title or it["title"]
pub_date = d_date or it.get("date", "")
summary = re.sub(r"<[^>]+>", " ", body) if body else title
summary = re.sub(r"\s+", " ", summary).strip()[:300]
atts = []
if body:
for m in re.finditer(r']*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar|ofd|wps))"[^>]*>([^<]*)', body, re.I):
atts.append({"name": m.group(2).strip(), "url": m.group(1)})
results.append({
"site_name": SITE_NAME, "source_url": it["href"], "url": it["href"],
"title": title, "pub_date": pub_date, "content": body,
"summary": summary, "category": CATEGORY,
"attachments": json.dumps(atts, ensure_ascii=False) if atts else "",
})
if (i + 1) % 10 == 0:
print(f" detail {i+1}/{len(all_items)}", flush=True)
time.sleep(0.3)
print(f" pushing {len(results)} items...", flush=True)
push_to_searchdb(results, batch_label=SITE_NAME)
empty = sum(1 for r in results if not (r.get("content") or "").strip())
print(f" DONE. pushed={len(results)} 空正文={empty}", flush=True)
if __name__ == "__main__":
main()