]*href=["\']([^"\']+\.(?:pdf|docx?|xlsx?|wps))["\']', c, re.I)
# 先删 script/style 再判断文本(script 内 JS 字符串含 ]*>.*?', '', c, flags=re.DOTALL|re.I)
c2 = re.sub(r']*>', '', c2)
text = re.sub(r'<[^>]+>', '', c2).strip()
# 有效内容: 文本>10 或 带文字的链接(空不算) 或 图片
has_named_link = bool(re.search(r']*>\s*[^<\s]', c2, re.DOTALL))
if len(text) > 10 or has_named_link or 'PDF原文
'
return ""
return ""
def extract_meta(html):
title = ""
tm = re.search(r']*name="ArticleTitle"[^>]*content="([^"]*)"', html)
if tm:
title = tm.group(1)
if not title:
ttm = re.search(r'(.*?)<', html)
if ttm:
title = ttm.group(1).strip()
date = ""
for pm in re.finditer(r'PubDate[^>]*content="([^"]*)"', html):
d = pm.group(1).strip()
cm = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', d)
if cm:
date = f"{cm.group(1)}-{cm.group(2).zfill(2)}-{cm.group(3).zfill(2)}"
break
if not date:
dm = re.search(r'(\d{4}-\d{2}-\d{2})', html)
if dm:
date = dm.group(1)
return title, date
def fetch_url(url):
for retry in range(3):
try:
r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
if r.status_code == 200:
r.encoding = "utf-8"
return r.text
except Exception as e:
if retry < 2:
time.sleep(2)
return None
def run(max_pages=200):
print(f"\n{'='*50}")
print(f"🚀 {SITE_NAME} ({DOMAIN})")
print(f"{'='*50}")
items, total = fetch_list(1)
if not items:
print("⚠ API无返回")
return
print(f" 共{total}条, 15条/页 = {(total+14)//15}页")
total_pages = min(max_pages, (total + 14) // 15)
all_list_items = list(items)
for pg in range(2, min(total_pages, _MAX_PG or total_pages) + 1):
its, _ = fetch_list(pg)
if not its:
print(f" 第{pg}页: 空 -> 结束")
break
all_list_items.extend(its)
dates = [it["date"] for it in its if it["date"]]
print(f" 第{pg}页: {len(its)} 条 ({dates[-1] if dates else '?'} ~ {dates[0] if dates else '?'})")
if dates and max(dates) < CUTOFF:
print(f" 该页已全部早于截断日{CUTOFF}，停止翻页")
break
time.sleep(0.15)
print(f"\n📊 API共 {len(all_list_items)} 条")
out = []
err = 0
for i, item in enumerate(all_list_items, 1):
if item["date"] and item["date"] < CUTOFF:
continue
html = fetch_url(item["url"])
if not html:
err += 1
continue
pt, pd = extract_meta(html)
real_title = pt if pt else item["title"]
real_date = pd if pd else item["date"]
if real_date and real_date < CUTOFF:
continue
content = extract_content(html)
text_len = len(re.sub(r'<[^>]+>', '', content).strip()) if content else 0
# 有效: 文本≥10 或 含图片 或 含链接(PDF附件段落 text_len 可能仅5)
if text_len < 10 and '<img' not in (content or '') and '<a href=' not in (content or ''):
print(f" ⚠ 空正文: {real_title[:40]} | {item["url"]}")
err += 1
continue
out.append({
"site_name": SITE_NAME,
"title": real_title,
"pub_date": real_date,
"content": content,
"source_url": item["url"],
"url": item["url"],
})
if i % 10 == 0:
print(f" ...{i}/{len(all_list_items)}")
time.sleep(0.2)
print(f"\n📦 待入库: {len(out)}, 失败: {err}")
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
push_to_searchdb(out, batch_label=SITE_NAME)
if __name__ == "__main__":
run(max_pages=200)