', sub_html, flags=re.I)
for bp in br_parts:
bt = re.sub(r'<[^>]+>', "", bp).strip()
bt = re.sub(r'&[nN][bB][sS][pP];', " ", bt)
bt = re.sub(r'\u3000', "", bt)
if bt:
parts.append(bt)
else:
parts.append(div_text)
else:
# tag
p_text = re.sub(r'<[^>]+>', "", inner)
p_text = re.sub(r'&[nN][bB][sS][pP];', " ", p_text)
p_text = re.sub(r'\u3000', "", p_text)
p_text = p_text.strip()
if p_text:
parts.append(p_text)
# Fallback: if no structured parts found
if not parts:
text = re.sub(r']*>.*?', "", processed_html, flags=re.DOTALL | re.I)
text = re.sub(r']*>.*?', "", processed_html, flags=re.DOTALL | re.I)
text = re.sub(r'', "\n", text, flags=re.I)
text = re.sub(r'<[^>]+>', "", text)
text = re.sub(r'&[nN][bB][sS][pP];', " ", text)
text = re.sub(r'\u3000', "", text)
for line in text.split("\n"):
line = line.strip()
if line:
parts.append(line)
# THIRD PASS: replace table placeholders with actual Markdown tables
final_parts = []
for p in parts:
ph = re.search(r'__TBL_(\d+)__', p)
if ph:
final_parts.append(tables_md[int(ph.group(1))])
else:
final_parts.append(p)
content = "\n\n".join(final_parts)
return {"title": title, "content": content, "date": pub_date, "url": source_url,
"attachments": attachments, "site_name": SITE_NAME, "group": GROUP}
def crawl(test_mode=False):
import sqlite3
all_items = []
seen_urls = set()
for page in range(1, MAX_PAGES + 1):
if page == 1:
url = "http://www.zhangpu.gov.cn/cms/html/zpxrmzf/gsgg/index.html"
else:
url = "http://www.zhangpu.gov.cn/cms/sitemanage/index.shtml?siteId=%s&page=%d" % (SITE_ID, page)
html = fetch_page(url, referer=LIST_REFERER)
items = extract_list_items(html)
print("[Page %d] %s" % (page, url))
print(" Found %d items" % len(items))
for item in items:
if item["url"] not in seen_urls:
seen_urls.add(item["url"])
all_items.append(item)
if len(items) == 0:
print(" No items found, stopping pagination")
break
print("\nTotal unique items: %d" % len(all_items))
if test_mode:
for item in all_items[:3]:
print("\n=== Detail: %s ===" % item["title"][:50])
html = fetch_page(item["url"], referer=LIST_REFERER)
result = extract_content(html, item["url"])
print(" Title: %s" % result["title"])
print(" Date: %s" % result["date"])
print(" Content (%d chars): %s" % (len(result["content"]), result["content"][:200]))
print(" Attachments: %d" % len(result["attachments"]))
for att in result["attachments"]:
print(" - %s: %s" % (att["name"], att["url"][:80]))
return
conn = sqlite3.connect(DB_PATH, timeout=60)
conn.execute("PRAGMA busy_timeout=30000")
c = conn.cursor()
inserted = 0
existing = 0
total = len(all_items)
for idx, item in enumerate(all_items):
c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
if c.fetchone():
existing += 1
continue
html = fetch_page(item["url"], referer=LIST_REFERER)
result = extract_content(html, item["url"])
attachments_json = json.dumps(result["attachments"], ensure_ascii=False) if result["attachments"] else ""
summary = result["title"] + " " + SITE_NAME
if result["content"]:
summary += " " + result["content"][:200]
c.execute("INSERT INTO gov_raw (title, content, summary, publish_date, page_url, site_name, attachments, group_name, source_url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(result["title"], result["content"], summary, result["date"] or item["date"], result["url"], result["site_name"], attachments_json, result["group"], result["url"]))
inserted += 1
if inserted % 5 == 0:
conn.commit()
print(" Progress: %d/%d inserted..." % (inserted, total))
conn.commit()
conn.close()
print("\nDone. Inserted: %d, Existing: %d (total: %d)" % (inserted, existing, total))
if __name__ == "__main__":
if "--test" in sys.argv:
crawl(test_mode=True)
else:
crawl(test_mode=False)