becomes one paragraph.
as Markdown. / recursed into.
"""
if processed is None:
processed = set()
parts = []
for child in node.children:
if isinstance(child, str):
txt = child.strip()
if txt:
parts.append(txt)
continue
if not child.name:
continue
if id(child) in processed:
continue
processed.add(id(child))
tag = child.name.lower()
if tag == "p":
p_parts = []
for elem in child.contents:
if isinstance(elem, str):
txt = elem.strip()
if txt:
p_parts.append(txt)
elif elem.name == "a":
processed.add(id(elem))
ahref = elem.get("href", "").strip()
atxt = elem.get_text(strip=True)
if ahref and atxt and not ahref.startswith("javascript"):
full_href = urljoin(url, ahref)
p_parts.append(f"[{atxt}]({full_href})")
elif atxt:
p_parts.append(atxt)
elif elem.name == "img":
processed.add(id(elem))
src = elem.get("src", "")
if src:
alt = elem.get("alt", "")
full_src = urljoin(url, src)
p_parts.append(f"" if alt else f"")
elif elem.name == "br":
pass
elif hasattr(elem, "get_text"):
txt = elem.get_text(" ", strip=True)
if txt:
p_parts.append(txt)
joined = " ".join(p_parts)
if joined:
parts.append(joined)
elif tag == "table":
rows = []
for tr in child.find_all("tr"):
cells = [td.get_text(" ", strip=True) for td in tr.find_all(["td", "th"])]
if cells:
rows.append("| " + " | ".join(cells) + " |")
if rows:
parts.append("\n".join(rows))
elif tag == "img":
src = child.get("src", "")
if src:
alt = child.get("alt", "")
full_src = urljoin(url, src)
parts.append(f"" if alt else f"")
elif tag in ("div", "section", "td", "th"):
parts.extend(_render_yuan_content(child, url, processed))
elif tag == "br":
pass
else:
txt = child.get_text(" ", strip=True)
if txt:
parts.append(txt)
return parts
def _clean_yuan_spaces(text):
"""Clean excessive spaces from inline font/span fragments (安徽政务CMS style)."""
text = re.sub(r"([\u4e00-\u9fff])[ \t\u3000]+([\u4e00-\u9fff])", r"\1\2", text)
text = re.sub(r"([\u4e00-\u9fff])[ \t\u3000]+([《（【「『\[({])", r"\1\2", text)
text = re.sub(r'\s+([，。、；：？！）】」』)"\]])', r"\1", text)
text = re.sub(r'([（【「『\[({《])\s+', r"\1", text)
text = re.sub(r"(\d)[ \t\u3000]+([\u4e00-\u9fff])", r"\1\2", text)
text = re.sub(r"([\u4e00-\u9fff])[ \t\u3000]+(\d)", r"\1\2", text)
text = re.sub(r"(\d)\s+\.", r"\1.", text)
text = re.sub(r"(\d)\s+(\d)", r"\1\2", text)
text = re.sub(r"(:\s*)(\d)", r":\2", text)
text = re.sub(r" {2,}", " ", text)
text = re.sub(r" +\n", "\n", text)
text = re.sub(r"\n +", "\n", text)
return text.strip()
def parse_detail(html, url):
"""Parse detail — title from , date from span.fbsj, content from div.xxgk-wenzhang."""
soup = BeautifulSoup(html, "html.parser")
# Title from <title>
title = ""
title_tag = soup.find("title")
if title_tag:
raw = title_tag.get_text(strip=True)
# Strip site suffix like " _六安市裕安区人民政府"
title = re.sub(r"_[^_]+$", "", raw).strip()
# Date from span.fbsj
pub_date = ""
date_span = soup.find("span", class_="fbsj")
if date_span:
raw = date_span.get_text(strip=True)
m = re.search(r"(\d{4}-\d{2}-\d{2})", raw)
if m:
pub_date = m.group(1)
# Content from div.wzcon.j-fontContent (actual article body)
content_html = ""
content_div = soup.find("div", class_="wzcon")
if not content_div:
content_div = soup.find("div", class_="gkwz_contnet")
if not content_div:
content_div = soup.find("div", class_=re.compile(r"xxgk.wenzhang|wenzhang"))
if content_div:
parts = _render_yuan_content(content_div, url)
content_html = "\n\n".join(parts)
# Clean TRS-style spaces from inline fonts
content_html = _clean_yuan_spaces(content_html)
# Attachments — search in xxgk-wenzhang (metadata area)
scope = soup.find("div", class_=re.compile(r"xxgk.wenzhang|wenzhang")) or soup
attachments = []
seen_urls = set()
for a in scope.find_all("a", href=True):
ahref = a["href"].strip().lower()
if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|txt)$", ahref):
full_url = urljoin(url, a["href"].strip())
if full_url not in seen_urls:
seen_urls.add(full_url)
attachments.append({"title": a.get_text(strip=True) or full_url.split("/")[-1], "url": full_url})
if len(content_html.strip()) < 20 and attachments:
content_html = f'<p><a href="{url}">{title}</a></p>\n\n附件列表：\n'
for att in attachments:
content_html += f" - [{att['title']}]({att['url']})\n"
return title, pub_date, content_html, attachments
def incremental_filter(items):
cutoff = datetime.now() - timedelta(days=INCREMENTAL_DAYS)
filtered = []
for title, url, date_str in items:
if date_str:
try:
item_date = datetime.strptime(date_str, "%Y-%m-%d")
if item_date >= cutoff:
filtered.append((title, url, date_str))
except ValueError:
filtered.append((title, url, date_str))
else:
filtered.append((title, url, date_str))
return filtered
def main():
is_incremental = any(arg in sys.argv for arg in ["--incremental", "incremental", "inc"])
try:
html = fetch(LIST_URL)
items = parse_list(html)
except Exception as e:
print(f"List error: {e}", file=sys.stderr)
items = []
print(f"Total: {len(items)} items", file=sys.stderr)
if is_incremental:
items = incremental_filter(items)
print(f"Incremental: {len(items)} items", file=sys.stderr)
seen = set()
unique_items = []
for item in items:
if item[1] not in seen:
seen.add(item[1])
unique_items.append(item)
print(f"Unique: {len(unique_items)}", file=sys.stderr)
results = []
for title, url, list_date in unique_items:
try:
html = fetch(url)
det_title, det_date, content, attachments = parse_detail(html, url)
final_title = det_title or title
final_date = det_date or list_date
results.append({
"title": final_title,
"page_url": url,
"publish_date": final_date,
"content": content,
"attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
"site_name": SITE_NAME,
})
print(f" OK: {final_title[:50]}", file=sys.stderr)
except Exception as e:
print(f" ERR {url}: {e}", file=sys.stderr)
for r in results:
print(json.dumps(r, ensure_ascii=False))
if __name__ == "__main__":
main()