/
/are recursed into. Standalone becomes a markdown image.
produces an empty part. """ if processed is None: processed = set() parts = [] for child in node.children: if isinstance(child, str): txt = child.strip() if txt: parts.append(txt) continue if not child.name: continue if id(child) in processed: continue processed.add(id(child)) tag = child.name.lower() if tag == "p": p_parts = [] for elem in child.contents: if isinstance(elem, str): txt = elem.strip() if txt: p_parts.append(txt) elif elem.name == "a": processed.add(id(elem)) ahref = elem.get("href", "").strip() atxt = elem.get_text(strip=True) if ahref and atxt: full_href = urljoin(url, ahref) p_parts.append(f"[{atxt}]({full_href})") elif atxt: p_parts.append(atxt) elif elem.name == "img": processed.add(id(elem)) src = elem.get("src", "") if src: alt = elem.get("alt", "") full_src = urljoin(url, src) p_parts.append(f"" if alt else f"") elif elem.name == "br": pass # line break within paragraph, ignore elif hasattr(elem, "get_text"): txt = elem.get_text(" ", strip=True) if txt: p_parts.append(txt) joined = " ".join(p_parts) if joined: parts.append(joined) elif tag == "table": rows = [] for tr in child.find_all("tr"): cells = [td.get_text(" ", strip=True) for td in tr.find_all(["td", "th"])] if cells: rows.append("| " + " | ".join(cells) + " |") if rows: parts.append("\n".join(rows)) elif tag == "img": src = child.get("src", "") if src: alt = child.get("alt", "") full_src = urljoin(url, src) parts.append(f"" if alt else f"") elif tag in ("div", "blockquote", "section", "td", "th"): # Wrapper — recurse into direct children parts.extend(_render_node_children(child, url, processed)) elif tag == "br": parts.append("") else: txt = child.get_text(" ", strip=True) if txt: parts.append(txt) return parts def parse_detail(html, url): """Parse detail — meta title + div.content-wrapper content.""" soup = BeautifulSoup(html, "html.parser") # Title from meta or h2 title = "" meta_title = soup.find("meta", attrs={"name": re.compile(r"Title|ArticleTitle", re.I)}) if meta_title and meta_title.get("content"): title = meta_title["content"].strip() if not title: title_div = soup.find("div", class_=re.compile(r"title", re.I)) if title_div and not title_div.find("a"): title = title_div.get_text(strip=True) # Date pub_date = "" text = soup.get_text() m = re.search(r"(\d{4}-\d{2}-\d{2})", text) if m: pub_date = m.group(1) # Content from div.content-wrapper — recursive descent parser content_html = "" content_div = soup.find("div", class_="content-wrapper") if content_div: parts = _render_node_children(content_div, url) content_html = "\n\n".join(parts) # Attachments scope = content_div or soup attachments = [] for a in scope.find_all("a", href=True): ahref = a["href"].strip().lower() if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|txt)$", ahref): full_url = urljoin(url, a["href"].strip()) attachments.append({"title": a.get_text(strip=True) or full_url.split("/")[-1], "url": full_url}) if len(content_html.strip()) < 20 and attachments: content_html = f'\n\n附件列表：\n' for att in attachments: content_html += f" - [{att['title']}]({att['url']})\n" return title, pub_date, content_html, attachments def incremental_filter(items): cutoff = datetime.now() - timedelta(days=INCREMENTAL_DAYS) filtered = [] for title, url, date_str in items: try: item_date = datetime.strptime(date_str, "%Y-%m-%d") if item_date >= cutoff: filtered.append((title, url, date_str)) except (ValueError, IndexError): filtered.append((title, url, date_str)) return filtered def main(): is_incremental = any(arg in sys.argv for arg in ["--incremental", "incremental", "inc"]) page_urls = [LIST_URL] for i in range(2, MAX_PAGES + 1): page_urls.append(BASE_URL + LIST_PATH + f"/index_{i}.htm") all_items = [] for idx, url in enumerate(page_urls): try: html = fetch(url) items = parse_list(html) print(f"Page {idx+1}: {len(items)} items", file=sys.stderr) all_items.extend(items) except Exception as e: print(f"Page {idx+1} error ({url}): {e}", file=sys.stderr) print(f"Total: {len(all_items)}", file=sys.stderr) if is_incremental: all_items = incremental_filter(all_items) print(f"Incremental: {len(all_items)}", file=sys.stderr) seen = set() unique_items = [] for item in all_items: if item[1] not in seen: seen.add(item[1]) unique_items.append(item) print(f"Unique: {len(unique_items)}", file=sys.stderr) results = [] for title, url, list_date in unique_items: try: html = fetch(url) det_title, det_date, content, attachments = parse_detail(html, url) final_title = det_title or title final_date = det_date or list_date results.append({ "title": final_title, "page_url": url, "publish_date": final_date, "content": content, "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "", "site_name": SITE_NAME, }) print(f" OK: {final_title[:50]}", file=sys.stderr) except Exception as e: print(f" ERR {url}: {e}", file=sys.stderr) saved = 0 for r in results: content_text = r.get('content', '')[:50000] summary = re.sub(r'\s+', ' ', content_text[:200]).strip() or r['title'] attachments = r.get('attachments', '') if isinstance(attachments, str): pass else: attachments = json.dumps(attachments, ensure_ascii=False) try: conn = sqlite3.connect(DB_PATH, timeout=60) c = conn.cursor() c.execute('''INSERT OR IGNORE INTO gov_raw (page_url, title, publish_date, site_name, group_name, summary, content, attachments, source_url) VALUES (?,?,?,?,?,?,?,?,?)''', (r['page_url'], r['title'], r['publish_date'], SITE_NAME, GROUP, summary, content_text, attachments, r['page_url'])) if c.rowcount > 0: saved += 1 conn.commit() conn.close() except Exception as e: print(f" DB error for {r['title'][:30]}: {e}", file=sys.stderr) print(f"Saved to DB: {saved}", file=sys.stderr) for r in results: print(json.dumps(r, ensure_ascii=False)) if __name__ == "__main__": main()