(handled by render_paragraph) if sub.find_parent("p"): continue txt = sub.get_text(" ", strip=True) if txt and not any(sk in txt for sk in SKIP_TEXTS): if not parts or txt not in parts[-1]: parts.append(txt) else: txt = child.get_text(" ", strip=True) if txt and not any(sk in txt for sk in SKIP_TEXTS): if not parts or txt not in parts[-1]: parts.append(txt) # Step 4: Assemble in order: text parts, tables, images result = "\n\n".join(parts) if table_htmls: result += "\n\n" + "\n\n".join(table_htmls) if img_lines: result += "\n\n" + "\n\n".join(img_lines) return result.strip() # ---------- Attachments ---------- def extract_attachments(soup, detail_url): atts = [] seen = set() for a in soup.find_all("a", href=True): href = a["href"] if re.search(r"\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$", href, re.I): full_url = urljoin(detail_url, href) name = a.get_text(strip=True) or href.split("/")[-1].split("?")[0] if full_url not in seen: seen.add(full_url) atts.append({"name": name, "url": full_url}) return atts # ---------- Detail ---------- def fetch_detail(url): try: r = requests.get(url, headers=HEADERS, timeout=20) r.encoding = "utf-8" except Exception as e: print(f" fetch_detail error {url}: {e}", file=sys.stderr) return None, None, None, [] soup = BeautifulSoup(r.text, "html.parser") # --- Title from Meta --- title = "" mt = soup.find("meta", attrs={"name": "ArticleTitle"}) if mt and mt.get("content"): title = mt["content"].strip() if not title: tag = soup.find("title") if tag: t = tag.get_text(strip=True) t = re.sub(r"\s*[-_—|].*(?:通知公告|高端制造).*$", "", t).strip() title = t # --- Date --- date_str = "" mt = soup.find("meta", attrs={"name": "PubDate"}) if mt and mt.get("content"): date_str = mt["content"].strip()[:10] # --- Content (UCAPCONTENT inside .c_d_content) --- content_div = soup.find("div", class_="c_d_content") if content_div: ucap = content_div.find("UCAPCONTENT") if ucap: content_div = ucap attachments = extract_attachments(soup, url) if content_div: content = extract_content(content_div, url) else: content = "" # Image-only fallback if len(content.strip()) < 20: content = f'
' imgs = soup.find_all("img") img_lines = [] for img in imgs: src = img.get("src", "") if src: full_src = urljoin(url, src) alt = img.get("alt", "") img_lines.append(f'' if alt else f'') if img_lines: content += "\n\n" + "\n\n".join(img_lines) # Append attachments to content (skip if already inline) if attachments: for a in attachments: inline_marker = f"[{a['name']}]({a['url']})" if inline_marker not in content: content += f"\n\n附件：[{a['name']}]({a['url']})" return title, date_str, content, attachments # ---------- List ---------- def get_list_url(page_num): if page_num == 1: return LIST_URL return f"http://www.sipac.gov.cn/szgyyqgdzzygjmyq/tzgg/list_{page_num}.shtml" def extract_list_page(page_num): url = get_list_url(page_num) try: r = requests.get(url, headers=HEADERS, timeout=20) r.encoding = "utf-8" except Exception as e: print(f" Error fetching {url}: {e}", file=sys.stderr) return [] if r.status_code != 200: print(f" Page {page_num}: HTTP {r.status_code}", file=sys.stderr) return [] soup = BeautifulSoup(r.text, "html.parser") ul = soup.select_one("div.listBox ul") if not ul: return [] items = [] for li in ul.find_all("li"): a = li.find("a") span = li.find("span") if not a or not a.get("href"): continue href = a["href"].strip() title = a.get("title", "").strip() or a.get_text(strip=True) page_url = urljoin(url, href) date_str = span.get_text(strip=True) if span else "" if title and page_url: items.append((title, page_url, date_str)) return items # ---------- DB ---------- def init_db(): conn = sqlite3.connect(DB_PATH, timeout=60) conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA busy_timeout=5000") return conn def save_article(conn, title, page_url, publish_date, content, attachments): summary = content if content else title summary = re.sub(r"\s+", " ", summary).strip() att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]" try: cur = conn.execute( """INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (SITE_NAME, page_url, page_url, title.strip(), publish_date, summary, content, CATEGORY, att_json, GROUP), ) return cur.rowcount > 0 except Exception as e: print(f" DB error: {e}", file=sys.stderr) return False # ---------- Main ---------- def main(): max_pages = MAX_PAGES if len(sys.argv) > 1: try: max_pages = int(sys.argv[1]) except ValueError: if sys.argv[1] in ("--max-pages",) and len(sys.argv) > 2: try: max_pages = int(sys.argv[2]) except ValueError: pass print(f"Scraping {SITE_NAME}, max pages: {max_pages}", file=sys.stderr) conn = init_db() total_new = 0 total_found = 0 # Clean old data for this site # QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等) conn.execute("DELETE FROM gov_raw WHERE site_name = ?", (SITE_NAME,)) conn.commit() print(f" Cleared old data for {SITE_NAME}", file=sys.stderr) for pn in range(1, max_pages + 1): articles = extract_list_page(pn) if not articles: print(f" Page {pn}: 0 articles (end)", file=sys.stderr) break total_found += len(articles) for title, page_url, date in articles: if date and date < "2020-01-01": print(f" Skip old: {date} {title[:40]}", file=sys.stderr) continue # Check if already exists cur = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,)) if cur.fetchone(): continue det_title, det_date, det_content, det_atts = fetch_detail(page_url) final_title = det_title or title final_date = det_date or date final_content = det_content or f'' final_atts = det_atts or [] if save_article(conn, final_title, page_url, final_date, final_content, final_atts): total_new += 1 if total_new <= 5: print(f" + {final_date} {final_title[:50]}", file=sys.stderr) time.sleep(0.3) print(f" Page {pn}: {len(articles)} found, {total_new} total", file=sys.stderr) conn.commit() conn.close() print(f"\nDone! Found: {total_found}, New: {total_new}", file=sys.stderr) if __name__ == "__main__": main()