#!/usr/bin/env python3 """ Crawler: 江门市广海湾经济开发区管理委员会 - 部门文件 (NFCMS) JSON API: /postmeta/i/22332.json returns all article metadata Detail: /content/post_ID.html, content in content_div """ import sys, os, json, re, time, requests from datetime import datetime, timedelta from bs4 import BeautifulSoup from urllib.parse import urljoin DB_PATH = os.getenv("SEARCH_DB", "/root/search.db") SITE = "江门市广海湾经济开发区管理委员会" COLUMN = "部门文件" BASE_URL = "http://www.cnts.gov.cn" JSON_URL = f"{BASE_URL}/postmeta/i/22332.json" HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"} def log(msg): print(f"[{SITE}] {msg}", flush=True) def fetch_article_list(): try: r = requests.get(JSON_URL, headers=HEADERS, timeout=30) return r.json() except Exception as e: log(f"JSON API error: {e}") return None def fetch_detail(detail_url): try: r = requests.get(detail_url, headers=HEADERS, timeout=30) r.encoding = "utf-8" return r.text except Exception as e: log(f"Detail error: {e}") return None def parse_detail(html): result = {"title": "", "publish_date": "", "content": ""} soup = BeautifulSoup(html, "html.parser") # Title title_tag = soup.find("title") if title_tag: t = title_tag.get_text(strip=True) t = re.sub(r'[_\-—]\s*部门文件\s*[_\-—]\s*江门市广海湾经济开发区管理委员会\s*$', '', t) t = re.sub(r'[_\-—]\s*江门市广海湾经济开发区管理委员会\s*$', '', t) result["title"] = t.strip() # Date from meta PubDate meta_date = soup.find("meta", attrs={"name": "PubDate"}) if meta_date and meta_date.get("content"): result["publish_date"] = meta_date["content"][:10] # Content from content_div content_div = soup.find("div", class_="content_div") if not content_div: content_div = soup.find("div", class_=lambda c: c and "content" in (c or "")) if content_div: result["content"] = extract_clean_text(str(content_div)) else: body = soup.find("body") if body: result["content"] = body.get_text(separator="\n", strip=True)[:2000] return result def extract_clean_text(content_html): if not content_html: return "" soup = BeautifulSoup(content_html, "html.parser") for a_tag in soup.find_all("a", href=True): href = a_tag.get("href", "") text = a_tag.get_text(strip=True) or "附件" full_url = urljoin(BASE_URL, href) a_tag.replace_with(f"[{text}]({full_url})") for tag in soup.find_all(['span', 'b', 'strong', 'font', 'em', 'i', 'u', 's']): tag.unwrap() parts = [] for el in soup.find_all(['table', 'p']): if el.name == 'table': parts.append(str(el)) elif el.name == 'p' and not el.find_parent('table'): t = el.get_text(separator='', strip=True) if t: parts.append(t) return '\n\n'.join(parts) if parts else content_html.strip() def crawl_all(months_back=36): cutoff = (datetime.now() - timedelta(days=months_back * 30)).strftime("%Y-%m-%d") log(f"Full crawl (cutoff: {cutoff})") data = fetch_article_list() if not data: return articles = data.get("articles", []) log(f"Total articles: {len(articles)}") enriched = [] for idx, art in enumerate(articles, 1): created = (art.get("created_at", "") or "")[:10] if created < cutoff: log(f" Skip {art['title'][:40]}... ({created})") continue detail_url = art.get("url", "") html = fetch_detail(detail_url) if html: detail = parse_detail(html) title = detail["title"] or art.get("title", "") pub_date = detail["publish_date"] or created content = detail["content"] else: title = art.get("title", "") pub_date = created content = "" enriched.append({ "title": title, "page_url": detail_url, "publish_date": pub_date, "content": content, "site_name": f"{SITE}-{COLUMN}", "column": COLUMN, }) log(f" {idx}/{len(articles)}: {title[:40]}...") time.sleep(0.5) stored = 0 skipped = 0 import sqlite3 conn = sqlite3.connect(DB_PATH, timeout=60) conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA busy_timeout=10000") c = conn.cursor() for item in enriched: try: c.execute( "INSERT OR IGNORE INTO gov_raw " "(site_name, source_url, page_url, title, publish_date, content, summary, category)" " VALUES (?,?,?,?,?,?,?,?)", (item["site_name"], item["page_url"], item["page_url"], item["title"], item["publish_date"], item["content"], (item["content"] or "")[:500], item["column"])) if c.rowcount > 0: stored += 1 else: skipped += 1 except Exception as e: log(f"DB error: {e}") skipped += 1 conn.commit() conn.close() log(f"Result: {stored} new, {skipped} skipped") def crawl_incremental(): cutoff = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d") log(f"Incremental (since {cutoff})") data = fetch_article_list() if not data: return articles = data.get("articles", []) enriched = [] for art in articles: created = (art.get("created_at", "") or "")[:10] if created < cutoff: continue detail_url = art.get("url", "") html = fetch_detail(detail_url) detail = parse_detail(html) if html else {"title": "", "publish_date": "", "content": ""} enriched.append({ "title": detail["title"] or art.get("title", ""), "page_url": detail_url, "publish_date": detail["publish_date"] or created, "content": detail["content"], "site_name": f"{SITE}-{COLUMN}", "column": COLUMN, }) time.sleep(0.5) stored = 0 skipped = 0 import sqlite3 conn = sqlite3.connect(DB_PATH, timeout=60) conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA busy_timeout=10000") c = conn.cursor() for item in enriched: try: c.execute( "INSERT OR IGNORE INTO gov_raw " "(site_name, source_url, page_url, title, publish_date, content, summary, category)" " VALUES (?,?,?,?,?,?,?,?)", (item["site_name"], item["page_url"], item["page_url"], item["title"], item["publish_date"], item["content"], (item["content"] or "")[:500], item["column"])) if c.rowcount > 0: stored += 1 else: skipped += 1 except Exception as e: log(f"DB error: {e}") skipped += 1 conn.commit() conn.close() log(f"Result: {stored} new, {skipped} skipped") if __name__ == "__main__": mode = sys.argv[1] if len(sys.argv) > 1 else "incremental" if mode == "incremental": crawl_incremental() elif mode == "full": crawl_all() elif mode == "list": data = fetch_article_list() if data: for art in data.get("articles", [])[:5]: print(f" {art.get('created_at','')[:10]} | {art.get('title','')[:60]}")