#!/usr/bin/env python3 """江苏扬农化工集团有限公司 - 环保信息公开""" import re, sys, os, json import urllib.request, urllib.error from bs4 import BeautifulSoup import sqlite3 BASE_URL = "http://www.yangnong.com.cn" LIST_URL = BASE_URL + "/yangnong/kcxfz/hbxxgk/A142006003Gone{}.html" SITE_NAME = "扬农化工-环保信息公开" SITE_DISPLAY = "江苏扬农化工集团有限公司" GROUP_NAME = "环评平台" DB = os.environ.get("SEARCH_DB", "/root/search.db") HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"} _MAX_PG = None for i, a in enumerate(sys.argv): if a == "--pages" and i + 1 < len(sys.argv): _MAX_PG = int(sys.argv[i + 1]) break def fetch(url): req = urllib.request.Request(url, headers=HEADERS) try: return urllib.request.urlopen(req, timeout=30).read().decode("utf-8", errors="replace") except Exception as e: print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr) return "" def extract_items(html): soup = BeautifulSoup(html, "html.parser") items = [] for li in soup.select("li.list.top"): zi_a = li.select_one("div.zi > a[href]") if not zi_a: continue href = zi_a["href"].strip() title_a = li.find("a", class_="news-title") title = title_a.get("title", "") or title_a.get_text(strip=True) if title_a else "" if not title: title = zi_a.get_text(strip=True) time_div = li.select_one("div.time") date_text = "" if time_div: day = time_div.find("p") year = time_div.find("i") d = day.get_text(strip=True) if day else "" y = year.get_text(strip=True) if year else "" if d and y: date_text = f"{y}-{d}" if href and not href.startswith("http"): href = BASE_URL + href items.append({"title": title.strip(), "url": href, "date": date_text}) return items def get_total_pages(html): soup = BeautifulSoup(html, "html.parser") max_page = 1 for a in soup.select("div.fy a[href]"): m = re.search(r'Gone(\d+)\.html', a["href"]) if m: p = int(m.group(1)) if p > max_page: max_page = p tp = soup.select_one("div.tp") if tp: m = re.search(r'/(\d+)', tp.get_text()) if m: max_page = int(m.group(1)) return max_page def parse_detail(html, url): soup = BeautifulSoup(html, "html.parser") title = "" h1 = soup.find("h1", id="Title") if h1: title = h1.get_text(strip=True) if not title and soup.title: title = soup.title.string.strip() title = re.sub(r"_环保信息公开_江苏扬农化工集团有限公司.*", "", title).strip() date_text = "" time_span = soup.find("span", id="PublishTime") if time_span: m = re.search(r"(\d{4}-\d{2}-\d{2})", time_span.get_text()) if m: date_text = m.group(1) body = soup.find("div", id="BodyLabel") if not body: cd = soup.find("div", id="Content") if cd: body = cd.find("div", id="BodyLabel") if not body: print(f"[WARN] No content found: {url}", file=sys.stderr) return {"title": title, "date": date_text, "content": "", "attachments": []} attachments = [] parts = [] for el in body.find_all(["p", "table", "img"], recursive=True): if el.name == "p" and el.find_parent("table"): continue if el.name == "table" and el.find_parent("table"): continue if el.name == "p": text = el.get_text(" ", strip=True) for a in el.find_all("a", href=True): href = a["href"].strip() if re.search(r'\.(pdf|docx?|xlsx?|zip|rar)$', href, re.I): atitle = a.get_text(strip=True) or a.get("download", "") ahref = BASE_URL + href if href.startswith("/") else (url.rsplit("/", 1)[0] + "/" + href if not href.startswith("http") else href) if not any(att["url"] == ahref for att in attachments): attachments.append({"title": atitle, "url": ahref}) if text: parts.append(text) elif el.name == "table": rows = [] for tr in el.find_all("tr"): cols = [] for cell in tr.find_all(["td", "th"]): cols.append(cell.get_text(" ", strip=True)) if cols: rows.append("| " + " | ".join(cols) + " |") if rows: cc = len(rows[0].split("|")) - 2 rows.insert(1, "| " + " | ".join(["---"] * cc) + " |") parts.append("\n".join(rows)) elif el.name == "img": src = el.get("src", "") alt = el.get("alt", "") if src: src = BASE_URL + src if src.startswith("/") else (url.rsplit("/", 1)[0] + "/" + src if not src.startswith("http") else src) parts.append(f"![{alt}]({src})" if alt else f"![]({src})") seen = set() ua = [] for att in attachments: if att["url"] not in seen: seen.add(att["url"]) ua.append(att) return {"title": title, "date": date_text, "content": "\n\n".join(parts), "attachments": ua} def main(): html = fetch(LIST_URL.format(1)) if not html: print("[ERROR] Cannot fetch list page", file=sys.stderr) sys.exit(1) total_pages = get_total_pages(html) print(f"[INFO] Total pages: {total_pages}", file=sys.stderr) if _MAX_PG and total_pages > _MAX_PG: print(f"[INFO] Limiting to {_MAX_PG} pages (--pages={_MAX_PG})", file=sys.stderr) total_pages = _MAX_PG all_items = [] for pg in range(1, total_pages + 1): html = fetch(LIST_URL.format(pg)) if not html: continue all_items.extend(extract_items(html)) print(f"[INFO] Total items from list: {len(all_items)}", file=sys.stderr) conn = sqlite3.connect(DB, timeout=60) conn.execute("PRAGMA busy_timeout = 60000") c = conn.cursor() new_count = empty_content = total_attachments = 0 for item in all_items: c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],)) if c.fetchone(): continue html = fetch(item["url"]) if not html: continue detail = parse_detail(html, item["url"]) title = detail["title"] or item["title"] date_text = detail["date"] or item["date"] content = detail["content"] attachments = detail["attachments"] plain = re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', '', content)).strip() summary = plain[:200] if not content: empty_content += 1 date_rank = 0 if date_text: m = re.search(r"(\d{4})-(\d{2})-(\d{2})", date_text) if m: date_rank = int(m.group(1) + m.group(2) + m.group(3)) att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]" try: c.execute("INSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank) VALUES (?,?,?,?,?,?,?,?,?)", (title, SITE_NAME, GROUP_NAME, item["url"], date_text, content, summary, att_json, date_rank)) if c.rowcount > 0: new_count += 1 total_attachments += len(attachments) except Exception as e: print(f"[ERROR] DB insert failed: {title[:30]} - {e}", file=sys.stderr) conn.commit() conn.close() print(f"\n[RESULT] {SITE_NAME}: {new_count} new, {empty_content} empty content, {total_attachments} attachments") if __name__ == "__main__": main()