#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""crawl_newpower.py - 山东新势立生物科技-公司新闻(含化工环评公示) (www.new-power.com.cn)
站点: 静态HTML, 无WAF
入口: /news/15184693.html (公司新闻栏目页, 含相关新闻列表 news_1/{N}.html)
详情: /news_1/{N}.html (环评公示/公司新闻, 含建设地点/环评单位等)
运行: python3 crawl_newpower.py [--pages=N] [--dryrun]
"""
import sys, os, re, json, time, requests, sqlite3
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import urllib3
urllib3.disable_warnings()

SITE_NAME = "德州新势立-公司新闻"
BASE = "https://www.new-power.com.cn"
ENTRY_URL = "https://www.new-power.com.cn/news/15184693.html"
MAX_PAGES_DEFAULT = 5
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
OUT = "/tmp/newpower.jsonl"
DB_PATH = "/root/search.db"
SCRIPT_NAME = "crawl_newpower.py"


def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
    return MAX_PAGES_DEFAULT


def fetch(url):
    try:
        r = requests.get(url, headers={"User-Agent": UA}, timeout=20, verify=False)
        r.encoding = "utf-8"
        return r.text
    except Exception:
        return ""


def list_items():
    """从栏目页提取文章列表 (news_1/{N}.html)"""
    html = fetch(ENTRY_URL)
    if not html:
        return []
    items = []
    seen = set()
    for m in re.finditer(r'href="(/news_1/(\d+)\.html)"[^>]*>\s*([^<]{8,120})', html):
        href, id_, title = m.group(1), m.group(2), m.group(3).strip()
        title = re.sub(r"\s+", " ", title).strip()
        if href in seen or len(title) < 8:
            continue
        seen.add(href)
        items.append({"title": title, "url": BASE + href, "id": id_})
    return items


def parse_detail(url):
    html = fetch(url)
    if not html:
        return "", "", ""
    title = ""
    m = re.search(r"<title>([^<]*?)(?:\s*[-_]\s*山东新势立生物科技有限公司)?</title>", html, re.S)
    if m:
        title = re.sub(r"<[^>]+>", "", m.group(1)).strip()
    date_text = ""
    m = re.search(r"(\d{4}-\d{2}-\d{2})", html)
    if m:
        date_text = m.group(1)
    soup = BeautifulSoup(html, "html.parser")
    content = ""
    # 正文容器: e_richText-11 (文章正文, 在日期之后) 优先; e_richText-33 是页头"新闻中心"
    zoom = None
    for sel in ["div.e_richText-11", "div[class*='richText-11']"]:
        zoom = soup.select_one(sel)
        if zoom:
            break
    if not zoom:
        for div in soup.find_all("div"):
            txt = div.get_text(" ", strip=True)
            if len(txt) > 200 and ("公示" in txt or "环境影响评价" in txt or "建设地点" in txt):
                zoom = div
                break
    if zoom:
        parts = []
        for el in zoom.find_all(["p", "pre", "table"], recursive=True):
            if el.name in ("p", "pre") and el.find_parent("table"):
                continue
            if el.name == "table":
                parts.append(str(el))
                continue
            txt = el.get_text(" ", strip=True)
            txt = re.sub(r"&nbsp;", " ", txt)
            if txt and not re.match(r"^(责任编辑|初审|复审|终审|分享|打印)", txt):
                parts.append(txt)
        dedup = []
        for p in parts:
            if p and (not dedup or dedup[-1] != p):
                dedup.append(p)
        content = "\n\n".join(dedup)
    return title, date_text, content


def main():
    max_pages = parse_pages_arg()
    dryrun = "--dryrun" in sys.argv
    print(f"[newpower] pages={max_pages} cutoff={CUTOFF}", flush=True)
    items = list_items()
    print(f"[列表] {len(items)} 篇文章", flush=True)
    results = []
    for it in items[:max_pages * 20]:
        try:
            title, date_text, content = parse_detail(it["url"])
            if date_text and date_text < CUTOFF:
                continue
            results.append({"url": it["url"], "title": title or it["title"], "date": date_text, "content": content})
            print(f"  ✓ {results[-1]['title'][:45]} | {date_text}", flush=True)
        except Exception as e:
            print(f"  ✗ {str(e)[:60]}", flush=True)
        time.sleep(0.3)
    print(f"[完成] {len(results)} 条", flush=True)
    if dryrun:
        for r in results[:3]:
            print(json.dumps(r, ensure_ascii=False)[:250])
        return
    with open(OUT, "w", encoding="utf-8") as f:
        for r in results:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    if os.path.exists(DB_PATH):
        conn = sqlite3.connect(DB_PATH, timeout=120)
        conn.execute("PRAGMA busy_timeout=120000")
        c = conn.cursor()
        added = 0
        for r in results:
            c.execute("SELECT id FROM gov_raw WHERE page_url=?", (r["url"],))
            if c.fetchone():
                continue
            c.execute(
                "INSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank, script_name) VALUES (?,?,?,?,?,?,?,?,?,?)",
                (r["title"], SITE_NAME, "山东", r["url"], r["date"], r["content"], "", "",
                 int(r["date"].replace("-", "")) if re.match(r"\d{4}-\d{2}-\d{2}", r["date"]) else 0, SCRIPT_NAME))
            if c.rowcount:
                added += 1
        conn.commit()
        conn.close()
        print(f"[入库] 新增 {added}", flush=True)


if __name__ == "__main__":
    main()