#!/usr/bin/env python3
"""
莆田市生态环境局-公示公告 爬虫 (avalon 动态列表, playwright 渲染)
列表: https://sthjj.putian.gov.cn/xxgk/gsgg/ (avalon ms-controller, 静态页仅第一条)
分页: 点击下一页按钮 / pgBar_form 模板
详情: TRS 静态页 t20YYYYMMDD_ID.htm, div.view/content 容器
"""
import os, sys, re, time, json, asyncio
from urllib.parse import urljoin
from bs4 import BeautifulSoup

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE_URL = "https://sthjj.putian.gov.cn"
LIST_URL = BASE_URL + "/xxgk/gsgg/"
SITE_NAME = "莆田市生态环境局-公示公告"
MAX_PAGES = 60
CUTOFF = "2015-01-01"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"

def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return None

def clean_title(title):
    import html as html_mod
    title = html_mod.unescape(title or "")
    title = re.sub(r"·| ", "", title)
    title = re.sub(r"[\u200b\ufeff]", "", title)
    title = re.sub(r"^[•·]\s*", "", title)
    return title.strip()

async def main():
    max_pages = parse_pages_arg()
    if max_pages is None:
        max_pages = MAX_PAGES
    print(f"[AutoPg] max_pages={max_pages} (SITE={SITE_NAME})")

    from playwright.async_api import async_playwright
    all_items = []
    seen = set()

    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled", "--no-sandbox"])
        ctx = await browser.new_context(user_agent=UA, locale="zh-CN")
        page = await ctx.new_page()

        for pg in range(1, max_pages + 1):
            if pg == 1:
                await page.goto(LIST_URL, wait_until="domcontentloaded", timeout=45000)
            else:
                # avalon 分页: 点数字按钮 (href="#")
                try:
                    clicked = await page.evaluate("""(pg) => {
                        const links = document.querySelectorAll('.pgStyle_green a[href="#"]');
                        for (const a of links) {
                            if (a.textContent.trim() === String(pg)) { a.click(); return true; }
                        }
                        return false;
                    }""", pg)
                    if not clicked:
                        print(f"  Page {pg}: 无数字按钮, 停止")
                        break
                except Exception as e:
                    print(f"  Page {pg}: 翻页失败 {e}")
                    break
            await page.wait_for_timeout(3000)

            items = await page.evaluate("""() => {
                const out = [];
                const links = document.querySelectorAll('a[href*="t20"], a[href*="t19"], a[href*="t18"]');
                const seen = new Set();
                links.forEach(a => {
                    const href = a.href.split('?')[0];
                    if (!seen.has(href) && a.textContent.trim().length > 4) {
                        seen.add(href);
                        let li = a.closest('li');
                        let date = '';
                        if (li) {
                            const sp = li.querySelector('span.rt, span.date, span.time');
                            if (sp) date = sp.textContent.trim();
                        }
                        if (!date) {
                            const m = href.match(/t(\\d{8})_/);
                            if (m) { const d = m[1]; date = d.slice(0,4)+'-'+d.slice(4,6)+'-'+d.slice(6,8); }
                        }
                        out.push({url: href, title: a.textContent.trim(), date});
                    }
                });
                return out;
            }""")
            new_items = [x for x in items if x["url"] not in seen]
            for x in new_items:
                seen.add(x["url"])
            all_items.extend(new_items)
            print(f"  Page {pg}: {len(items)}条(新{len(new_items)}) 累计{len(all_items)}")
            if new_items:
                dates = [x["date"] for x in new_items if x["date"]]
                if dates and min(dates) < CUTOFF:
                    print(f"  已达 CUTOFF {CUTOFF}")
                    break
            if not new_items:
                # 可能翻页后内容没变 -> 停止
                print("  无新增, 停止")
                break
            time.sleep(1)

        print(f"\n共 {len(all_items)} 条, 抓详情...")
        results = []
        for i, item in enumerate(all_items):
            try:
                await page.goto(item["url"], wait_until="domcontentloaded", timeout=45000)
                await page.wait_for_timeout(2000)
                html = await page.content()
            except Exception as e:
                print(f"  [WARN] detail {item['url'][:50]}: {e}")
                results.append({"site_name": SITE_NAME, "title": item["title"], "pub_date": item["date"], "content": "", "source_url": item["url"], "url": item["url"], "category": "公示公告", "attachments": ""})
                continue

            soup = BeautifulSoup(html, "html.parser")
            title = ""
            h1 = soup.find("h1")
            if h1:
                title = clean_title(h1.get_text(strip=True))
            if not title:
                m = re.search(r'<meta[^>]*name=["\']ArticleTitle["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
                if m:
                    title = clean_title(m.group(1))
            if not title:
                t = soup.find("title")
                if t:
                    title = clean_title(t.get_text(strip=True).split("_")[0])

            date = ""
            m = re.search(r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\'](\d{4}-\d{2}-\d{2})', html, re.I)
            if m:
                date = m.group(1)
            if not date:
                for kw in ["发布时间", "发布日期"]:
                    m = re.search(r'%s[：:]\s*(\d{4}-\d{2}-\d{2})' % kw, html)
                    if m:
                        date = m.group(1)
                        break

            body_el = None
            for sel in ["div.view", "div.content", "#zoom", "div.article", "div.xl_text"]:
                body_el = soup.select_one(sel)
                if body_el and body_el.get_text(strip=True):
                    break
            content = ""
            attachments = []
            if body_el:
                for a in body_el.find_all("a", href=True):
                    href = a["href"].strip()
                    if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|wps)$', href.lower()) or "download" in href.lower():
                        full = urljoin(item["url"], href)
                        name = a.get_text(strip=True) or href.split("/")[-1]
                        attachments.append({"name": name, "url": full})
                parts = []
                for el in body_el.find_all(["p", "table", "img"]):
                    if el.name == "p":
                        if el.find_parent("table") or el.find("table") or el.find("p"):
                            continue
                        for a in el.find_all("a", href=True):
                            href = a["href"].strip()
                            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|wps)$', href.lower()) or "download" in href.lower():
                                a["href"] = urljoin(item["url"], href)
                        txt = re.sub(r"\s+", " ", el.get_text(" ", strip=True))
                        if txt:
                            parts.append(f"<p>{txt}</p>")
                    elif el.name == "table":
                        parts.append(str(el))
                    elif el.name == "img":
                        src = el.get("src", "")
                        if src:
                            parts.append(f'<p><img src="{urljoin(item["url"], src)}"></p>')
                for att in attachments:
                    parts.append(f'<p><a href="{att["url"]}">{att["name"]}</a></p>')
                content = "\n".join(parts)
                if not content:
                    txt = body_el.get_text(" ", strip=True)
                    if txt:
                        content = f"<p>{txt}</p>"

            if not title:
                title = item["title"]
            if not date:
                date = item["date"]
            results.append({"site_name": SITE_NAME, "title": title, "pub_date": date, "content": content,
                            "source_url": item["url"], "url": item["url"], "category": "公示公告",
                            "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else ""})
            if (i + 1) % 20 == 0:
                print(f"  [{i+1}/{len(all_items)}] {title[:40]}")
            time.sleep(0.4)

        await browser.close()

    push_to_searchdb(results, batch_label=SITE_NAME)
    dates = [r["pub_date"] for r in results if r["pub_date"]]
    print(f"\n站点: {SITE_NAME} 采集: {len(results)} 日期: {min(dates) if dates else '-'} ~ {max(dates) if dates else '-'}")

if __name__ == "__main__":
    asyncio.run(main())