#!/usr/bin/env python3
"""利津县人民政府-公告公示 爬虫
CMS: 山东省信息公开平台 (AJAX动态加载列表)
列表: Playwright 爬取5页, 分页 funGoPage('/module/xxgk/search.jsp', N)
详情: /art/YYYY/M/D/art_{columnId}_{articleId}.html, 容器 div#zoom
"""
import os, re, sys, json, time, asyncio, sqlite3
from bs4 import BeautifulSoup
from urllib import request
from urllib.parse import urljoin
import ssl

SITE_NAME = "利津县人民政府-公告公示"
BASE_URL = "http://www.lijin.gov.cn"
LIST_URL = "http://www.lijin.gov.cn/col/col9113/index.html?vc_xxgkarea=12370522K21505011R&number=L2601&jh=263"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
DELAY = 1.5
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
}
ssl._create_default_https_context = ssl._create_unverified_context


def fetch(url, timeout=30):
    req = request.Request(url, headers=HEADERS)
    try:
        resp = request.urlopen(req, timeout=timeout)
        return resp.read().decode("utf-8", errors="ignore")
    except Exception as e:
        print("  [ERROR] %s: %s" % (url, e), file=sys.stderr)
        return None


def parse_detail(html, url):
    """提取正文和附件"""
    parts = []
    attachments = []

    # 正文容器 div#zoom
    m = re.search(r'<div\s+id="zoom"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        zoom_html = m.group(1)
        for p_html in re.findall(r'<p[^>]*>(.*?)</p>', zoom_html, re.DOTALL):
            # 移除HTML注释
            p_text = re.sub(r"<!--.*?-->", "", p_html, flags=re.DOTALL)
            text = re.sub(r"<[^>]+>", "", p_text).strip()
            text = text.replace("&nbsp;", " ")
            text = re.sub(r"\s+", " ", text)
            if text:
                parts.append(text)
        # 表格
        for t_html in re.findall(r'<table[^>]*>.*?</table>', zoom_html, re.DOTALL):
            md = html_table_to_html(t_html)
            if md:
                parts.append(md)

    # 附件
    for a_href, a_text in re.findall(
        r'<a[^>]*href="([^"]+\.(?:docx?|pdf|xlsx?|rar|zip|wps|et))"[^>]*>([^<]+)</a>',
        html, re.I
    ):
        full = urljoin(url, a_href)
        attachments.append({"title": a_text.strip(), "url": full})

    content = "\n\n".join(parts)
    # 清理多余空行
    content = re.sub(r"\n{3,}", "\n\n", content)
    return content, attachments


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def clean_html(text):
    """去除HTML标签和注释"""
    text = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL)
    text = re.sub(r"<[^>]+>", " ", text)
    text = text.replace("&nbsp;", " ")
    text = re.sub(r"\s+", " ", text)
    return text.strip()

def get_detail_title(html):
    m = re.search(r'<div\s+class="biaoti"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        return clean_html(m.group(1))
    return None


async def get_list_items(max_pages=5):
    """用 Playwright 获取列表页文章URL"""
    from playwright.async_api import async_playwright

    all_items = []  # (title, url, date)

    async with async_playwright() as pw:
        browser = await pw.chromium.launch(headless=True)
        page = await browser.new_page(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/126.0.0.0 Safari/537.36")
        await page.goto(LIST_URL, timeout=45000, wait_until="commit")
        await asyncio.sleep(12)

        for pg in range(1, max_pages + 1):
            print("[Playwright] 第%d页" % pg, flush=True)

            # 提取当前页文章
            items = await page.evaluate("""() => {
                const links = document.querySelectorAll('a[href*="/art/"]');
                return Array.from(links).map(a => {
                    const parent = a.closest('li') || a.parentElement;
                    const dateEl = parent ? parent.querySelector('span') : null;
                    const date = dateEl ? dateEl.textContent.trim() : '';
                    const parts = a.href.split('/');
                    const dateMatch = a.href.match(/art\\/(\\d{4})\\/(\\d{1,2})\\/(\\d{1,2})/);
                    const pubDate = dateMatch ? dateMatch[1]+'-'+dateMatch[2].padStart(2,'0')+'-'+dateMatch[3].padStart(2,'0') : date;
                    return {
                        title: a.textContent.trim(),
                        url: a.href,
                        date: pubDate
                    };
                });
            }""")

            for item in items:
                if item["title"] and len(item["title"]) > 5:
                    all_items.append((item["title"], item["url"], item["date"]))

            print("  本页 %d 条, 累计 %d 条" % (len(items), len(all_items)), flush=True)

            # 点击下一页
            if pg < max_pages:
                next_btn = await page.query_selector('a:has-text("\u4e0b\u4e00\u9875")')
                if next_btn:
                    await next_btn.click()
                    await asyncio.sleep(4)
                else:
                    print("  [完成] 无下一页按钮", flush=True)
                    break

        await browser.close()

    return all_items


def crawl(max_pages=5):
    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()

    # 1. 获取列表
    items = asyncio.run(get_list_items(max_pages))
    if not items:
        print("[FAIL] 未获取到任何列表项", flush=True)
        return

    print("\n[总共] 获取 %d 条列表项" % len(items), flush=True)

    new_count = 0
    skip_count = 0
    error_count = 0

    for title, detail_url, date in items:
        cur.execute("SELECT id, content FROM gov_raw WHERE page_url=? AND site_name=?",
                   (detail_url, SITE_NAME))
        row = cur.fetchone()

        if row and row[1] and len(row[1]) > 10:
            skip_count += 1
            continue

        time.sleep(DELAY)

        detail_html = fetch(detail_url)
        if not detail_html:
            print("  [跳过] 详情页: %s" % title[:30], flush=True)
            error_count += 1
            continue

        content, attachments = parse_detail(detail_html, detail_url)
        attachments_json = json.dumps(attachments, ensure_ascii=False)

        detail_title = get_detail_title(detail_html)
        if detail_title:
            title = detail_title

        if content:
            summary = content[:200]
            print("  [正文] %s -> %d chars, %d附件" %
                  (title[:30], len(content), len(attachments)), flush=True)
        else:
            summary = title
            print("  [无文本] %s" % title[:30], flush=True)

        if row:
            cur.execute(
                "UPDATE gov_raw SET content=?, title=?, attachments=?, summary=? WHERE id=?",
                (content, title, attachments_json, summary, row[0])
            )
        else:
            cur.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, page_url, title, content, publish_date, attachments, summary)
                   VALUES (?, ?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, detail_url, title, content, date, attachments_json, summary)
            )

        if cur.rowcount > 0 or (row and True):
            new_count += 1
        else:
            skip_count += 1

        conn.commit()

    conn.close()
    print("\n[DONE] %s: 新增=%d, 跳过=%d, 错误=%d" %
          (SITE_NAME, new_count, skip_count, error_count), flush=True)


if __name__ == "__main__":
    mp = int(sys.argv[1]) if len(sys.argv) > 1 else 5
    crawl(mp)