#!/usr/bin/env python3
"""
藁城区人民政府 - 公告公示
https://www.gc.gov.cn/columns/f144c2a8-8f30-4b5d-85a7-f8054f1d8a22/index.html
JPAAS CMS, 静态分页 index_{N}.html (或 index.html), 详情 id=conN
注意: conN 内容可能为纯图片（征地公告扫描件）或文本（环评公示）
"""

import urllib.request, urllib.parse, json, re, sys, ssl, time, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
from bs4 import BeautifulSoup

BASE = "https://www.gc.gov.cn"
COL_ID = "f144c2a8-8f30-4b5d-85a7-f8054f1d8a22"
WEB_ID = "203"
TOTAL_PAGES = 419
MAX_PAGES = 5
SITE_NAME = "藁城区人民政府-公告公示"

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

# === 自动页数控制 ===
_MAX_PG = None
for i, a in enumerate(sys.argv):
    if a == '--pages' and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break
if _MAX_PG is not None:
    print("[AutoPg] max_pages=" + str(_MAX_PG))
elif len(sys.argv) > 1 and sys.argv[1].isdigit():
    _MAX_PG = int(sys.argv[1])

if _MAX_PG is not None:
    MAX_PAGES = _MAX_PG


def fetch(url):
    req = urllib.request.Request(url, headers=HEADERS)
    resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
    return resp.read().decode("utf-8", errors="replace")


def extract_items(html):
    """Extract list items from static HTML"""
    items = []
    for m in re.finditer(
        r"<li[^>]*>\s*<a\s+href='([^']+)'[^>]*title='([^']*)'[^>]*>.*?</a>\s*<span[^>]*>(\d{4}-\d{2}-\d{2})</span>\s*</li>",
        html, re.DOTALL
    ):
        url = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()
        items.append({"url": url, "title": title, "pub_date": date})
    return items


def fetch_api_page(page):
    """Try JPAAS API for page N (fallback if static page fails)"""
    api_url = BASE + "/api-gateway/jpaas-publish-server/front/page/build/unit"
    params = urllib.parse.urlencode({
        "pageSize": "15",
        "pageNo": str(page),
        "webId": WEB_ID,
        "colId": COL_ID,
        "appUrl": BASE,
        "paramJson": json.dumps({"pageNo": page, "pageSize": 15}, ensure_ascii=False),
        "parseType": "bulidstatic",
        "pageType": "column",
    }).encode()
    req = urllib.request.Request(api_url, data=params,
        headers={**HEADERS, "Content-Type": "application/x-www-form-urlencoded"})
    resp = urllib.request.urlopen(req, timeout=15, context=ssl_ctx)
    return resp.read().decode("utf-8", errors="replace")


def extract_con_content(con_div, detail_url):
    """提取 conN 的内容（text + img + table + pdf link）"""
    parts = []
    
    for el in con_div.children:
        if not el.name:
            continue
        
        if el.name == "p":
            txt = el.get_text(" ", strip=True)
            if txt and len(txt) > 3:
                parts.append(txt)
                parts.append("\n\n")
            # 图片
            for img in el.find_all("img"):
                src = img.get("src", "")
                if src:
                    full = src if src.startswith("http") else BASE + "/" + src.lstrip("/")
                    alt = img.get("alt", "")
                    parts.append("![{}]({})".format(alt, full))
                    parts.append("\n\n")
            # 附件链接
            for a in el.find_all("a", href=True):
                href = a["href"]
                if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", href, re.I):
                    full = href if href.startswith("http") else BASE + "/" + href.lstrip("/")
                    parts.append("[{}]({})".format(a.get_text(strip=True) or "附件", full))
                    parts.append("\n\n")
        
        elif el.name == "div":
            # 递归检查 div 内的内容
            txt = el.get_text(" ", strip=True)
            if txt and len(txt) > 3:
                parts.append(txt)
                parts.append("\n\n")
            for img in el.find_all("img"):
                src = img.get("src", "")
                if src:
                    full = src if src.startswith("http") else BASE + "/" + src.lstrip("/")
                    parts.append("![]({})".format(full))
                    parts.append("\n\n")
        
        elif el.name == "img":
            src = el.get("src", "")
            if src:
                full = src if src.startswith("http") else BASE + "/" + src.lstrip("/")
                parts.append("![]({})".format(full))
                parts.append("\n\n")
        
        elif el.name == "table":
            parts.append(str(el))
            parts.append("\n\n")
    
    result = "".join(parts).strip()
    result = re.sub(r"\n{4,}", "\n\n", result)
    return result


def get_detail(detail_url):
    """Fetch detail page for full title, date, content"""
    try:
        html = fetch(detail_url)
    except:
        return None, None, None, []

    # Title
    title = None
    m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
    if m:
        title = re.sub(r"\s+", " ", m.group(1)).strip()
    if not title:
        m = re.search(r"<h1[^>]*>(.*?)</h1>", html, re.DOTALL)
        if m:
            title = re.sub(r"\s+", " ", m.group(1)).strip()

    # Date
    pub_date = None
    m = re.search(r'<meta\s+name="PubDate"\s+content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        pub_date = m.group(1)
    if not pub_date:
        m = re.search(r"发布时间[：:]\s*(\d{4}-\d{2}-\d{2})", html)
        if m:
            pub_date = m.group(1)

    # Content — 优先 conN
    content = ""
    soup = BeautifulSoup(html, "html.parser")
    con_div = soup.find("div", id="conN")
    if con_div:
        content = extract_con_content(con_div, detail_url)

    # Attachments (从全文提取)
    atts = []
    for m in re.finditer(r'<a\s+[^>]*href="([^"]*\.(?:doc|docx|pdf|xls|xlsx|zip|rar))"[^>]*>([^<]*)</a>', html, re.I):
        href = m.group(1)
        if href.startswith("http"):
            full_url = href
        elif href.startswith("/"):
            full_url = BASE + href
        else:
            full_url = detail_url.rsplit("/", 1)[0] + "/" + href
        atts.append({"name": m.group(2).strip(), "url": full_url})

    return title, pub_date, content, atts


def main():
    pages_to_fetch = min(TOTAL_PAGES, MAX_PAGES)
    
    print("=== {} ===".format(SITE_NAME))
    print("  总页数: {}, 本次: {}".format(TOTAL_PAGES, pages_to_fetch))

    all_records = []
    empty = 0

    for page in range(1, pages_to_fetch + 1):
        if page == 1:
            url = "{}/columns/{}/index.html".format(BASE, COL_ID)
        else:
            url = "{}/columns/{}/index_{}.html".format(BASE, COL_ID, page)

        print("  第 {}/{} 页: {}".format(page, pages_to_fetch, url))
        
        try:
            html = fetch(url)
        except Exception as e:
            print("    获取失败: {}, 尝试API...".format(e))
            try:
                html = fetch_api_page(page)
            except:
                print("    也无法获取API, 停止")
                continue

        items = extract_items(html)
        if not items:
            try:
                api_result = fetch_api_page(page)
                items = extract_items(api_result)
            except:
                pass

        if not items:
            print("    空列表, 停止")
            break

        print("    找到 {} 条".format(len(items)))

        for item in items:
            time.sleep(0.3)
            title, pub_date, content, atts = get_detail(item["url"])
            if title:
                item["title"] = title
            if pub_date:
                item["pub_date"] = pub_date
            item["content"] = content or ""
            item["attachments"] = atts
            item["site_name"] = SITE_NAME
            if not content or len(content) < 20:
                empty += 1
            all_records.append(item)
            print("    + {}".format(item["title"][:50]))

        print("  累计: {}".format(len(all_records)))

    print("\n  列表总计: {}, 入库准备: {}, 空正文: {}".format(len(all_records), len(all_records), empty))

    if all_records:
        results = []
        for item in all_records:
            summary = re.sub(r'<[^>]+>', '', item.get("content", ""))[:200].strip()
            if not summary:
                summary = item["title"][:200]
            results.append({
                "site_name": SITE_NAME,
                "title": item["title"],
                "url": item["url"],
                "content": item.get("content", ""),
                "summary": summary,
                "pub_date": item.get("pub_date", ""),
                "source_url": item["url"],
            })
        push_to_searchdb(results, "gc_gggs")
    
    print("完成")


if __name__ == "__main__":
    main()