#!/usr/bin/env python3
"""
南大港产业园区管理委员会 — 公告公示 (nandagang.gov.cn)
====================================================
列表: /nandagang/c100975/listDisplaySon.shtml
翻页: listDisplaySon_N.shtml (N=2..142, 10条/页)
详情: /nandagang/c100975/YYYYMM/uuid.shtml
正文: <div class="articlecon">
标题: <h1>
日期: <meta PubDate> (含时间组件，需截取)
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb, fetch_page, clean_html, parse_date, extract_date_from_html
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "南大港产业园区-公告公示"
BASE_URL = "http://www.nandagang.gov.cn"
LIST_MAIN = "http://www.nandagang.gov.cn/nandagang/c100975/listDisplaySon.shtml"
MAX_PAGES = 5  # 实测共10页（page11=404），JS中142是总数
ITEMS_PER_PAGE = 10
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")

def fetch(url):
    try:
        r = requests.get(url, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def parse_list(html):
    """<li><span class="date">YYYY-MM-DD</span><a href="..." target="_blank">Title</a></li>"""
    items = []
    for m in re.finditer(
        r'<li>\s*<span[^>]*class="date"[^>]*>(\d{4}-\d{2}-\d{2})</span>\s*<a[^>]*href="([^"]+)"[^>]*target="_blank"[^>]*>(.*?)</a>',
        html, re.DOTALL
    ):
        date = m.group(1).strip()
        href = m.group(2).strip()
        title = re.sub(r'<[^>]+>', '', m.group(3)).strip()
        if not href.startswith('http'):
            href = BASE_URL + href
        if title and date:
            items.append((title, href, date))
    return items

def fetch_detail(url):
    """提取详情页"""
    html = fetch(url)
    if not html:
        return "", "", ""

    # 标题 — h1（跳过站点logo的h1，取正文区的h1）
    title = ""
    # 先取正文区内的h1（在 div.detail 内）
    m = re.search(r'<div[^>]*class="detail"[^>]*>.*?<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
    if not m:
        # 兜底：取第二个h1
        h1s = re.findall(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
        if len(h1s) >= 2:
            title = h1s[1]
        elif h1s:
            title = h1s[0]
    else:
        title = m.group(1)
    title = re.sub(r'<[^>]+>', '', title).strip()

    # 日期 — <meta PubDate content="YYYY-MM-DD HH:MM:SS">
    pub_date = ""
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})', html, re.I)
    if m:
        pub_date = m.group(1)
    if not pub_date:
        # fallback: "发布日期：YYYY-MM-DD"
        m = re.search(r'发布日期[：:]\s*(\d{4}-\d{1,2}-\d{1,2})', html)
        if m:
            pub_date = m.group(1)

    # 正文 — <div class="articlecon"> (去掉内部的 share div)
    content = ""
    m = re.search(r'<div[^>]*class="articlecon"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        raw = m.group(1)
        # 去掉 share 等 UI 元素
        raw = re.sub(r'<div[^>]*class="share"[^>]*>.*?</div>', '', raw, flags=re.DOTALL)
        raw = re.sub(r'<!--.*?-->', '', raw, flags=re.DOTALL)
        raw = clean_html(raw)
        # 清理多余空行
        raw = re.sub(r'>\s+<', '><', raw)
        content = raw.strip()

    return title, pub_date, content


def run(args=None):
    incremental = False
    if args and '1' in args:
        incremental = True
    print(f"爬取: {SITE_NAME}")
    results = []

    pages_to_crawl = 1 if incremental else MAX_PAGES
    for page in range(1, pages_to_crawl + 1):
        if page == 1:
            page_url = LIST_MAIN
        else:
            page_url = f"http://www.nandagang.gov.cn/nandagang/c100975/listDisplaySon_{page}.shtml"

        page_html = fetch(page_url)
        if not page_html or len(page_html) < 500:
            print(f"  第{page}页: 无内容，停止翻页")
            break

        items = parse_list(page_html)
        if not items:
            print(f"  第{page}页: 0 条")
            if page > 1:
                break
            continue
        print(f"  第{page}页: {len(items)} 条")

        for i, (title, url, date) in enumerate(items):
            print(f"  [{i+1}/{len(items)}] {title[:35]}...", end=" ", flush=True)
            if date < CUTOFF:
                print("⏭ 超时范围")
                continue

            detail_title, pub_date, content = fetch_detail(url)
            if not content or len(content) < 50:
                print("⚠ 无正文")
                continue

            final_title = detail_title or title
            final_date = pub_date or date

            summary = re.sub(r'<[^>]+>', '', content)[:200].strip()
            results.append({
                "site_name": SITE_NAME,
                "title": final_title,
                "url": url,
                "content": content,
                "summary": summary,
                "pub_date": final_date,
                "category": "公告公示",
                "tags": "南大港,沧州,公告公示",
            })
            print(f"✅ ({len(content)}字)")
            time.sleep(0.3)

        if incremental:
            break

    if results:
        print(f"\n  入库 {len(results)} 条")
        push_to_searchdb(results, "nandagang_gggs")
    else:
        print("  ⏭ 无新数据入库")
    print("完成")


if __name__ == "__main__":
    run(sys.argv[1] if len(sys.argv) > 1 else None)