#!/usr/bin/env python3
"""
五通桥区人民政府-公告公示 (wtq.leshan.cn)
列表: /Home/List/MFkbD4IbPTn2nY-DuyXtLQ==/页码
条目: <li><a href="链接">标题</a><div class="time">日期</div>
链接类型:
  - WeChat (mp.weixin.qq.com) → 抓取公众号正文
  - 本站文章 (/Home/Article/XXXXXX) → 抓取 <div class="content-text"> 正文
  - 其他外链 (wtqrm.com, nrta.gov.cn 等) → 仅存标题+日期+URL，无正文
"""
import sys, os, re, time, html as html_mod, json
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "五通桥区-公告公示"
BASE_URL = "http://wtq.leshan.cn"
LIST_PATH = "/Home/List/MFkbD4IbPTn2nY-DuyXtLQ=="
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"}

def fetch(url):
    for _ in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=20)
            r.encoding = "utf-8"
            if r.text.strip():
                return r.text
        except:
            pass
        time.sleep(2)
    return None

def get_total_pages(html):
    """从分页导航获取总页数"""
    nums = re.findall(r'<a[^>]*href="[^"]+/(\d+)"[^>]*>(\d+)</a>', html)
    max_p = 1
    for href_num, display_num in nums:
        n = int(display_num)
        if n > max_p:
            max_p = n
    return max_p

def parse_list_items(html):
    """提取所有条目 (title, url, date_str) — 不限 WeChat"""
    items = []
    ul_m = re.search(r'<ul>(.*?)</ul>', html, re.DOTALL)
    if not ul_m:
        return items
    ul = ul_m.group(1)
    for li in re.findall(r'<li>(.*?)</li>', ul, re.DOTALL):
        href_m = re.search(r'href="([^"]+)"', li)
        title_m = re.search(r'<a[^>]*>([^<]+)</a>', li)
        date_m = re.search(r'<div class="time">([^<]+)</div>', li)
        if not (href_m and title_m and date_m):
            continue
        href = html_mod.unescape(href_m.group(1)).strip()
        title = html_mod.unescape(title_m.group(1)).strip()
        date_str = date_m.group(1)
        if title:
            items.append((title, href, date_str))
    return items

def fetch_local_content(url):
    """抓取本站文章 /Home/Article/XXXX 的正文"""
    html = fetch(url)
    if not html:
        return ""
    m = re.search(r'<div class="content-text">(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
        # 清理空段落和多余空格
        content = re.sub(r'<p[^>]*>\s*(?:&nbsp;)*\s*</p>', '', content)
        return content
    return ""

def fetch_wechat_content(url):
    """抓取微信公众号文章正文"""
    for _ in range(2):
        try:
            r = requests.get(url, headers=HEADERS, timeout=20)
            r.encoding = "utf-8"
            html = r.text
            m = re.search(r'<div[^>]*class="rich_media_content[^"]*"[^>]*>(.*?)</div>\s*(?:<script|</div>)', html, re.DOTALL)
            if m:
                content = m.group(1).strip()
                content = re.sub(r'data-src="', 'src="', content)
                content = re.sub(r'data-cropsrc="', 'src="', content)
                content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL | re.I)
                content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL | re.I)
                return content
        except:
            pass
        time.sleep(2)
    return ""

def fetch_external_content(url):
    """尝试抓取外链正文（wtqrm.com 等），失败则返回空字符串"""
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        r.encoding = "utf-8"
        if r.status_code == 200 and len(r.text) > 200:
            # 尝试常见的选择器
            html = r.text
            for pattern in [
                r'<div class="content-text">(.*?)</div>',
                r'<div class="article-content">(.*?)</div>',
                r'<div class="content">(.*?)</div>',
                r'<article[^>]*>(.*?)</article>',
                r'<div class="maintext">(.*?)</div>',
            ]:
                m = re.search(pattern, html, re.DOTALL)
                if m and len(m.group(1)) > 100:
                    return m.group(1).strip()
    except:
        pass
    return ""

def classify_url(url):
    """判断 URL 类型: local / wechat / external"""
    if url.startswith('/') or BASE_URL in url:
        return 'local'
    if 'mp.weixin.qq.com' in url:
        return 'wechat'
    return 'external'

def main(incremental=False, limit=None):
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")

    html = fetch(BASE_URL + LIST_PATH)
    if not html:
        print("❌ 首页获取失败")
        return

    total_pages = get_total_pages(html)
    print(f"📊 共 {total_pages} 页")

    if incremental:
        total_pages = min(total_pages, 5)
        print(f"🔄 增量模式: 取前 {total_pages} 页")

    all_items, seen = [], set()
    stop = False
    stats = {"local": 0, "wechat": 0, "external": 0}

    for pg in range(1, total_pages + 1):
        if stop:
            break
        if pg > 1:
            page_url = f"{BASE_URL}{LIST_PATH}/{pg}"
            html = fetch(page_url)
            if not html:
                continue

        items = parse_list_items(html)
        if not items:
            continue

        # 日期过滤
        page_dates = [d for _, _, d in items]
        if page_dates:
            oldest = min(page_dates)
            if oldest < THREE_YEARS_AGO:
                print(f"  ⏹ 第{pg}页最早 {oldest}，停止")
                break
            print(f"  📄 第{pg}页 ({min(page_dates)}~{max(page_dates)})")

        for title, url, list_date in items:
            if url in seen:
                continue
            if list_date and list_date < THREE_YEARS_AGO:
                continue
            seen.add(url)

            url_type = classify_url(url)
            # 构造完整 URL
            if url.startswith('/'):
                full_url = BASE_URL + url
            else:
                full_url = url

            content = ""
            if url_type == 'local':
                stats['local'] += 1
                print(f"  [{len(all_items)+1}] 📄 本站: {title[:40]}...", end=" ", flush=True)
                content = fetch_local_content(full_url)
            elif url_type == 'wechat':
                stats['wechat'] += 1
                print(f"  [{len(all_items)+1}] 💬 微信: {title[:40]}...", end=" ", flush=True)
                content = fetch_wechat_content(full_url)
                time.sleep(1.5)
            else:
                stats['external'] += 1
                print(f"  [{len(all_items)+1}] 🔗 外链: {title[:40]}...", end=" ", flush=True)
                content = fetch_external_content(full_url)

            summary = re.sub(r"<[^>]+>", " ", content).strip()[:300]
            summary = re.sub(r"\s+", " ", summary) if summary else title

            print(f"✅ ({len(content)} chars)")
            all_items.append({
                "site_name": SITE_NAME,
                "title": title,
                "url": full_url,
                "content": content,
                "pub_date": list_date,
                "summary": summary,
                "tags": SITE_NAME,
            })

            if limit and len(all_items) >= limit:
                stop = True
                break

    if all_items:
        push_to_searchdb(all_items, "wtq_leshan")
        print(f"\n✅ 完成! 共 {len(all_items)} 条 (本站:{stats['local']} 微信:{stats['wechat']} 外链:{stats['external']})")
    else:
        print("\n⏭ 无新数据")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--incremental", action="store_true")
    parser.add_argument("--limit", type=int)
    args = parser.parse_args()
    incremental = args.incremental or (len(sys.argv) > 1 and sys.argv[1] == "1")
    t0 = time.time()
    main(incremental=incremental, limit=args.limit)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")