#!/usr/bin/env python3
"""


黎川县人民政府 - 两个栏目
1. 乡镇动态(col2434): https://www.jxlcx.gov.cn/col/col2434/index.html
2. 重点民生项目(col3880): https://www.jxlcx.gov.cn/col/col3880/index.html?number=LC0006LC0001
Hanweb, AJAX分页, 正文: ZJEG_RSS.content
"""
import sys, os, re, time, json, sqlite3
from datetime import datetime, timezone, timedelta
import requests, urllib3
import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "黎川乡镇动态"
BASE_URL = "https://www.jxlcx.gov.cn"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
PROXY_URL = "https://www.jxlcx.gov.cn/module/web/jpage/dataproxy.jsp"
MAX_PAGES = 5
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")

# 栏目配置：(columnid, 栏目名, 标签, 输出前缀)
COLUMNS = [
    (2434, "黎川乡镇动态", "黎川乡镇动态", "lcx_xzdt"),
    (3880, "黎川重点民生项目", "黎川重点民生项目", "lcx_zsxm"),
]

def fetch(url, data=None):
    try:
        if data:
            r = requests.post(url, data=data, headers=HEADERS, timeout=20, verify=False)
        else:
            r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"❌ {e}")
        return None

def parse_list(html):
    """解析AJAX响应中的列表"""
    items = []
    for m in re.finditer(r"href='(/art/\d{4}/\d+/\d+/art_\d+_\d+\.html)'\s+title='([^']+)'[^>]*>.*?<span class=\"bt-data-time\">(\d{4}-\d{2}-\d{2})", html, re.DOTALL):
        href = m.group(1)
        title = m.group(2).strip()
        date_str = m.group(3)
        items.append((title, BASE_URL + href, date_str))
    return items

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None
    result = {}
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^\"]+)"', html)
    if m:
        result["title"] = m.group(1)
    if not result.get("title"):
        m = re.search(r'<title>(.*?)</title>', html)
        if m:
            t = m.group(1).split("|")[0].split("-")[0].strip()
            if t and len(t) > 5:
                result["title"] = t
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        result["publish_date"] = m.group(1)
    m = re.search(r'<!--ZJEG_RSS\.content\.begin-->(.*?)<!--ZJEG_RSS\.content\.end-->', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
        content = re.sub(r'<meta[^>]*>', '', content)
        if len(content) > 50:
            result["content"] = content
    if not result.get("content"):
        m = re.search(r'<meta name="ContentStart">(.*?)<meta name="ContentEnd">', html, re.DOTALL)
        if m and len(m.group(1).strip()) > 50:
            result["content"] = m.group(1).strip()
    return result.get("title"), result.get("content"), result.get("publish_date")

def crawl_column(column_id, column_name, site_tag, output_tag):
    """爬取指定栏目的文章（5页 + 近3年过滤）"""
    print(f"\n{'='*50}\n🏠 {column_name} (columnid={column_id})\n{'='*50}")
    all_list = []
    for page in range(1, min(MAX_PAGES, _MAX_PG or MAX_PAGES)+1):
        data = {"page": str(page), "appid": "1", "webid": "8", "path": "/",
                "columnid": str(column_id), "unitid": "66104",
                "webname": "黎川县人民政府", "permissiontype": "0"}
        print(f"📄 第 {page}/{MAX_PAGES} 页...", end=" ", flush=True)
        html = fetch(PROXY_URL, data)
        if not html:
            print("❌ (请求失败)")
            break
        items = parse_list(html)
        if not items:
            print("0 条 (无更多)")
            break
        print(f"✅ {len(items)} 条")
        all_list.extend(items)

    print(f"📊 列表总计: {len(all_list)} 条")

    try:
        _cx = sqlite3.connect(os.getenv("SEARCH_DB", "/root/search.db"), timeout=60)
        _existing = set(r[0] for r in _cx.execute(
            "SELECT page_url FROM gov_raw WHERE site_name=?", (column_name,)).fetchall())
        _cx.close()
    except Exception as e:
        print(f"  [WARN] 读取已入库URL失败: {e}")
        _existing = set()
    print(f"📚 已入库 {len(_existing)} 条，重复跳过详情")

    all_items, seen = [], set()
    # 近3年过滤
    filtered = 0
    for i, (title, url, list_date) in enumerate(all_list):
        if url in seen:
            continue
        seen.add(url)
        if list_date and list_date < THREE_YEARS_AGO:
            filtered += 1
            continue
        if url in _existing:
            continue
        # 无关键词过滤，全部入库
        print(f"  [{len(all_items)+1}/{len(all_list)}] {title[:55]}...", end=" ", flush=True)
        dt, content, date = fetch_detail(url)
        if dt:
            title = dt
        pub_date = date or list_date
        if pub_date and pub_date < THREE_YEARS_AGO:
            filtered += 1
            print("⏭ (超3年)")
            continue
        summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
        summary = re.sub(r"\s+", " ", summary)
        all_items.append({
            "site_name": column_name, "title": title, "url": url,
            "content": content or "", "pub_date": pub_date,
            "summary": summary, "tags": site_tag,
        })
        print("✅")
        time.sleep(0.3)

    if filtered:
        print(f"⏭ 跳过超3年: {filtered} 条")

    if all_items:
        push_to_searchdb(all_items, output_tag)
    print(f"\n✅ {column_name}: 完成! 共 {len(all_items)} 条")
    return len(all_items)

def main():
    t0 = time.time()
    total = 0
    for col_id, col_name, tag, out_tag in COLUMNS:
        total += crawl_column(col_id, col_name, tag, out_tag)
    print(f"\n{'='*50}\n✅ 全部完成! 共 {total} 条\n⏱ 总耗时: {time.time()-t0:.1f}s\n{'='*50}")

if __name__ == "__main__":
    main()