#!/usr/bin/env python3
"""爬取曹县人民政府 - 通知公告 / 政务公开栏目



API: POST http://www.caoxian.gov.cn/els-service/article/{page}/15
列表数据: JSON {dq, catas, order}
内容提取: 从详情页script中提取 var memo = "..." 的HTML
"""

import sys, re, json, sqlite3, requests, urllib.parse
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os

import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES
API_URL = "http://www.caoxian.gov.cn/els-service/article"
BASE_URL = "http://www.caoxian.gov.cn"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "曹县人民政府-通知公告"
THREE_YEARS_AGO = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
PAGE_SIZE = 15

HEADERS = {"User-Agent": "Mozilla/5.0", "Content-Type": "application/json;charset=utf-8"}

API_PAYLOAD = {
    "dq": "2c90808883d172a50183e89e80f5003b",
    "catas": ["1590299564160716800"],
    "order": "up"
}


def fetch_list(page):
    """获取列表页数据"""
    url = "%s/%d/%d" % (API_URL, page, PAGE_SIZE)
    r = requests.post(url, headers=HEADERS, json=API_PAYLOAD, timeout=30)
    data = r.json()
    return data.get("data", {})


def make_abs_url(base_url, src):
    """将相对路径转为绝对URL"""
    if not src or src.startswith("http://") or src.startswith("https://") or src.startswith("data:"):
        return src
    if src.startswith("//"):
        return "http:" + src
    parsed = urllib.parse.urlparse(base_url)
    path_dir = parsed.path.rsplit("/", 1)[0] + "/"
    if src.startswith("./"):
        src = src[2:]
    elif src.startswith("/"):
        return parsed.scheme + "://" + parsed.netloc + src
    return parsed.scheme + "://" + parsed.netloc + path_dir + src.lstrip("/")


def extract_content(detail_url):
    """从详情页script中提取 var memo 的HTML内容"""
    try:
        r = requests.get(detail_url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        html = r.text
        
        # 查找 var memo = "..."
        match = re.search(r'var memo\s*=\s*\"', html)
        if not match:
            return ""
        
        start = match.end()
        pos = start
        while pos < len(html):
            if html[pos] == '"':
                if pos > start and html[pos-1] == '\\':
                    pos += 1
                    continue
                raw = html[start:pos]
                break
            pos += 1
        else:
            return ""
        
        # JS unescape - handle \\uXXXX sequences, \\" and \\/
        content = raw.replace('\\"', '"').replace('\\/', '/')
        content = content.replace('\\n', '\n').replace('\\t', '\t')
        # Decode \\uXXXX Unicode escape sequences
        content = re.sub(r'\\u([0-9a-fA-F]{4})', lambda m: chr(int(m.group(1), 16)), content)
        
        # 将相对/upload-service路径转为绝对URL
        # 先找到基础目录
        parsed = urllib.parse.urlparse(detail_url)
        base = parsed.scheme + "://" + parsed.netloc
        content = content.replace('src="/upload-service/', 'src="%s/upload-service/' % base)
        content = content.replace('href="/upload-service/', 'href="%s/upload-service/' % base)
        
        return content
    except Exception as e:
        print("  [ERROR] extract content: %s" % str(e))
        return ""


def main():
    is_inc = "incremental" in sys.argv
    mode = "增量" if is_inc else "全量"
    print("=== %s模式: %s ===" % (mode, SITE_NAME))
    
    # 获取第1页获取总数
    data = fetch_list(1)
    total = data.get("elementsTotal", 0)
    total_pages = (total + PAGE_SIZE - 1) // PAGE_SIZE
    print("总数: %d, 总页数: %d" % (total, total_pages))
    
    max_pages = 1 if is_inc else total_pages
    
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    ins, skip, streak = 0, 0, 0
    
    for pi in range(1, min(max_pages, _MAX_PG or max_pages)+1):
        print("\n--- 第%d页 ---" % pi)
        data = fetch_list(pi)
        contents = data.get("contents", [])
        print("  条目数: %d" % len(contents))
        if not contents: break
        
        has_r = False
        for item in contents:
            subject = item.get("subject", "").strip()
            fwdate = item.get("fwdate", "")
            xxid = item.get("xxid", "")
            dwid = item.get("dwid", "")
            
            if fwdate and fwdate < THREE_YEARS_AGO:
                skip += 1
                continue
            if fwdate: has_r = True
            
            # 构造详情页URL
            url = "%s/2c90808883d172a50183e89e80f5003b/%s/%s.html" % (BASE_URL, dwid, xxid)
            
            html = extract_content(url)
            if not html:
                print("  [WARN] 无正文: %s..." % subject[:40])
            
            try:
                c.execute("INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary, date_rank) VALUES (?,?,?,?,?,?,?)",
                    (SITE_NAME, url, subject, html, fwdate, SITE_NAME, fwdate))
                if c.rowcount > 0: ins += 1
                else: skip += 1
            except Exception as e:
                print("  [ERR] %s" % str(e)); skip += 1
        
        if not has_r:
            streak += 1
            if streak >= 2 and pi >= 5:
                print("  连续2页无新数据，停止")
                break
        else: streak = 0
    
    conn.commit(); conn.close()
    print("\n=== 完成 新增:%d 跳过:%d ===" % (ins, skip))


if __name__ == "__main__": main()