#!/usr/bin/env python3
"""江都区通用 - 通知公告 爬虫 (JPAAS publish-system)
站点: jd.yangzhou.gov.cn
支持 --site-name, --page-id, --script-name 参数实现通用
"""

import requests, re, sqlite3, time, sys, json, warnings
warnings.filterwarnings('ignore')
from datetime import datetime, timedelta
import os

# --- CLI 参数解析 ---
_MAX_PG = None
_SITE_NAME = "江都区浦头镇-通知公告"
_PAGE_ID = "VDozh0QjlXic3EzTpmlVf"
_SCRIPT_NAME = "crawl_jd_common.py"

i = 0
while i < len(sys.argv):
    a = sys.argv[i]
    if a.startswith("--pages="):
        _MAX_PG = int(a.split("=", 1)[1])
    elif a == "--pages" and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        i += 1
    elif a.startswith("--site-name="):
        _SITE_NAME = a.split("=", 1)[1]
    elif a == "--site-name" and i + 1 < len(sys.argv):
        _SITE_NAME = sys.argv[i + 1]
        i += 1
    elif a.startswith("--page-id="):
        _PAGE_ID = a.split("=", 1)[1]
    elif a == "--page-id" and i + 1 < len(sys.argv):
        _PAGE_ID = sys.argv[i + 1]
        i += 1
    elif a.startswith("--script-name="):
        _SCRIPT_NAME = a.split("=", 1)[1]
    elif a == "--script-name" and i + 1 < len(sys.argv):
        _SCRIPT_NAME = sys.argv[i + 1]
        i += 1
    i += 1

print(f'[Config] site_name={_SITE_NAME} page_id={_PAGE_ID} script_name={_SCRIPT_NAME}')
if _MAX_PG is not None:
    print(f'[Config] max_pages={_MAX_PG}')

SITE_NAME = _SITE_NAME
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SCRIPT_NAME = _SCRIPT_NAME
DOMAIN = "jd.yangzhou.gov.cn"

CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

API_URL = "https://jd.yangzhou.gov.cn/api-gateway/jpaas-publish-server/front/page/build/unit"
API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "AgmxeI2t5aGgxplX60MdA",
    "tplSetId": "CU8i6WLXe0BLnem5xuDSq",
    "pageType": "column",
    "tagId": "\u5217\u8868\u5217\u8868",
    "editType": "null",
    "pageId": _PAGE_ID,
}

def fetch_list(page, page_size=15):
    params = dict(API_PARAMS)
    params["paramJson"] = json.dumps({"pageNo": page, "pageSize": page_size})
    for retry in range(3):
        try:
            r = requests.get(API_URL, params=params, headers=HEADERS, timeout=15, verify=False)
            if r.status_code == 200:
                d = r.json()
                html = d.get("data", {}).get("html", "")
                items = []
                for li in re.findall(r'<li[^>]*>(.*?)</li>', html, re.DOTALL):
                    m = re.search(
                        r'href=\"([^\"]+)\"[^>]*>([^<]+)</a>\s*<span[^>]*>(\d{4}-\d{2}-\d{2})</span>',
                        li)
                    if m:
                        title = m.group(2).strip()
                        items.append({
                            "url": "https://" + DOMAIN + m.group(1),
                            "title": title,
                            "date": m.group(3)
                        })
                total_pages = 0
                m2 = re.search(r'class=\"totalpages\"[^>]*>(\d+)<', html)
                if m2:
                    total_pages = int(m2.group(1))
                if not total_pages:
                    m3 = re.search(r'count=\"(\d+)\"', html)
                    if m3:
                        total_items = int(m3.group(1))
                        total_pages = (total_items + page_size - 1) // page_size
                print(f"  page {page}: {len(items)} items, total_pages={total_pages}")
                return items, total_pages
        except Exception as e:
            print(f"  [WARN] fetch_list page {page} retry {retry}: {e}", file=sys.stderr)
            time.sleep(2)
    return [], 0

def fetch_detail(url):
    for retry in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
            r.encoding = "utf-8"
            html = r.text
            m = re.search(
                r'<!--\u6b63\u6587-->\s*<div[^>]*class=\"bt-content[^\"]*zoom[^\"]*clearfix[^\"]*\"[^>]*>(.*?)</div>\s*\n\s*(?:<style|<div class=\"bt-art-assist)',
                html, re.DOTALL)
            if m:
                content = m.group(1).strip()
            else:
                m = re.search(r'<div[^>]*class=\"bt-content[^>]*>(.*?)</div>\s*\n\s*(?:<style|<div class=\"bt-art-assist)', html, re.DOTALL)
                if m:
                    content = m.group(1).strip()
                else:
                    content = ""
            return content
        except Exception as e:
            print(f"  [WARN] fetch_detail retry {retry}: {e}", file=sys.stderr)
            time.sleep(2)
    return ""

def extract_text(html):
    if not html:
        return ""
    table_tags = ['table', '/table', 'tr', '/tr', 'td', '/td', 'th', '/th', 'tbody', '/tbody', 'thead', '/thead']
    placeholders = {}
    for i, tag in enumerate(table_tags):
        ph = f'__TBL{i}__'
        placeholders[ph] = f'<{tag}>'
        html = re.sub(rf'(?i)<{tag}[^>]*>', ph, html)
    for tag in ['p', '/p', 'div', '/div', 'li', '/li', '/h[1-6]', 'h[1-6]']:
        html = re.sub(rf'(?i)<{tag}[^>]*>', '\n', html)
    html = re.sub(r'(?i)<br\s*/?>', '\n', html)
    text = re.sub(r'<[^>]+>', '', html)
    for ph, tag in sorted(placeholders.items(), reverse=True):
        text = text.replace(ph, tag)
    text = re.sub(r'\n{3,}', '\n\n', text)
    lines = [l.strip() for l in text.split('\n')]
    text = '\n'.join(lines)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()

def has_table(html):
    return 1 if '<table' in html else 0

def save_to_db(items):
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()
    new_count = 0
    for item in items:
        url = item["url"]
        title = item["title"]
        pub_date = item["date"]
        content_html = item.get("content_html", "")
        content_text = extract_text(content_html)
        if not content_text:
            continue
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
        if c.fetchone():
            continue
        ht = has_table(content_html)
        c.execute(
            "INSERT OR IGNORE INTO gov_raw(site_name,source_url,page_url,title,publish_date,content,summary,status,category,group_name,has_table,script_name) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
            (SITE_NAME, url, url, title, pub_date, content_text, content_text[:200], "published", "通知公告", "江苏", ht, SCRIPT_NAME)
        )
        if c.rowcount > 0:
            new_count += 1
            print(f"  [+] {title[:40]} | {pub_date}")
    conn.commit()
    # QC20260926 去掉手写 gov_search 整站删除(抢锁源; FTS 由 gov_raw 触发器维护) 
    # c.execute("DELETE FROM gov_search WHERE site_name=?", (SITE_NAME,))
    # 2026-09-22: 先提交 gov_raw —— 下面手动写 FTS 会因触发器已写过同一
    #   rowid 而 IntegrityError，若不先 commit，这条记录会被一并回滚（静默丢数据）
    conn.commit()
    c.execute("INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) SELECT rowid,title,site_name,summary FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    conn.commit()
    conn.close()
    return new_count

def main():
    start_time = time.time()
    all_items = []
    items, total_pages = fetch_list(1)
    all_items.extend(items)
    max_pages = _MAX_PG if _MAX_PG else total_pages
    if max_pages <= 0:
        max_pages = 3
    for pg in range(2, max_pages + 1):
        if pg > total_pages:
            break
        items, _ = fetch_list(pg)
        all_items.extend(items)
        time.sleep(0.5)
    print(f"\nTotal list items: {len(all_items)}")
    for i, item in enumerate(all_items):
        html = fetch_detail(item["url"])
        item["content_html"] = html
        text_len = len(extract_text(html))
        print(f"  [{i+1}/{len(all_items)}] {item['title'][:30]}... ({text_len} chars)")
        time.sleep(0.5)
    count = save_to_db(all_items)
    elapsed = time.time() - start_time
    print(f"\n{'='*50}")
    print(f"Site: {SITE_NAME}")
    print(f"New records: {count}/{len(all_items)}")
    print(f"Script: {SCRIPT_NAME}")
    print(f"Time: {elapsed:.1f}s")
    print(f"{'='*50}")

if __name__ == "__main__":
    main()