#!/usr/bin/env python3
"""
凯里市-通知公告 爬虫 (BeautifulSoup版)
站点: www.kaili.gov.cn
栏目: 通知公告
"""
import requests, sqlite3, time, sys, re as re_mod
from bs4 import BeautifulSoup
import warnings
warnings.filterwarnings('ignore')
from datetime import datetime, timedelta
import os

SEARCH_DB = os.getenv("SEARCH_DB", "/mnt/data/search.db")
SITE_NAME = "凯里-通知公告"
SCRIPT_NAME = "crawl_kaili_tzgg.py"
GROUP_NAME = "贵州"
DOMAIN = "www.kaili.gov.cn"
BASE_PATH = "/xwzx/tzgg"

CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


def clean_title(title):
    """strip ·  实体前缀 和省略号截断后缀"""
    title = title.replace("·", "").replace(" ", "")
    title = title.replace("\u200b", "").replace("\ufeff", "")
    title = re_mod.sub(r"^[\s\xa0·\u00b7]+", "", title)
    title = re_mod.sub(r"\s*\.{3,}\s*$", "", title)
    return title.strip()


def fetch_list_page(page):
    """获取列表页"""
    if page == 1:
        url = f"https://{DOMAIN}{BASE_PATH}/index.html"
    else:
        url = f"https://{DOMAIN}{BASE_PATH}/index_{page-1}.html"

    for retry in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            if r.status_code == 200:
                r.encoding = "utf-8"
                soup = BeautifulSoup(r.text, 'html.parser')

                items = []
                ul = soup.find('ul', class_='ul_1')
                if not ul:
                    return [], 0

                for li in ul.find_all('li'):
                    a = li.find('a')
                    if not a:
                        continue
                    href = a.get('href', '').strip()
                    title = a.get_text(strip=True)
                    span = li.find('span')
                    date = span.get_text(strip=True) if span else ""

                    # Normalize URL
                    if not href.startswith('http'):
                        if href.startswith('//'):
                            href = f'https:{href}'
                        elif href.startswith('/'):
                            href = f'https://{DOMAIN}{href}'
                        elif href.startswith('./'):
                            href = f'https://{DOMAIN}{BASE_PATH}/{href[2:]}'
                        else:
                            href = f'https://{DOMAIN}{BASE_PATH}/{href}'

                    items.append({"url": href, "title": title, "date": date})

                # Total pages from createPageHTML
                pm = re_mod.search(r'createPageHTML\((\d+)', r.text)
                total_pages = int(pm.group(1)) if pm else 93
                return items, total_pages
        except Exception as e:
            if retry < 2:
                time.sleep(2)
    return [], 0


def extract_content(html, page_url):
    """从 trs_editor_view 提取正文: \\n\\n 分段 + 表格保留HTML + 附件内嵌绝对URL"""
    soup = BeautifulSoup(html, 'html.parser')
    div = None
    for cls in ['trs_editor_view', 'TRS_UEDITOR']:
        div = soup.find('div', class_=lambda c: c and cls in c)
        if div:
            break
    if not div:
        cont = soup.find('div', class_='cont')
        if cont:
            div = cont
    if not div:
        return ""

    for tag in div.find_all(['script', 'style']):
        tag.decompose()

    # 附件链接转绝对 URL + 清理前置图标 img
    attachments = []
    for a in div.find_all('a', href=True):
        href = a['href']
        txt = a.get_text(strip=True) or a.get('title', '')
        if (re_mod.search(r'(?i)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|et|ofd)(\?|$)', href)
                or 'attach' in href.lower() or 'download' in href.lower() or txt.startswith('附件')):
            abs_url = requests.compat.urljoin(page_url, href)
            if not abs_url.startswith(('http://', 'https://')):
                abs_url = 'https:' + abs_url if abs_url.startswith('//') else abs_url
            name = txt if txt else abs_url.split('/')[-1]
            attachments.append((a, name, abs_url))
    for a, name, abs_url in attachments:
        new_a = soup.new_tag('a', href=abs_url, target='_blank')
        new_a.string = name
        if a.parent is not None:
            p = a.parent
            idx_a = None
            for ci, child in enumerate(p.contents):
                if child is a:
                    idx_a = ci
                    break
            if idx_a is not None:
                for child in list(p.contents[:idx_a]):
                    if getattr(child, 'name', None) == 'img':
                        child.decompose()
        a.replace_with(new_a)
    attached_hrefs = {abs_url for _, _, abs_url in attachments}

    # 正文段落提取
    parts = []
    for el in div.find_all(recursive=False):
        if el.name == 'table':
            parts.append(str(el))
            continue
        if el.name in ('p', 'div', 'h1', 'h2', 'h3', 'ul', 'ol', 'li', 'blockquote'):
            txt = el.get_text('', strip=True)
            if not txt and not el.find('img'):
                continue
            inner_tables = el.find_all('table')
            el_has_attach = any(
                requests.compat.urljoin(page_url, x['href']) in attached_hrefs or x['href'] in attached_hrefs
                for x in el.find_all('a', href=True))
            if inner_tables or el_has_attach:
                parts.append(str(el))
                continue
            parts.append(txt)
        else:
            txt = el.get_text('', strip=True)
            if txt:
                parts.append(txt)

    # 段落去重
    seen = set()
    final_parts = []
    for p in parts:
        key = re_mod.sub(r'\s+', '', re_mod.sub(r'<[^>]+>', '', p))
        if key and key not in seen:
            seen.add(key)
            final_parts.append(p)

    content = '\n\n'.join(final_parts)
    content_unescaped = content.replace('&amp;', '&')
    for _, name, abs_url in attachments:
        if abs_url not in content and abs_url not in content_unescaped:
            content += f'\n\n<p><a href="{abs_url}" target="_blank">{name}</a></p>'

    content = re_mod.sub(r'打印本页[^\n]*', '', content)
    content = re_mod.sub(r'\n{3,}', '\n\n', content).strip()
    return content


def extract_meta(html):
    soup = BeautifulSoup(html, 'html.parser')
    title = ""
    tm = soup.find('meta', attrs={"name": "ArticleTitle"})
    if tm and tm.get('content'):
        title = clean_title(tm['content'])
    if not title:
        tt = soup.find('title')
        if tt:
            title = clean_title(tt.get_text(strip=True))

    date = ""
    pm = soup.find('meta', attrs={"name": "PubDate"})
    if pm and pm.get('content'):
        d = pm['content'].strip()
        cm = re_mod.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', d)
        if cm:
            date = f"{cm.group(1)}-{cm.group(2).zfill(2)}-{cm.group(3).zfill(2)}"
    if not date:
        dm = re_mod.search(r'(\d{4}-\d{2}-\d{2})', html)
        if dm:
            date = dm.group(1)
    return title, date


def insert_article(url, title, date, content):
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    try:
        summary = re_mod.sub(r'<[^>]+>', '', content)[:200]
        summary = re_mod.sub(r'\s+', ' ', summary).strip()
        has_table = 1 if '<table' in content else 0
        db.execute(
            "INSERT OR IGNORE INTO gov_raw(site_name,source_url,page_url,title,publish_date,content,summary,status,category,script_name,group_name,has_table) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
            (SITE_NAME, url, url, title, date, content, summary, "active", "通知公告", SCRIPT_NAME, GROUP_NAME, has_table))
        affected = db.total_changes
        db.commit()
        if affected:
            db.execute(
                "INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.page_url=? AND r.id NOT IN (SELECT rowid FROM gov_search)",
                (url,))
            db.commit()
        db.close()
        return affected
    except Exception as e:
        db.close()
        print(f"  DB error: {e}")
        return 0


def run(max_pages=100):
    print(f"\n{'='*50}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*50}")

    items, total_pages = fetch_list_page(1)
    if not items:
        print("❌ 列表页无数据")
        return

    total_pages = min(max_pages, total_pages)
    print(f"  共{total_pages}页, 约{1830}条")

    all_items = list(items)
    for pg in range(2, total_pages + 1):
        its, _ = fetch_list_page(pg)
        if not its:
            break
        all_items.extend(its)
        dates = [it["date"] for it in its if it["date"]]
        if dates:
            print(f"  第{pg}页: {len(its)} 条 ({min(dates)} ~ {max(dates)})")
        else:
            print(f"  第{pg}页: {len(its)} 条")
        time.sleep(0.2)

    print(f"\n📊 共 {len(all_items)} 条")
    new = skip = no_body = err = 0
    for i, item in enumerate(all_items, 1):
        if item["date"] and item["date"] < CUTOFF:
            skip += 1
            continue

        for retry in range(3):
            try:
                r = requests.get(item["url"], headers=HEADERS, timeout=30)
                if r.status_code == 200:
                    r.encoding = "utf-8"
                    html = r.text
                    break
            except:
                if retry < 2:
                    time.sleep(2)
                html = None

        if not html:
            err += 1
            continue

        pt, pd = extract_meta(html)
        real_title = pt if pt else item["title"]
        real_date = pd if pd else item["date"]

        if real_date and real_date < CUTOFF:
            skip += 1
            continue

        content = extract_content(html, item["url"])
        text_len = len(re_mod.sub(r'<[^>]+>', '', content).strip()) if content else 0
        if text_len < 10:
            no_body += 1
            continue

        if insert_article(item["url"], real_title, real_date, content):
            new += 1
        else:
            skip += 1

        if i % 5 == 0:
            print(f"  ...{i}/{len(all_items)}")
        time.sleep(0.2)

    # FTS sync
    print(f"\n📊 FTS同步...")
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    # QC20260926 去掉手写 gov_search 整站删除(抢锁源; FTS 由 gov_raw 触发器维护) 
    # db.execute("DELETE FROM gov_search WHERE site_name=?", (SITE_NAME,))
    db.execute("INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) SELECT rowid,title,site_name,summary FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    db.commit()
    c = db.execute("SELECT COUNT(*) FROM gov_search WHERE site_name=?", (SITE_NAME,))
    cnt = c.fetchone()[0]
    db.close()
    print(f"  FTS: {cnt} 条")
    print(f"\n✅ {SITE_NAME}: 新增{new}, 跳过{skip}, 空正文{no_body}, 错误{err}, FTS共{cnt}条")


if __name__ == "__main__":
    # 兼容 --pages=N / --pages N / 裸数字 三种格式 (配置 args="1" 也兼容)
    mp = 100
    args = sys.argv[1:]
    i = 0
    while i < len(args):
        a = args[i]
        if a.startswith('--pages='):
            try:
                mp = int(a.split('=', 1)[1])
            except ValueError:
                pass
        elif a == '--pages' and i + 1 < len(args):
            try:
                mp = int(args[i + 1])
            except ValueError:
                pass
            i += 1
        else:
            try:
                mp = int(a)
            except ValueError:
                pass
        i += 1
    run(mp)