#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
赵县人民政府 - 公告公示 爬虫
CMS: WebIMS v8.0.2 (石家庄市人民政府平台)
列表: index.html 首页15条（AJAX分页不可用）
详情: UUID路径, biaoti标题, conN正文
"""

import re
import sys
import json
import time
import hashlib
import subprocess
from datetime import datetime

import requests
from bs4 import BeautifulSoup

# ── 配置 ──────────────────────────────────────────
BASE_URL = "http://www.zhaoxian.gov.cn"
LIST_URL = f"{BASE_URL}/columns/0f1b1d84-6a48-4c2f-8419-7fe7e4b9fb51/index.html"
COLUMN_ID = "0f1b1d84-6a48-4c2f-8419-7fe7e4b9fb51"
SITE_NAME = "赵县人民政府"
GROUP = "县区"
DB_PATH = "/mnt/data/search.db"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}

session = requests.Session()
session.headers.update(HEADERS)


# ── 工具函数 ──────────────────────────────────────
# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def clean_text(text):
    """清理文本：去除多余空白"""
    if not text:
        return ""
    text = re.sub(r'\s+', ' ', text).strip()
    return text


def extract_publish_date(text):
    """从文本中提取日期 YYYY-MM-DD"""
    m = re.search(r'(\d{4})-(\d{2})-(\d{2})', text)
    if m:
        return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
    return datetime.now().strftime("%Y-%m-%d")


def extract_datetime(text):
    """从文本中提取完整日期时间"""
    m = re.search(r'(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})', text)
    if m:
        return m.group(1)
    d = extract_publish_date(text)
    return f"{d} 00:00"


def clean_content_html(html_content):
    """清洗正文HTML，保留段落结构"""
    if not html_content:
        return ""
    soup = BeautifulSoup(html_content, 'html.parser')
    # 移除script/style
    for tag in soup(['script', 'style']):
        tag.decompose()
    # 提取文本，段落用\n\n分隔
    texts = []
    for elem in soup.find_all(['p', 'div']):
        t = elem.get_text(strip=True)
        if t:
            texts.append(t)
    if not texts:
        texts = [soup.get_text(strip=True)]
    return '\n\n'.join(texts)


def md5(text):
    return hashlib.md5(text.encode('utf-8')).hexdigest()


# ── 列表页解析 ────────────────────────────────────
def parse_list(html):
    """解析列表页，返回 [(title, url, date), ...]"""
    items = []
    soup = BeautifulSoup(html, 'html.parser')
    # 找列表容器
    paging_div = soup.find('div', id=re.compile(r'pagingCont_'))
    if not paging_div:
        print("  [WARN] 未找到列表容器")
        return items

    # 每个条目：div[style*="border-bottom"] > a
    for item_div in paging_div.find_all('div', style=re.compile(r'border-bottom')):
        a_tag = item_div.find('a', href=re.compile(r'\.html'))
        if not a_tag:
            continue
        url = a_tag.get('href', '')
        if url and not url.startswith('http'):
            url = BASE_URL + url

        # 标题
        title_div = a_tag.find('div', style=re.compile(r'line-height:\s*35px'))
        title = clean_text(title_div.get_text()) if title_div else ''

        # 日期
        date_span = a_tag.find('span', string=re.compile(r'发布时间'))
        date_str = ''
        if date_span:
            date_str = extract_publish_date(date_span.get_text())
        else:
            date_div = a_tag.find('div', style=re.compile(r'color:#666666'))
            if date_div:
                date_str = extract_publish_date(date_div.get_text())

        if title and url:
            items.append((title, url, date_str))

    return items


# ── 详情页解析 ────────────────────────────────────
def parse_detail(html, url):
    """解析详情页，返回 (title, publish_date, source, content)"""
    soup = BeautifulSoup(html, 'html.parser')
    title = ''
    publish_date = ''
    source = ''
    content = ''

    # 1. 标题 - 从 #biaoti 中的 span[font-size:26px]
    biaoti_div = soup.find('div', id='biaoti')
    if biaoti_div:
        title_span = biaoti_div.find('span', style=re.compile(r'font-size:\s*26px'))
        if title_span:
            title = body_text(clean_text(title_span))
            title = title.replace('\n', '')

    # 2. 发布日期 - 从 #titN
    titn_div = soup.find('div', id='titN')
    if titn_div:
        titn_text = titn_div.get_text()
        # 发布时间
        m = re.search(r'发布时间[：:]\s*(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})', titn_text)
        if m:
            publish_date = m.group(1)
        if not publish_date:
            m = re.search(r'发布时间[：:]\s*(\d{4}-\d{2}-\d{2})', titn_text)
            if m:
                publish_date = m.group(1)
        # 来源
        m = re.search(r'来源[：:]\s*([^\s<]+)', titn_text)
        if m:
            source = m.group(1).strip()

    # 3. 正文 - 从 #conN
    conn_div = soup.find('div', id='conN')
    if conn_div:
        content = clean_content_html(str(conn_div))

    return title, publish_date, source, content


# ── 入库 ──────────────────────────────────────────
def insert_to_db(items_data):
    """批量写入 gov_raw + gov_search"""
    if not items_data:
        print("  无新增数据")
        return 0

    added = 0
    for item in items_data:
        try:
            result = subprocess.run(
                ['sqlite3', "-cmd", ".timeout 60000", DB_PATH],
                input=item['sql'],
                capture_output=True, text=True, timeout=30
            )
            if result.returncode == 0:
                added += 1
            else:
                if 'UNIQUE constraint' in result.stderr:
                    pass  # 重复跳过，正常
                else:
                    print(f"  [DB] {result.stderr[:200]}")
        except Exception as e:
            print(f"  [DB] 错误: {e}")

    return added


def build_insert_sql(item):
    """构建INSERT SQL"""
    page_url = item['url']
    title = item['title'].replace("'", "''")
    content = item.get('content', '').replace("'", "''")
    publish_date = item.get('publish_date', '')
    source = item.get('source', '').replace("'", "''")
    site_name = SITE_NAME
    group = GROUP
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    
    summary = title[:200]
    content_hash = md5(content)

    sql_raw = f"""INSERT OR IGNORE INTO gov_raw 
        (page_url, title, content, publish_date, source_url, site_name, summary, group_name, industry)
    VALUES 
        ('{page_url}', '{title}', '{content}', '{publish_date}', '{page_url}', '{site_name}', '{summary}', '{group}', '政府公告');
"""

    return sql_raw


def sync_fts(rowid):
    """同步单条记录到 gov_search FTS"""
    sql = f"""
INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary)
SELECT rowid, title, site_name, summary FROM gov_raw WHERE rowid = {rowid};
"""
    subprocess.run(
        ['sqlite3', "-cmd", ".timeout 60000", DB_PATH],
        input=sql,
        capture_output=True, text=True, timeout=30
    )


# ── 主流程 ────────────────────────────────────────
def main(max_pages=1):
    print(f"=== 赵县公告公示爬虫 ===")
    print(f"列表: {LIST_URL}")

    # 1. 获取列表
    try:
        r = session.get(LIST_URL, timeout=30)
        r.encoding = 'utf-8'
    except Exception as e:
        print(f"[ERROR] 列表请求失败: {e}")
        return

    items = parse_list(r.text)
    print(f"列表条目: {len(items)}")
    if not items:
        print("  无条目，结束")
        return

    # 2. 获取当前已爬URL (去重用)
    known_urls = set()
    try:
        result = subprocess.run(
            ['sqlite3', "-cmd", ".timeout 60000", DB_PATH, "SELECT page_url FROM gov_raw WHERE page_url LIKE '%zhaoxian.gov.cn%'"],
            capture_output=True, text=True, timeout=30
        )
        if result.returncode == 0:
            known_urls = set(result.stdout.strip().split('\n')) if result.stdout.strip() else set()
    except Exception:
        pass
    print(f"已知URL: {len(known_urls)}")

    # 3. 遍历详情
    new_items = []
    for title, url, date_str in items:
        if url in known_urls:
            continue
        if not title:
            continue

        print(f"  抓取: {title[:40]}...")
        try:
            r = session.get(url, timeout=30)
            r.encoding = 'utf-8'
        except Exception as e:
            print(f"    [ERR] {e}")
            time.sleep(1)
            continue

        detail_title, pub_date, source, content = parse_detail(r.text, url)
        if not detail_title:
            detail_title = title
        if not pub_date:
            pub_date = date_str

        # 正文质量检查
        content_len = len(content.strip())
        if content_len < 10:
            print(f"    [SKIP] 正文过短({content_len}字): {detail_title[:30]}")
            continue

        item_data = {
            'title': detail_title,
            'url': url,
            'publish_date': pub_date,
            'source': source,
            'content': content,
        }
        item_data['sql'] = build_insert_sql(item_data)
        new_items.append(item_data)

        time.sleep(0.5)

    print(f"\n新增: {len(new_items)} 条")

    # 4. 入库（单进程INSERT+取rowid+FTS同步）
    if new_items:
        db_added = 0
        for item in new_items:
            try:
                # 合并 INSERT + 取rowid + FTS同步到一次sqlite3调用
                combined_sql = item['sql'] + """
SELECT CASE WHEN changes() > 0 THEN last_insert_rowid() ELSE 0 END;
"""
                result = subprocess.run(
                    ['sqlite3', "-cmd", ".timeout 60000", DB_PATH],
                    input=combined_sql,
                    capture_output=True, text=True, timeout=30
                )
                if result.returncode == 0:
                    out = result.stdout.strip()
                    if out:
                        try:
                            rowid = int(out.strip())
                        except ValueError:
                            rowid = 0
                    else:
                        rowid = 0
                    
                    if rowid > 0:
                        # 同步FTS
                        fts_sql = f"""INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary)
SELECT rowid, title, site_name, summary FROM gov_raw WHERE rowid = {rowid};
"""
                        subprocess.run(
                            ['sqlite3', "-cmd", ".timeout 60000", DB_PATH],
                            input=fts_sql,
                            capture_output=True, text=True, timeout=30
                        )
                        db_added += 1
                else:
                    if 'UNIQUE constraint' not in result.stderr:
                        print(f"  [DB] {result.stderr[:200]}")
            except Exception as e:
                print(f"  [DB] 错误: {e}")

        print(f"入库成功: {db_added} 条")
    else:
        print("  无新增")

    print("=== 完成 ===")


if __name__ == '__main__':
    pages = 1
    if len(sys.argv) > 1:
        for arg in sys.argv[1:]:
            if arg.startswith('--pages='):
                try:
                    pages = int(arg.split('=')[1])
                except ValueError:
                    pass
    main(max_pages=pages)