#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
九原区人民政府 - 生态环境 爬虫
CMS: 九原区自建 CMS (layui)
列表: /zwgk/zfxxgk/fdzdgknr/zdlyxxgk/sthj/index.html (page1)
       /zwgk/zfxxgk/fdzdgknr/zdlyxxgk/sthj/index_N.html (page N+1)
详情: /zwgk/zfxxgk/fdzdgknr/zdlyxxgk/sthj/YYYYMM/tYYYYMMDD_N.html
"""

import re
import sys
import time
import subprocess
from datetime import datetime

import requests
from bs4 import BeautifulSoup

# ── 配置 ──────────────────────────────────────────
BASE_URL = "https://www.jiuyuanqu.gov.cn"
LIST_DIR = "/zwgk/zfxxgk/fdzdgknr/zdlyxxgk/sthj"
LIST_URL = BASE_URL + LIST_DIR + "/"
SITE_NAME = "九原区人民政府"
GROUP = "内蒙古"
DB_PATH = "/mnt/data/search.db"
TOTAL_PAGES = 20  # 193条 ÷ 10条/页

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}

session = requests.Session()
session.headers.update(HEADERS)


# ── 工具函数 ──────────────────────────────────────
def clean_text(text):
    if not text:
        return ""
    # strip ·   实体前缀
    text = re.sub(r'(·| |\s)+', ' ', text)
    return text.strip()


def clean_content_html(html_content):
    """提取正文，只处理直接子元素避免重复，表格保持HTML"""
    if not html_content:
        return ""
    soup = BeautifulSoup(html_content, 'html.parser')
    for tag in soup(['script', 'style']):
        tag.decompose()
    parts = []
    for child in soup.children:
        if not child.name:
            continue
        if child.name == 'p':
            t = child.get_text(strip=True)
            if t:
                parts.append(t)
        elif child.name == 'table':
            parts.append(str(child))
        elif child.name in ('div', 'section', 'article'):
            for p in child.find_all('p', recursive=True):
                if p.find_parent('table'):
                    continue
                t = p.get_text(strip=True)
                if t:
                    parts.append(t)
            for tbl in child.find_all('table', recursive=True):
                parts.append(str(tbl))
        elif child.name == 'img':
            src = child.get('src', '')
            alt = child.get('alt', '')
            if src:
                if src.startswith('/'):
                    src = BASE_URL + src
                parts.append(f'![{alt}]({src})')
    return '\n\n'.join(parts)


# ── 列表获取 ──────────────────────────────────────
def fetch_list(max_pages=TOTAL_PAGES):
    """遍历所有分页获取文章列表"""
    all_items = []
    for page in range(max_pages):
        if page == 0:
            url = LIST_URL + "index.html"
        else:
            url = LIST_URL + f"index_{page}.html"
        print(f"  第{page+1}页...", end=' ')
        try:
            r = session.get(url, timeout=30)
            r.encoding = 'utf-8'
            if r.status_code != 200:
                print(f"HTTP {r.status_code}")
                break
        except Exception as e:
            print(f"ERR {e}")
            break

        soup = BeautifulSoup(r.text, 'html.parser')
        ul = soup.find('ul', class_='module_list')
        if not ul:
            ul = soup.select('ul.module_list')
            if not ul:
                print("无列表")
                break
            ul = ul[0]

        lis = ul.find_all('li')
        count = 0
        for li in lis:
            a = li.find('a')
            if not a:
                continue
            href = a.get('href', '')
            spans = a.find_all('span')
            if len(spans) < 2:
                continue
            title = clean_text(spans[0].get_text())
            date = clean_text(spans[1].get_text())
            if not href or not title:
                continue
            full_url = href
            if href.startswith('./'):
                full_url = LIST_URL + href[2:]
            elif href.startswith('/'):
                full_url = BASE_URL + href
            elif not href.startswith('http'):
                full_url = LIST_URL + href
            all_items.append({'url': full_url, 'title': title, 'date': date})
            count += 1

        print(f"{count}条")
        time.sleep(0.3)
    return all_items


# ── 详情页解析 ────────────────────────────────────
def parse_detail(html, url):
    """解析详情页，返回 (title, publish_date, source, content)"""
    soup = BeautifulSoup(html, 'html.parser')
    title = ''
    publish_date = ''
    source = ''
    content = ''

    # 1. 标题 - meta优先，然后div.info_title
    meta_title = soup.find('meta', attrs={'name': 'ArticleTitle'})
    if meta_title and meta_title.get('content'):
        title = clean_text(meta_title['content'])
    if not title:
        div_title = soup.find('div', class_='info_title')
        if div_title:
            p = div_title.find('p')
            if p:
                title = clean_text(p.get_text())
    if not title:
        meta_t = soup.find('meta', attrs={'name': 'NewTitle'})
        if meta_t and meta_t.get('content'):
            title = clean_text(meta_t['content'])

    # 2. 发布日期 - meta优先
    meta_date = soup.find('meta', attrs={'name': 'PubDate'})
    if meta_date and meta_date.get('content'):
        m = re.search(r'(\d{4}-\d{2}-\d{2})', meta_date['content'])
        if m:
            publish_date = m.group(1)
    if not publish_date:
        date_span = soup.find('span', class_='info_msg_fl')
        if date_span:
            m = re.search(r'(\d{4}-\d{2}-\d{2})', date_span.get_text())
            if m:
                publish_date = m.group(1)

    # 3. 来源 - meta优先
    meta_src = soup.find('meta', attrs={'name': 'ContentSource'})
    if meta_src and meta_src.get('content'):
        source = meta_src['content'].strip()
    if not source:
        src_span = soup.find('span', class_='info_msg_fr')
        if src_span:
            m = re.search(r'来源[：:]\s*(.+)', src_span.get_text())
            if m:
                source = m.group(1).strip()

    # 4. 正文 - div#content.info_msg
    content_div = soup.find('div', class_='info_msg')
    if not content_div:
        content_div = soup.find('div', id='content')
    if content_div:
        # 移除附件列表ul（底部导航ul）
        uls = content_div.find_all('ul')
        for ul in uls:
            ul.decompose()
        content = clean_content_html(str(content_div))

    # fallback
    if not content:
        ps = [clean_text(p.get_text()) for p in soup.find_all('p') if len(clean_text(p.get_text())) > 20]
        content = '\n\n'.join(ps)

    return title, publish_date, source, content


# ── 入库 ──────────────────────────────────────────
def insert_article(item, detail_title, pub_date, source, content):
    """写入gov_raw + FTS gov_search"""
    page_url = item['url'].replace("'", "''")
    title = (detail_title or item['title']).replace("'", "''")
    publish_date = pub_date or item.get('date', '')
    source_name = source or SITE_NAME
    source_name = source_name.replace("'", "''")
    content = content.replace("'", "''")
    summary = title[:200]
    site_name = SITE_NAME

    sql_raw = f"""INSERT OR IGNORE INTO gov_raw 
        (page_url, title, content, publish_date, source_url, site_name, summary, group_name, industry)
    VALUES 
        ('{page_url}', '{title}', '{content}', '{publish_date}', '{page_url}', '{site_name}', '{summary}', '{GROUP}', '政府公告');
"""
    combined_sql = sql_raw + """SELECT CASE WHEN changes() > 0 THEN last_insert_rowid() ELSE 0 END;
"""
    try:
        result = subprocess.run(
            ['sqlite3', "-cmd", ".timeout 60000", DB_PATH],
            input=combined_sql,
            capture_output=True, text=True, timeout=30
        )
        if result.returncode == 0:
            out = result.stdout.strip()
            try:
                rowid = int(out.strip())
            except ValueError:
                rowid = 0
            if rowid > 0:
                fts_sql = f"""INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary)
SELECT rowid, title, site_name, summary FROM gov_raw WHERE rowid = {rowid};
"""
                subprocess.run(['sqlite3', "-cmd", ".timeout 60000", DB_PATH], input=fts_sql, capture_output=True, text=True, timeout=30)
                return True
        else:
            if 'UNIQUE constraint' not in result.stderr:
                print(f"    [DB] {result.stderr[:200]}")
    except Exception as e:
        print(f"    [DB] 错误: {e}")
    return False


# ── 主流程 ────────────────────────────────────────
def main(max_pages=TOTAL_PAGES):
    print(f"=== 九原区 - 生态环境 爬虫 ===\n")

    # 1. 获取列表
    articles = fetch_list(max_pages)
    if not articles:
        print("无文章列表")
        return
    print(f"\n列表总计: {len(articles)} 条")

    # 2. 获取已知URL
    known_urls = set()
    try:
        result = subprocess.run(
            ['sqlite3', "-cmd", ".timeout 60000", DB_PATH, "SELECT page_url FROM gov_raw WHERE page_url LIKE '%jiuyuanqu.gov.cn%'"],
            capture_output=True, text=True, timeout=30
        )
        if result.returncode == 0 and result.stdout.strip():
            known_urls = set(result.stdout.strip().split('\n'))
    except Exception:
        pass
    print(f"已知URL: {len(known_urls)}")

    # 3. 遍历详情
    added = 0
    skipped = 0
    for i, a in enumerate(articles):
        url = a['url']
        if url in known_urls:
            skipped += 1
            continue

        print(f"  [{i+1}/{len(articles)}] {a['title'][:50]}...", end=' ')
        try:
            r = session.get(url, timeout=30)
            r.encoding = 'utf-8'
        except Exception as e:
            print(f"ERR {e}")
            time.sleep(1)
            continue

        d_title, d_date, d_source, content = parse_detail(r.text, url)
        if not content.strip():
            print("SKIP 无正文")
            skipped += 1
            continue

        if insert_article(a, d_title, d_date, d_source, content):
            added += 1
            print(f"OK ({len(content.strip())}字)")
        else:
            skipped += 1
            print("SKIP (重复或入库失败)")

        time.sleep(0.3)

    print(f"\n结果: 新增 {added} 条, 跳过 {skipped} 条")
    print("=== 完成 ===")


if __name__ == '__main__':
    pages = TOTAL_PAGES
    if len(sys.argv) > 1:
        for arg in sys.argv[1:]:
            if arg.startswith('--pages='):
                try:
                    pages = int(arg.split('=')[1])
                except ValueError:
                    pass
    main(max_pages=pages)