#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
薛城区人民政府-审批信息 (枣庄市生态环境局薛城分局) 爬虫
站点: www.xuecheng.gov.cn
"""
import sys, os, re, sqlite3, warnings, requests
from bs4 import BeautifulSoup
import urllib.parse
warnings.filterwarnings('ignore')

_HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, _HERE)
sys.path.insert(0, os.path.join(_HERE, '..', 'crawler'))

SITE_NAME = "薛城区-审批信息"
DOMAIN = "www.xuecheng.gov.cn"
BASE_URL = "http://www.xuecheng.gov.cn"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
PAGE_SIZE = 20

session = requests.Session()
session.headers.update({
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
})


def insert_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA busy_timeout=8000")
    db.execute("PRAGMA synchronous=NORMAL")
    db.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT UNIQUE, page_url TEXT,
        title TEXT, publish_date TEXT, summary TEXT,
        content TEXT, status TEXT, category TEXT, tags TEXT,
        attachments TEXT
    )""")
    db.execute("""CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
        title, site_name, summary, content='gov_raw', content_rowid='id'
    )""")
    db.commit()
    ok = skip = 0
    for item in items:
        try:
            title = (item.get("title") or "")[:500]
            summary = (item.get("summary") or title)[:500]
            content = item.get("content") or ""
            pub_date = (item.get("publish_date") or "")[:10]
            source_url = item.get("source_url") or ""
            page_url = item.get("url") or source_url
            attachments = item.get("attachments") or ""
            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, attachments)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, source_url, page_url, title, pub_date,
                 summary, content, "active", attachments))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    try:
        db.execute("""INSERT INTO gov_search(rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r
            WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)""",
            (SITE_NAME,))
        db.commit()
    except Exception as e:
        print(f"  [FTS ERROR] {e}")
    db.close()
    print(f"  [DB] 新增: {ok}, 跳过: {skip}")


def clean_html(html):
    html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<!--.*?-->', '', html, flags=re.DOTALL)
    return html.strip()


def extract_text(html):
    if not html:
        return ""
    text = html
    # 表格保留为HTML
    tables = re.findall(r'<table[^>]*>.*?</table>', text, re.DOTALL | re.IGNORECASE)
    placeholders = []
    for i, tbl in enumerate(tables):
        ph = f'__TABLE_{i}__'
        placeholders.append((ph, tbl))
        text = text.replace(tbl, ph, 1)
    text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</p>', '\n\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</div>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</tr>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</th>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</td>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'<(?!/?(?:table|tr|td|th|thead|tbody|tfoot|img))[^>]*>', '', text)
    text = re.sub(r'&nbsp;', ' ', text)
    text = re.sub(r'&lt;', '<', text)
    text = re.sub(r'&gt;', '>', text)
    text = re.sub(r'&amp;', '&', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t]+(?=[\u4e00-\u9fff])', '', text)
    # 恢复表格为Markdown
    for ph, tbl in placeholders:
        md = html_table_to_html(tbl)
        text = text.replace(ph, '\n' + md + '\n')
    return text.strip()


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def fetch_list_page(page_num):
    """获取列表页，返回文章列表"""
    url = f"{BASE_URL}/govsearch/searPageNewXueCheng.jsp?siteid=44&classinfoid=974"
    data = {"page": str(page_num)}
    try:
        r = session.post(url, data=data, timeout=20)
        r.encoding = 'utf-8'
        html = r.text
    except Exception as e:
        print(f"  [ERROR] 请求失败: {e}")
        return []

    items = []
    # 解析 <li> 内容
    for m in re.finditer(r'<li[^>]*>(.*?)</li>', html, re.DOTALL):
        li = m.group(1)
        # 找链接
        link = re.search(r'<a[^>]*href="([^"]*)"[^>]*title="([^"]*)"[^>]*>', li)
        if not link:
            continue
        href = link.group(1)
        title = link.group(2).strip()
        # 找日期
        date_m = re.search(r'<b[^>]*>(\d{4}年\d{1,2}月\d{1,2}日)</b>', li)
        if not date_m:
            date_m = re.search(r'(\d{4}-\d{2}-\d{2})', li)
        date_str = ""
        if date_m:
            raw_date = date_m.group(1)
            # 标准化日期
            m2 = re.match(r'(\d{4})年(\d{1,2})月(\d{1,2})日', raw_date)
            if m2:
                date_str = f"{m2.group(1)}-{m2.group(2).zfill(2)}-{m2.group(3).zfill(2)}"
            else:
                date_str = raw_date

        items.append({
            "title": title,
            "url": href if href.startswith("http") else BASE_URL + href,
            "date": date_str,
        })
    return items


def extract_content_from_detail(html):
    """从详情页提取正文"""
    # TRS_UEDITOR 正文容器
    m = re.search(r'<div[^>]*class="[^"]*(?:TRS_UEDITOR|trs_paper|trs_web)[^"]*"[^>]*>(.*?)</div>\s*</div>\s*</div>', html, re.DOTALL | re.IGNORECASE)
    if not m:
        m = re.search(r'<div[^>]*class="[^"]*(?:TRS_UEDITOR|trs_paper|trs_web)[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL | re.IGNORECASE)
    if m:
        body_html = m.group(1)
        text = extract_text(clean_html(body_html))
        if len(text) > 10:
            return text.strip()

    # 兜底：zwnr
    m = re.search(r'<div[^>]*class="[^"]*(?:zwnr|detail-page|article-content)[^"]*"[^>]*>(.*?)(?:</div>\s*){1,3}(?:<div[^>]*class="[^"]*(?:foot|footer|share|print)', html, re.DOTALL | re.IGNORECASE)
    if m:
        text = extract_text(clean_html(m.group(1)))
        if len(text) > 10:
            return text.strip()

    # 再兜底：body
    bm = re.search(r'<body[^>]*>(.*?)</body>', html, re.DOTALL)
    if bm:
        body = bm.group(1)
        cleaned = clean_html(body)
        text = extract_text(cleaned)
        lines = [l.strip() for l in text.split('\n') if l.strip() and len(l.strip()) > 4]
        lines = [l for l in lines if not any(x in l for x in [
            '首页', '信息公开', '政务公开', '网站地图', '联系我们', '版权',
            '无障碍', '登录', '注册',
        ])]
        if lines:
            return '\n\n'.join(lines)
    return ""


def fetch_detail(detail_url):
    """获取详情页"""
    try:
        r = session.get(detail_url, timeout=20)
        r.encoding = 'utf-8'
        html = r.text
    except Exception as e:
        return None

    # 标题
    title = None
    tt = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if tt:
        title = tt.group(1).strip()
    if not title:
        tt = re.search(r'<title>([^<]+)</title>', html)
        if tt:
            t = tt.group(1).strip()
            t = re.sub(r'[-–—]\s*薛城区政府信息公开$', '', t)
            t = t.strip()
            if t:
                title = t

    # 日期
    date_str = ""
    dp = re.search(r'<meta[^>]*name="PubDate"[^>]*content="([^"]*)"', html)
    if dp:
        date_str = dp.group(1)[:10]

    content = extract_content_from_detail(html)
    return {"title": title, "content": content, "date": date_str}


def run(max_pages=None):
    print(f"\n{'='*60}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*60}")

    # 先获取第1页，确定总记录数
    print("\n📥 获取列表页1...")
    items1 = fetch_list_page(1)
    if not items1:
        print("  ❌ 列表页无数据")
        return
    print(f"  第1页: {len(items1)} 条")

    # 从页1数据推断总页数
    total_records = 401  # 已知
    total_pages = (total_records + PAGE_SIZE - 1) // PAGE_SIZE
    if max_pages and max_pages < total_pages:
        total_pages = max_pages
    print(f"  总记录: {total_records}, 总页数: {total_pages}")

    # 收集所有条目
    all_items = list(items1)
    for pn in range(2, total_pages + 1):
        items = fetch_list_page(pn)
        if not items:
            print(f"  第 {pn}/{total_pages} 页: 空")
            continue
        seen_urls = {i["url"] for i in all_items}
        new_items = [i for i in items if i["url"] not in seen_urls]
        all_items.extend(new_items)
        print(f"  第 {pn}/{total_pages} 页: {len(items)} 条 (新增 {len(new_items)})")

    print(f"\n  共获取: {len(all_items)} 条")

    # 逐条详情
    success = fail = 0
    db_items = []

    for i, item in enumerate(all_items, 1):
        print(f"\n[{i}/{len(all_items)}] {item['title'][:60]}...")
        detail = fetch_detail(item["url"])
        if not detail:
            print(f"   ❌ 请求失败")
            fail += 1
            continue

        content_text = detail.get("content", "")
        if not content_text or len(content_text) < 5:
            print(f"   ❌ 正文为空")
            fail += 1
            continue

        db_title = detail.get("title") or item["title"]
        pub_date = detail.get("date") or item["date"]

        db_items.append({
            "title": db_title,
            "source_url": item["url"],
            "url": item["url"],
            "publish_date": pub_date[:10],
            "summary": db_title[:200],
            "content": content_text,
            "attachments": "",
        })
        success += 1
        print(f"   ✅ 正文: {len(content_text)} 字, 日期: {pub_date}")

    print(f"\n{'='*60}")
    print(f"📦 写入数据库... 成功: {success}, 失败: {fail}")
    if db_items:
        insert_db(db_items)
    print(f"{'='*60}")
    print(f"✅ 完成! 共处理 {len(all_items)} 条，成功 {success}，失败 {fail}")


if __name__ == "__main__":
    max_p = None
    args = sys.argv[1:]
    for i, arg in enumerate(args):
        if arg == "--pages" and i + 1 < len(args):
            try:
                max_p = int(args[i + 1])
            except:
                pass
    run(max_pages=max_p)