#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
英德市-清远市生态环境局英德分局-建设项目环境影响评价信息 爬虫 (gkmlpt)
站点: www.yingde.gov.cn
栏目: 997 - 建设项目环境影响评价信息
API: /yqydsthjj/gkmlpt/api/all/997?page=N&sid=763114
"""

import sys, os, re, json, sqlite3, math, warnings, urllib.request, urllib.error
import datetime
warnings.filterwarnings('ignore')

_HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, _HERE)
sys.path.insert(0, os.path.join(_HERE, '..', 'crawler'))

from crawler_lib import fetch_page

# ─── 常量 ───
SITE_NAME = "英德市-生态环境局-建设项目环评"
DOMAIN = "www.yingde.gov.cn"
BASE_URL = "http://www.yingde.gov.cn"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

SID = "763114"
COLUMN_ID = "997"
API_TEMPLATE = f"{BASE_URL}/yqydsthjj/gkmlpt/api/all/{COLUMN_ID}?page={{page}}&sid={SID}"

# ─── DB 写入 ───
def insert_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA busy_timeout=8000")
    db.execute("PRAGMA synchronous=NORMAL")

    db.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT UNIQUE, page_url TEXT,
        title TEXT, publish_date TEXT, summary TEXT,
        content TEXT, status TEXT, category TEXT, tags TEXT,
        attachments TEXT
    )""")
    db.execute("""CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
        title, site_name, summary, content='gov_raw', content_rowid='id'
    )""")
    db.commit()

    ok = skip = 0
    for item in items:
        try:
            title = (item.get("title") or "")[:500]
            summary = (item.get("summary") or title)[:500]
            content = item.get("content") or ""
            pub_date = (item.get("publish_date") or "")[:10]
            source_url = item.get("source_url") or ""
            page_url = item.get("url") or source_url
            attachments = item.get("attachments") or ""

            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, attachments, script_name)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (
                SITE_NAME, source_url, page_url, title, pub_date,
                summary, content, "active", attachments, "crawl_yingde.py"
            ))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except Exception:
            skip += 1

    db.commit()

    try:
        db.execute("""INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r
            WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)""",
            (SITE_NAME,))
        db.commit()
    except Exception as e:
        print(f"  [FTS ERROR] {e}")

    db.close()
    print(f"  [DB] 新增: {ok}, 跳过: {skip}")
    return ok

# ─── 正文清洗 ───
def clean_content(html):
    if not html:
        return ""
    html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<!--.*?-->', '', html, flags=re.DOTALL)
    return html.strip()

def extract_text_content(html):
    """将清洗后的 HTML 转为纯文本"""
    if not html:
        return ""
    text = html
    text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</p>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</div>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</li>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</tr>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</th>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</td>', '\n', text, flags=re.IGNORECASE)
    # 保留表格标签
    text = re.sub(r'<(?!/?table|/?tr|/?td|/?th|/?thead|/?tbody|/?tfoot)[^>]*>', '', text)
    text = re.sub(r'&nbsp;', ' ', text)
    text = re.sub(r'&lt;', '<', text)
    text = re.sub(r'&gt;', '>', text)
    text = re.sub(r'&amp;', '&', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = re.sub(r'[ \t]+', ' ', text)
    # 中文间空格清理
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=[\u4e00-\u9fff])', '', text)
    return text.strip()

def extract_attachments(html, base_url=BASE_URL):
    if not html:
        return ""
    ext_pattern = r'\.(doc|docx|pdf|xls|xlsx|rar|zip)(\?[^\s"\'<>]*)?'
    links = re.findall(
        r'<a[^>]*href=["\']([^"\']*?(?:' + ext_pattern + r'))["\'][^>]*>([^<]*)</a>',
        html, re.IGNORECASE
    )
    if not links:
        return ""
    parts = []
    for href, _, _, text in links:
        full_url = href if href.startswith('http') else (base_url.rstrip('/') + '/' + href.lstrip('/'))
        parts.append(f"[附件: {text.strip()}]({full_url})")
    return "\n".join(parts)

# ─── 获取列表 ───
def fetch_list(page=1):
    url = API_TEMPLATE.format(page=page)
    try:
        req = urllib.request.Request(url)
        req.add_header("User-Agent", "Mozilla/5.0")
        resp = urllib.request.urlopen(req, timeout=30)
        data = json.loads(resp.read())
    except Exception as e:
        print(f"  ❌ API请求失败(page={page}): {e}")
        return None, 0
    articles = data.get("articles", [])
    total = data.get("total", 0)
    return articles, total

# ─── 获取详情 ───
def fetch_detail(url):
    html = fetch_page(url, timeout=30)
    if not html:
        print(f"  ⚠ 详情页获取失败: {url}")
        return None

    # 提取完整标题
    mt = re.search(r'<meta\s+name=["\']ArticleTitle["\'][^>]*content=["\']([^"\']+)["\']', html, re.IGNORECASE)
    detail_title = None
    if mt:
        detail_title = mt.group(1).strip()
    else:
        tt = re.search(r'<title>([^<]+)</title>', html, re.IGNORECASE)
        if tt:
            detail_title = tt.group(1).strip()

    # 提取内容区 .article-content
    cm = re.search(r'<div[^>]*class=["\']article-content["\'][^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if not cm:
        cm = re.search(r'<div[^>]*class=["\']content["\'][^>]*>(.*?)</div>\s*</div>\s*(?=<div[^>]*class=["\']footer|<script)', html, re.DOTALL)
    if not cm:
        print(f"  ⚠ 未找到内容容器: {url[:60]}")
        return None

    raw_html = cm.group(1)
    cleaned = clean_content(raw_html)
    attachments = extract_attachments(raw_html)
    text_content = extract_text_content(cleaned)

    if attachments:
        text_content = text_content.rstrip() + "\n\n--- 附件 ---\n" + attachments

    return {
        "title": detail_title,
        "content": text_content,
        "attachments": attachments,
    }

# ─── 日期转换 ───
def ts_to_date(ts):
    """Unix timestamp 转 YYYY-MM-DD"""
    if not ts:
        return ""
    try:
        dt = datetime.datetime.fromtimestamp(int(ts))
        return dt.strftime("%Y-%m-%d")
    except:
        return ""

# ─── 主入口 ───
def run(max_pages=None):
    print(f"\n{'='*60}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*60}")

    # Step 1: 获取第1页，确定总页数
    print("\n📥 获取列表...")
    articles, total = fetch_list(1)
    if articles is None:
        return
    page_size = len(articles) if articles else 0
    total_pages = math.ceil(total / page_size) if page_size else 1
    print(f"   总条数: {total}, 每页: {page_size}, 总页数: {total_pages}")

    if max_pages and max_pages > 0 and max_pages < total_pages:
        total_pages = max_pages
        print(f"   --pages 限制: 取前 {max_pages} 页")

    # Step 2: 获取所有页
    all_articles = list(articles) if articles else []
    for p in range(2, total_pages + 1):
        arts, _ = fetch_list(p)
        if arts:
            all_articles.extend(arts)
            print(f"   第 {p}/{total_pages} 页: {len(arts)} 条")
        else:
            print(f"   第 {p}/{total_pages} 页: 空")

    print(f"   共获取: {len(all_articles)} 条")

    # Step 3: 逐条详情
    success = fail = 0
    db_items = []

    for i, item in enumerate(all_articles, 1):
        url = item.get("url", "")
        title = item.get("title", "")
        
        # 跳过微信外部链接
        if "mp.weixin.qq.com" in url:
            print(f"\n[{i}/{len(all_articles)}] ⏭ 微信链接跳过: {title[:40]}...")
            continue
        if not url or "yingde.gov.cn" not in url:
            print(f"\n[{i}/{len(all_articles)}] ⏭ 外部链接跳过: {title[:40]}...")
            continue

        print(f"\n[{i}/{len(all_articles)}] {title[:50]}...")

        detail = fetch_detail(url)
        if detail is None:
            fail += 1
            continue

        pub_date = ts_to_date(item.get("create_time"))
        db_title = detail.get("title") or title
        summary = db_title[:200]
        content_text = detail["content"]

        db_items.append({
            "title": db_title,
            "source_url": url,
            "url": url,
            "publish_date": pub_date,
            "summary": summary,
            "content": content_text,
            "attachments": detail.get("attachments", ""),
        })
        success += 1
        print(f"   ✅ 正文: {len(content_text)} 字, 日期: {pub_date}")

    # Step 4: 写入DB
    print(f"\n{'='*60}")
    print(f"📦 写入数据库... 成功: {success}, 失败: {fail}, 跳过: {len(all_articles)-success-fail}")
    if db_items:
        insert_db(db_items)
    print(f"{'='*60}")
    print(f"✅ 完成! 共处理 {len(all_articles)} 条，成功 {success}，失败 {fail}")


if __name__ == "__main__":
    max_p = None
    args = sys.argv[1:]
    for i, arg in enumerate(args):
        if arg == "--pages" and i + 1 < len(args):
            try:
                max_p = int(args[i + 1])
            except:
                pass
    if max_p is None:
        for arg in args:
            try:
                max_p = int(arg.lstrip('-'))
                break
            except:
                pass
    run(max_pages=max_p)