#!/usr/bin/env python3
"""
张掖市生态环境局 - 环评管理 (www.zhangye.gov.cn/hbj/ztzl/hpgl/)
CMS: TRS (拓尔思)，静态分页: index.html / index_{N}.html
列表: ul.news_list > li > a[href] + span 日期
详情: meta[ArticleTitle], meta[PubDate], div.view.TRS_UEDITOR 正文
"""
import sys, os, re, time, json, requests, sqlite3
from datetime import datetime, timedelta
from urllib.parse import urljoin

SITE_NAME = "张掖市-环评管理"
BASE_URL = "https://www.zhangye.gov.cn/hbj/ztzl/hpgl/"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 5
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  ⚠ 请求失败: {e}")
        return ""

def parse_list(html):
    """解析列表页"""
    items = []
    # 定位 ul.news_list
    m = re.search(r'<ul[^>]*class="news_list"[^>]*>(.*?)</ul>', html, re.DOTALL)
    if not m:
        return items
    ul_html = m.group(1)
    for li in re.finditer(r'<li>(.*?)</li>', ul_html, re.DOTALL):
        li_html = li.group(1)
        a_href = re.search(r'href="([^"]+)"', li_html)
        a_title = re.search(r'title="([^"]*)"', li_html)
        title = ""
        if a_title:
            title = a_title.group(1).strip()
        if not title:
            # 从 <a> 标签文本取标题（去掉 <TRS_DOCUMENT> 标签和 <span>）
            title_clean = re.sub(r'<TRS_DOCUMENT>[^<]*</TRS_DOCUMENT>', '', li_html)
            title_clean = re.sub(r'<span[^>]*>.*?</span>', '', title_clean)
            title_clean = re.sub(r'<[^>]+>', '', title_clean).strip()
            title = title_clean
        if not title:
            continue
        href = a_href.group(1) if a_href else ""
        if not href:
            continue
        full_url = urljoin(BASE_URL, href)
        # 日期
        date_m = re.search(r'<span[^>]*>(\d{4}-\d{2}-\d{2})</span>', li_html)
        date_str = date_m.group(1) if date_m else ""
        items.append({"title": title, "url": full_url, "date": date_str})
    return items

def fetch_detail(url):
    """抓详情页，返回正文HTML"""
    html = fetch(url)
    if not html:
        return "", "", ""
    
    # 标题 from meta
    title = ""
    m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()
    if not title:
        m = re.search(r'<title>([^<]*)</title>', html)
        if m:
            title = m.group(1).strip()
            # 去掉站点名前缀
            title = re.sub(r'^[^-]*-', '', title).strip()
    
    # 日期 from meta
    date_str = ""
    m = re.search(r'<meta\s+name="PubDate"\s+content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        date_str = m.group(1)
    
    # 正文: div.view.TRS_UEDITOR
    content = ""
    m = re.search(r'<div[^>]*class="view\s+TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    if not content:
        m = re.search(r'<div[^>]*class="[^"]*TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    if not content or len(content) < 100:
        # 兜底: 试试 news_content
        m = re.search(r'<div[^>]*class="news_content"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    
    # 清理内联样式标签
    if content:
        content = re.sub(r'</?(?:span|font|strong|b|em|i|u|a|s|style|o:p)\b[^>]*>', '', content, flags=re.I)
        # 保留 <p> <table> <br> <img>
        content = re.sub(r'<p\s[^>]*>', '<p>', content)
        content = re.sub(r'<br\s*/?>', '\n', content)
        content = re.sub(r'&nbsp;', ' ', content)
        content = re.sub(r'\n\s*\n+', '\n\n', content)
    
    return title, date_str, content

def insert_db(items):
    """入库"""
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for item in items:
        try:
            # 检查是否已存在
            c.execute("SELECT page_url FROM gov_raw WHERE page_url=?", (item["url"],))
            if c.fetchone():
                continue
            clean_text = re.sub(r'<[^>]+>', '', item["content"]).strip()
            if not clean_text or len(clean_text) < 50:
                print(f"  ⚠ 正文过短或为空: {item['title'][:30]}...")
                continue
            c.execute("""INSERT OR IGNORE INTO gov_raw
                (title, page_url, site_name, summary, content, publish_date)
                VALUES (?, ?, ?, ?, ?, ?)""",
                (item["title"], item["url"], SITE_NAME,
                 clean_text[:500], item["content"], item["date"]))
            if c.rowcount > 0:
                inserted += 1
                # 同步写入 FTS
        except Exception as e:
            print(f"  ⚠ 入库失败: {e}")
    conn.commit()
    conn.close()
    return inserted

def main():
    is_incremental = len(sys.argv) >= 2
    max_pages = int(sys.argv[1]) if is_incremental else MAX_PAGES
    print(f"=== {SITE_NAME} === {'增量' if is_incremental else '全量'} 模式, 最多{max_pages}页")
    
    all_items = []
    total_pages = 17  # 已知17页
    if max_pages < total_pages:
        total_pages = max_pages
    
    for page in range(total_pages):
        if page == 0:
            url = BASE_URL
        else:
            url = f"{BASE_URL}index_{page}.html"
        print(f"  列表页 {page+1}/{total_pages}: {url}")
        html = fetch(url)
        if not html:
            print(f"    ⚠ 获取失败，停止翻页")
            break
        items = parse_list(html)
        if not items:
            print(f"    ⚠ 列表为空，停止翻页")
            break
        print(f"    → {len(items)} 条")
        all_items.extend(items)
        time.sleep(0.5)
    
    print(f"\n列表总计: {len(all_items)} 条")
    
    # 日期过滤
    filtered = [it for it in all_items if it["date"] >= CUTOFF]
    print(f"日期过滤后: {len(filtered)} 条 (截止 {CUTOFF})")
    
    # 去重 + 抓详情
    conn = sqlite3.connect(DB_PATH, timeout=60)
    existing = set()
    for row in conn.execute("SELECT page_url FROM gov_raw"):
        existing.add(row[0])
    conn.close()
    
    to_fetch = [it for it in filtered if it["url"] not in existing]
    print(f"需抓详情: {len(to_fetch)} 条")
    
    success = []
    for i, item in enumerate(to_fetch):
        print(f"  [{i+1}/{len(to_fetch)}] {item['title'][:40]}...", end=" ", flush=True)
        title, date_str, content = fetch_detail(item["url"])
        if not content or len(content) < 100:
            print(f"⚠ 内容为空")
            continue
        # 使用详情页更准确的标题和日期
        success.append({
            "title": title or item["title"],
            "url": item["url"],
            "date": date_str or item["date"],
            "content": content,
        })
        print(f"✅ {len(content)}字节")
        time.sleep(0.3)
    
    inserted = insert_db(success)
    print(f"\n=== 完成! 新增 {inserted} 条 ===")

if __name__ == "__main__":
    main()