#!/usr/bin/env python3
"""Crawler for 科尔沁区-生态环境 (keerqin.gov.cn /zwgk/zfxxgk/fdzdgknr/zdlyxx/sthj/)
   CMS: TRS 信息公开, pagination: index_{N}.html (50 pages, ~509 records)
   Detail: div.lis_list_part2_content or div.trs_editor_view
"""
import urllib.request, ssl, re, sqlite3, sys, time
from datetime import datetime

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE

BASE = "http://www.keerqin.gov.cn"
LIST_DIR = "/zwgk/zfxxgk/fdzdgknr/zdlyxx/sthj/"
DB = "/root/search.db"

SITE_NAME = "keerqin_sthj"
MAX_PAGES = 51  # index.html (0) + index_1..index_50
INCREMENTAL = "--incremental" in sys.argv


def log(msg):
    print(f"[{SITE_NAME}] {msg}")


def fetch(url):
    req = urllib.request.Request(url, headers={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    })
    r = urllib.request.urlopen(req, context=ssl_ctx, timeout=30)
    return r.read().decode("utf-8", errors="replace")


def parse_list(html, page_num):
    """解析 tbody > tr > td > a[href] + td date"""
    items = []
    tbody_match = re.search(r"<tbody>(.*?)</tbody>", html, re.DOTALL)
    if not tbody_match:
        log(f"第{page_num}页未找到 tbody")
        return items
    tbody_html = tbody_match.group(1)
    for tr_match in re.finditer(r"<tr>(.*?)</tr>", tbody_html, re.DOTALL):
        tr = tr_match.group(1)
        a_match = re.search(r'href="([^"]+)"[^>]*>([^<]+)', tr)
        if not a_match:
            continue
        href = a_match.group(1).strip()
        title = a_match.group(2).strip()
        # Date is in the last <td> before </tr>
        tds = re.findall(r"<td[^>]*>([^<]*)</td>", tr)
        date_str = tds[-1].strip() if tds else ""
        if not href.startswith("http"):
            href = BASE + href
        items.append({
            "title": title,
            "url": href,
            "date": date_str,
        })
    return items


def get_page_url(page_num):
    """page 0 = index.html, page 1..50 = index_N.html"""
    if page_num == 0:
        return f"{BASE}{LIST_DIR}index.html"
    return f"{BASE}{LIST_DIR}index_{page_num}.html"


def fetch_detail(url):
    """正文：div.lis_list_part2_content 或 div.trs_editor_view"""
    html = fetch(url)
    for pattern in [
        r'<div[^>]*class="lis_list_part2_content"[^>]*>(.*?)</div>',
        r'<div[^>]*class="trs_editor_view[^"]*"[^>]*>(.*?)</div>',
        r'<div[^>]*class="cont_cont"[^>]*>(.*?)</div>',
        r'<div[^>]*class="banxin[^"]*bbg"[^>]*>(.*?)</div>',
    ]:
        m = re.search(pattern, html, re.DOTALL)
        if m:
            inner = m.group(1)
            parts = []
            for p in re.findall(r"<p[^>]*>(.*?)</p>", inner, re.DOTALL):
                txt = re.sub(r"<[^>]+>", "", p).strip()
                if txt:
                    parts.append(txt)
            if not parts:
                txt = re.sub(r"<[^>]+>", "", inner).strip()
                if txt:
                    parts = [txt]
            if parts:
                return {"content": "\n".join(parts)}
    log(f"未找到正文: {url}")
    return None


def main():
    conn = sqlite3.connect(DB)
    cur = conn.cursor()
    total_new = 0
    
    page_end = min(MAX_PAGES - 1, 4) if INCREMENTAL else MAX_PAGES - 1
    
    for page_num in range(0, page_end + 1):
        url = get_page_url(page_num)
        log(f"抓取列表页 {page_num + 1}/{page_end + 1}: {url}")
        try:
            html = fetch(url)
        except Exception as e:
            log(f"列表页 {page_num} 失败: {e}")
            continue
        items = parse_list(html, page_num)
        log(f"  解析到 {len(items)} 条")
        
        for item in items:
            cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
            if cur.fetchone():
                if INCREMENTAL:
                    log(f"  已有记录，增量结束")
                    conn.close()
                    log(f"增量完成，共 {total_new} 条新增")
                    return
                continue
            
            detail = fetch_detail(item["url"])
            if detail is None:
                continue
            
            content = detail["content"]
            now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            cur.execute(
                """INSERT OR IGNORE INTO gov_raw 
                (site_name, source_url, page_url, title, publish_date, summary, content, status, category)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (
                    SITE_NAME,
                    item["url"],
                    item["url"],
                    item["title"],
                    item["date"],
                    content[:200],
                    content,
                    "published",
                    "生态环境",
                )
            )
            if cur.rowcount > 0:
                total_new += 1
                if total_new % 10 == 0:
                    conn.commit()
                    log(f"  已入库 {total_new} 条...")
        
        conn.commit()
        log(f"第{page_num + 1}页完成，累计 {total_new} 条")
        time.sleep(0.3)
    
    conn.close()
    log(f"全量爬取完成，共新增 {total_new} 条")


if __name__ == "__main__":
    main()