#!/usr/bin/env python3
"""
莒县人民政府 — 环评信息 爬虫
site: www.juxian.gov.cn
栏目: /col/col310134/index.html?number=JU004002
列表: JS jpage 分页 → POST /module/web/jpage/dataproxy.jsp
      body: startrecord/endrecord/perpage/unitid/webid/path/webname/columnid/permissiontype
      845条, 10条/页, 85页, <datastore><totalrecord>845</totalrecord>
详情: /art/2026/{m}/{d}/art_310134_{id}.html
详情结构: meta ArticleTitle, div#zoom(正文)
"""
import re, sys, os, sqlite3, time, urllib.request, ssl, urllib.parse

SITE_NAME = "莒县-环评信息"
DOMAIN = "www.juxian.gov.cn"
BASE_URL = "http://www.juxian.gov.cn"
PROXY_URL = BASE_URL + "/module/web/jpage/dataproxy.jsp"
PROXY_PARAMS = {
    "perpage": "10",
    "unitid": "619813",
    "webid": "173",
    "path": "http://www.juxian.gov.cn/",
    "webname": "%E8%8E%92%E5%8E%BF%E4%BA%BA%E6%B0%91%E6%94%BF%E5%BA%9C",
    "columnid": "310134",
    "permissiontype": "0",
}
TOTAL_RECORDS = 845  # 85页

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CTX = ssl._create_unverified_context()
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"
}

def http_post(url, data):
    body = urllib.parse.urlencode(data).encode("utf-8")
    req = urllib.request.Request(url, data=body, headers={
        "User-Agent": HEADERS["User-Agent"],
        "Content-Type": "application/x-www-form-urlencoded",
    })
    try:
        resp = urllib.request.urlopen(req, timeout=30, context=CTX)
        return resp.read().decode("utf-8", errors="replace")
    except Exception:
        return None

def http_get(url):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=30, context=CTX)
        return resp.read().decode("utf-8", errors="replace")
    except Exception:
        return None

def fetch_list_page(page):
    """调 dataproxy 拿一页列表 → [{title, url, pub_date}]
    语义: page=N 返回从第N页开始的大块, 取前10条即为该页"""
    params = dict(PROXY_PARAMS)
    params["startrecord"] = "1"
    params["endrecord"] = "10"
    params["page"] = str(page)
    xml = http_post(PROXY_URL, params)
    if not xml:
        return []
    items = []
    # CDATA 块里的 <li>...</li>，只取前 10 条（每页 10 条）
    for m in re.finditer(r'<li>(.*?)</li>', xml, re.DOTALL):
        li = m.group(1)
        a = re.search(r'href="([^"]+)"[^>]*title="([^"]*)"', li)
        d = re.search(r'(\d{4}-\d{2}-\d{2})', li)
        if a:
            items.append({
                "title": a.group(2).strip(),
                "url": a.group(1).strip(),
                "publish_date": d.group(1) if d else "",
            })
        if len(items) >= 10:
            break
    return items

def fetch_detail(url):
    """获取详情：标题、正文HTML、日期"""
    html = http_get(url)
    if not html:
        return None

    title = ""
    m_t = re.search(r'<meta name="ArticleTitle" content="([^"]+)"', html)
    if m_t:
        title = m_t.group(1).strip()

    content = ""
    m_c = re.search(r'<div id="zoom"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m_c:
        content = m_c.group(1).strip()
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<!--.*?-->', '', content, flags=re.DOTALL)
        content = content.strip()

    return {"title": title or "", "content": content, "date": ""}

def init_db():
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT, page_url TEXT,
        title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0,
        summary TEXT, status TEXT, category TEXT DEFAULT '',
        visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT ''
    )''')
    try:
        conn.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
            title, content, site_name,
            content='gov_raw', content_rowid='id', tokenize='unicode61'
        )''')
    except sqlite3.OperationalError:
        pass
    conn.commit()
    conn.close()

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--full", action="store_true", help="全量85页")
    parser.add_argument("--pages", type=int, default=1, help="指定页数")
    args = parser.parse_args()

    init_db()
    total_pages = (TOTAL_RECORDS + 9) // 10
    pages_to_crawl = total_pages if args.full else args.pages
    mode = "全量" if args.full else f"增量({pages_to_crawl}页)"

    print(f"🔍 {SITE_NAME} — {mode}模式")
    print(f"  总条数: {TOTAL_RECORDS}, 本次爬取: {pages_to_crawl}页")
    print()

    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    cur = conn.cursor()

    new_total = 0
    skip_total = 0
    for page in range(1, pages_to_crawl + 1):
        items = fetch_list_page(page)
        if not items:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ API无数据")
            continue

        page_new = 0
        page_skip = 0
        for item in items:
            exists = cur.execute(
                "SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?",
                (item["url"], SITE_NAME)
            ).fetchone()
            if exists:
                page_skip += 1
                skip_total += 1
                continue

            detail = fetch_detail(item["url"])
            if detail:
                content = detail.get("content", "")
                title = detail.get("title") or item["title"]
                date = detail.get("date", item.get("publish_date", ""))
            else:
                content = ""
                title = item["title"]
                date = item.get("publish_date", "")

            if not content:
                page_skip += 1
                skip_total += 1
                continue

            try:
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw "
                    "(title, content, publish_date, page_url, source_url, site_name, status) "
                    "VALUES (?,?,?,?,?,?,?)",
                    (title, content, date, item["url"], DOMAIN, SITE_NAME, "published")
                )
                if cur.rowcount > 0:
                    row_id = cur.lastrowid
                    try:
                        plain = re.sub(r"<[^>]+>", " ", content)
                        plain = re.sub(r"\s+", " ", plain).strip()[:200] or title
                        cur.execute(
                            "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?,?,?,?)",
                            (row_id, title, SITE_NAME, plain)
                        )
                    except Exception:
                        pass
                    page_new += 1
                    new_total += 1
            except Exception:
                pass

        conn.commit()
        print(f"  [p{page:2d}/{pages_to_crawl}] ✓ {len(items)}条 (新增{page_new} 跳过{page_skip})")
        time.sleep(0.3)

    conn.close()
    print(f"\n📊 完成！新增: {new_total} | 跳过: {skip_total}")

if __name__ == "__main__":
    main()