#!/usr/bin/env python3
"""
宣州区人民政府 — 基层政务公开-生态环境 爬虫
site: www.xuanzhou.gov.cn
栏目: /Jczwgk/showList/0/111000000/page_N.html
"""
import re, sys, os, sqlite3, time, urllib.request, ssl

SITE_NAME = "宣州区-生态环境"
DOMAIN = "www.xuanzhou.gov.cn"
BASE_URL = "https://www.xuanzhou.gov.cn"
LIST_TEMPLATE = "/Jczwgk/showList/0/111000000/page_{page}.html"
TOTAL_PAGES = 79  # 1185条，15条/页

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CTX = ssl._create_unverified_context()
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"
}

def http_get(url):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=30, context=CTX)
        return resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        return None

def parse_list_page(html):
    """解析列表页：<li><span>日期</span><a href=...>标题</a></li>"""
    items = []
    # 提取列表区域
    m_area = re.search(r'class="m-liststyle2"[^>]*>(.*?)</div>\s*</div>\s*<script', html, re.DOTALL)
    area = m_area.group(1) if m_area else html

    for m in re.finditer(
        r'<li>\s*<span>([^<]*)</span>\s*<a href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>\s*</li>',
        area, re.DOTALL
    ):
        date = m.group(1).strip()
        href = m.group(2).strip()
        title = m.group(3).strip() or re.sub(r'<[^>]+>', '', m.group(4)).strip()
        if not href.startswith("http"):
            href = BASE_URL + href
        items.append({"title": title, "url": href, "publish_date": date[:10]})

    return items

def fetch_detail(url):
    """获取详情：标题、正文HTML、日期"""
    html = http_get(url)
    if not html:
        return None

    # 标题
    title = ""
    m_t = re.search(r'<meta name="ArticleTitle" content="([^"]+)"', html)
    if not m_t:
        m_t = re.search(r'<title>([^<]+)--标准化规范化', html)
    if m_t:
        title = m_t.group(1).strip()

    # 日期
    date = ""
    m_d = re.search(r'<meta name="PubDate" content="([^"]+)"', html)
    if m_d:
        date = m_d.group(1).strip()[:10]

    # 正文
    content = ""
    m_c = re.search(
        r'<div class="m-dttexts f-clearfix j-fontContent" id="zoom">(.*?)</div>\s*</div>',
        html, re.DOTALL
    )
    if m_c:
        content = m_c.group(1).strip()
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<div class="m-related-info">.*?</div>', '', content, flags=re.DOTALL)
        content = re.sub(r'<div class="m-share">.*?</div>', '', content, flags=re.DOTALL)
        content = content.strip()

    return {
        "title": title or "",
        "content": content,
        "date": date,
    }

def get_total_pages(html):
    """从pagination标签获取总页数"""
    m = re.search(r'pagecount="(\d+)"', html)
    if m:
        return int(m.group(1))
    return TOTAL_PAGES

def init_db():
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT, page_url TEXT,
        title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0,
        summary TEXT, status TEXT, category TEXT DEFAULT '',
        visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT ''
    )''')
    try:
        conn.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
            title, content, site_name,
            content='gov_raw', content_rowid='id', tokenize='unicode61'
        )''')
    except sqlite3.OperationalError:
        pass
    conn.commit()
    conn.close()

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--full", action="store_true", help="全量53页")
    parser.add_argument("--pages", type=int, default=1, help="指定页数")
    args = parser.parse_args()

    init_db()
    pages_to_crawl = TOTAL_PAGES if args.full else args.pages
    mode = "全量" if args.full else f"增量({pages_to_crawl}页)"

    print(f"🔍 {SITE_NAME} — {mode}模式")
    print(f"  总页数: {TOTAL_PAGES}, 本次爬取: {pages_to_crawl}页")
    print()

    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    cur = conn.cursor()

    new_total = 0
    skip_total = 0
    for page in range(1, pages_to_crawl + 1):
        list_url = BASE_URL + LIST_TEMPLATE.format(page=page)
        html = http_get(list_url)
        if not html:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ HTTP失败")
            continue

        items = parse_list_page(html)
        if not items:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ 未解析到条目")
            continue

        page_new = 0
        page_skip = 0
        for item in items:
            exists = cur.execute(
                "SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?",
                (item["url"], SITE_NAME)
            ).fetchone()
            if exists:
                page_skip += 1
                skip_total += 1
                continue

            detail = fetch_detail(item["url"])
            if detail:
                content = detail.get("content", "")
                title = detail.get("title") or item["title"]
                date = detail.get("date", item.get("publish_date", ""))
            else:
                content = ""
                title = item["title"]
                date = item.get("publish_date", "")

            if not content:
                page_skip += 1
                skip_total += 1
                continue

            try:
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw "
                    "(title, content, publish_date, page_url, source_url, site_name, status) "
                    "VALUES (?,?,?,?,?,?,?)",
                    (title, content, date, item["url"], DOMAIN, SITE_NAME, "published")
                )
                if cur.rowcount > 0:
                    row_id = cur.lastrowid
                    try:
                        # gov_search FTS5 列: (title, site_name, summary)，无 content 列
                        plain = re.sub(r"<[^>]+>", " ", content)
                        plain = re.sub(r"\s+", " ", plain).strip()[:200] or title
                        cur.execute(
                            "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?,?,?,?)",
                            (row_id, title, SITE_NAME, plain)
                        )
                    except Exception:
                        pass
                    page_new += 1
                    new_total += 1
            except Exception as e:
                pass

        conn.commit()

        # 检测最后几页的3年截止
        print(f"  [p{page:2d}/{pages_to_crawl}] ✓ {len(items)}条 (新增{page_new} 跳过{page_skip})")

        # 如果这一页所有条目都超过3年，停止
        all_old = True
        for item in items:
            d = item.get("publish_date", "")
            if d and d >= "2023-06-19":
                all_old = False
                break
        if all_old and page > 1:
            print(f"  ⏹ 超过3年时间范围，停止")
            break

        time.sleep(0.3)

    conn.close()
    print(f"\n📊 完成！新增: {new_total} | 跳过: {skip_total}")

if __name__ == "__main__":
    main()