#!/usr/bin/env python3
"""Crawler for 佛山市三水区-公告公示 (ss.gov.cn /gzjg/ssqdtz/gggs/)
   CMS: 自定义, pagination: index_{N}.html (33 pages, 654 records)
   Detail: div.m-con.zhengwen
"""
import urllib.request, ssl, re, sqlite3, sys, time
from datetime import datetime

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE

BASE = "https://www.ss.gov.cn"
LIST_DIR = "/gzjg/ssqdtz/gggs/"
DB = "/root/search.db"

SITE_NAME = "ss_gonggao"
MAX_PAGES = 33  # Math.ceil(654/20)
ITEMS_PER_PAGE = 20
INCREMENTAL = "--incremental" in sys.argv


def log(msg):
    print(f"[{SITE_NAME}] {msg}")


def fetch(url):
    req = urllib.request.Request(url, headers={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    })
    r = urllib.request.urlopen(req, context=ssl_ctx, timeout=30)
    return r.read().decode("utf-8", errors="replace")


def parse_list(html, page_num):
    """解析 ul.in-common-list > li > a[href][title] + span.fr 日期"""
    items = []
    ul_match = re.search(r'<ul[^>]*class="in-common-list[^"]*"[^>]*>(.*?)</ul>', html, re.DOTALL)
    if not ul_match:
        log(f"第{page_num}页未找到列表ul")
        return items
    ul_html = ul_match.group(1)
    for li_match in re.finditer(r'<li[^>]*>(.*?)</li>', ul_html, re.DOTALL):
        li = li_match.group(1)
        a_href = re.search(r'href="([^"]+)"', li)
        a_title = re.search(r'title="([^"]*)"', li)
        if not (a_href and a_title):
            continue
        href = a_href.group(1).strip()
        title = a_title.group(1).strip()
        # 日期
        span_match = re.search(r'<span[^>]*class="fr"[^>]*>([^<]*)</span>', li)
        date_str = span_match.group(1).strip() if span_match else ""
        if not href.startswith("http"):
            href = BASE + href
        items.append({
            "title": title,
            "url": href,
            "date": date_str,
        })
    return items


def get_page_url(page_num):
    if page_num == 1:
        return f"{BASE}{LIST_DIR}index.html"
    return f"{BASE}{LIST_DIR}index_{page_num}.html"


def fetch_detail(url):
    """正文：div.m-con.zhengwen 或 div.innerBox"""
    html = fetch(url)
    # Try m-con.zhengwen first (actual article content)
    for pattern in [
        r'<div[^>]*class="m-con[^"]*zhengwen[^"]*"[^>]*>(.*?)</div>',
        r'<div[^>]*class="content[^"]*innerBox"[^>]*>(.*?)</div>',
        r'<div[^>]*class="news-view"[^>]*>(.*?)</div>',
        r'<div[^>]*class="content[^"]*Box"[^>]*>(.*?)</div>',
    ]:
        m = re.search(pattern, html, re.DOTALL)
        if m:
            inner = m.group(1)
            parts = []
            for p in re.findall(r'<p[^>]*>(.*?)</p>', inner, re.DOTALL):
                txt = re.sub(r'<[^>]+>', '', p).strip()
                if txt:
                    parts.append(txt)
            if not parts:
                txt = re.sub(r'<[^>]+>', '', inner).strip()
                if txt:
                    parts = [txt]
            if parts:
                return {"content": "\n".join(parts)}
    log(f"未找到正文: {url}")
    return None


def main():
    conn = sqlite3.connect(DB, timeout=60)
    cur = conn.cursor()
    total_new = 0
    
    page_end = min(MAX_PAGES, 5) if INCREMENTAL else MAX_PAGES
    
    for page_num in range(1, page_end + 1):
        url = get_page_url(page_num)
        log(f"抓取列表页 {page_num}/{page_end}: {url}")
        try:
            html = fetch(url)
        except Exception as e:
            log(f"列表页 {page_num} 失败: {e}")
            continue
        items = parse_list(html, page_num)
        log(f"  解析到 {len(items)} 条")
        
        for item in items:
            cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
            if cur.fetchone():
                if INCREMENTAL:
                    log(f"  已有记录，增量结束")
                    conn.close()
                    log(f"增量完成，共 {total_new} 条新增")
                    return
                continue
            
            detail = fetch_detail(item["url"])
            if detail is None:
                continue
            
            content = detail["content"]
            now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            cur.execute(
                """INSERT OR IGNORE INTO gov_raw 
                (site_name, source_url, page_url, title, publish_date, summary, content, status, category)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (
                    SITE_NAME,
                    item["url"],
                    item["url"],
                    item["title"],
                    item["date"],
                    content[:200],
                    content,
                    "published",
                    "公告公示",
                )
            )
            if cur.rowcount > 0:
                total_new += 1
                if total_new % 10 == 0:
                    conn.commit()
                    log(f"  已入库 {total_new} 条...")
        
        conn.commit()
        log(f"第{page_num}页完成，累计 {total_new} 条")
        time.sleep(0.3)
    
    conn.close()
    log(f"全量爬取完成，共新增 {total_new} 条")


if __name__ == "__main__":
    main()