#!/usr/bin/env python3
"""Crawler for 清水河县-法定主动公开内容 (qingshuihe.gov.cn zfxxgk)
   http://www.qingshuihe.gov.cn/zfxxgkzl/fdzdgknr/?gk=3
   CMS: TRS WCM, content: div.trs_editor_view
   Multi sub-category list pages, yfxz pagination: index_N.html
"""
import urllib.request, urllib.parse, ssl, re, sqlite3, sys, time
from datetime import datetime

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE

BASE = "http://www.qingshuihe.gov.cn"
LIST_URL = BASE + "/zfxxgkzl/fdzdgknr/?gk=3"
DB = "/root/search.db"

SITE_NAME = "qingshuihe_gk3"
INCREMENTAL = "--incremental" in sys.argv

YFXZ_PAGES = 10  # index_N.html 分页
YFXZ_BASE = BASE + "/yfxz/xbjqzqd/"
YFXZ_ITEMS_PER_PAGE = 20


def log(msg):
    print(f"[{SITE_NAME}] {msg}")


def fetch(url):
    if "' + " in url or "rootPath" in url or url.startswith("javascript:") or "void" in url:
        log(f"[WARN] 跳过无效URL: {url[:100]}")
        return ""
    try:
        req = urllib.request.Request(url, headers={
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        })
        r = urllib.request.urlopen(req, context=ssl_ctx, timeout=30)
        return r.read().decode("utf-8", errors="replace")
    except Exception as e:
        log(f"[WARN] fetch失败 {url[:80]}: {str(e)[:50]}")
        return ""


def extract_subcategory_urls():
    """从 gk=3 主页面提取所有子栏目 data-url"""
    html = fetch(LIST_URL)
    # 找到 #xxgkml 附近的 data-url 链接（含子栏目路径）
    urls = set()
    # Find xxgkml section
    idx = html.find('id="xxgkml"')
    if idx < 0:
        log("未找到 xxgkml 区域")
        return []
    section = html[idx:idx+15000]
    for m in re.finditer(r"data-url='([^']+)'", section):
        url = m.group(1)
        if url.startswith("http"):
            urls.add(url)
        else:
            urls.add(BASE + "/zfxxgkzl/fdzdgknr/" + url.lstrip("./"))
    log(f"发现 {len(urls)} 个子栏目URL")
    return sorted(urls)


def parse_list_table(html, source):
    """解析列表页 table#table1 中的行"""
    items = []
    # 提取 tbody
    tbody = re.search(r"<tbody>(.*?)</tbody>", html, re.DOTALL)
    if not tbody:
        return items
    tbody_html = tbody.group(1)
    rows = re.findall(r"<tr>(.*?)</tr>", tbody_html, re.DOTALL)
    for row in rows:
        # 找链接和标题
        a_match = re.search(r'<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"', row)
        if not a_match:
            a_match = re.search(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', row, re.DOTALL)
            if not a_match:
                continue
            href = a_match.group(1).strip()
            title = re.sub(r'<[^>]+>', '', a_match.group(2)).strip()
        else:
            href = a_match.group(1).strip()
            title = a_match.group(2).strip()
        if not title or not href:
            continue
        # 跳过JS拼接/无效URL
        if "' + " in href or "rootPath" in href or href.startswith("javascript:") or "void" in href or href.startswith("#"):
            continue
        if not href.startswith("http"):
            href = urllib.parse.urljoin(BASE, href)
        # 日期
        date_match = re.search(r"发布日期[：:]\s*([\d-]+)", row)
        if not date_match:
            date_match = re.search(r"成文日期[：:]\s*([\d-]+)", row)
        date_str = date_match.group(1) if date_match else ""
        items.append({
            "title": title,
            "url": href,
            "date": date_str,
            "source": source,
        })
    return items


def parse_list_page(html, source):
    """解析列表页，兼容有/无table格式"""
    items = parse_list_table(html, source)
    if items:
        return items
    # Fallback: plain ul/li
    for a_match in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"', html):
        href = a_match.group(1).strip()
        title = a_match.group(2).strip()
        # 跳过JS拼接/无效URL
        if "' + " in href or "rootPath" in href or href.startswith("javascript:") or "void" in href or href.startswith("#"):
            continue
        if not href.startswith("http"):
            href = urllib.parse.urljoin(BASE, href)
        items.append({
            "title": title,
            "url": href,
            "date": "",
            "source": source,
        })
    return items


def fetch_detail(url):
    """获取详情页正文 div.trs_editor_view"""
    html = fetch(url)
    # Try various content containers
    for pattern in [r'<div[^>]*class="trs_editor_view"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="TRS_Editor"[^>]*>(.*?)</div>',
                    r'<div[^>]*id="Zoom"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="news-content"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="content"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="article"[^>]*>(.*?)</div>']:
        m = re.search(pattern, html, re.DOTALL)
        if m:
            inner = m.group(1)
            # Strip HTML tags, keep paragraphs
            parts = []
            for p in re.findall(r'<p[^>]*>(.*?)</p>', inner, re.DOTALL):
                txt = re.sub(r'<[^>]+>', '', p).strip()
                if txt:
                    parts.append(txt)
            if not parts:
                txt = re.sub(r'<[^>]+>', '', inner).strip()
                if txt:
                    parts = [txt]
            if parts:
                return {"content": "\n".join(parts)}
    log(f"未找到正文: {url}")
    return None


def main():
    conn = sqlite3.connect(DB)
    cur = conn.cursor()
    total_new = 0

    # Step 1: 获取所有子栏目URL
    sub_urls = extract_subcategory_urls()
    if not sub_urls:
        log("未发现子栏目，退出")
        return

    # Step 2: 遍历每个子栏目列表页
    processed_urls = set()  # 去重
    
    for sub_url in sub_urls:
        if sub_url in processed_urls:
            continue
        processed_urls.add(sub_url)
        
        try:
            html = fetch(sub_url)
        except Exception as e:
            log(f"子栏目 {sub_url} 失败: {e}")
            continue
        
        # 检查是否有 index_N.html 分页模式（yfxz路径）
        path = sub_url.replace(BASE, "")
        has_pagination = "zxf_pagediv" in html or "createPage" in html
        
        # 处理当前页
        items = parse_list_page(html, sub_url)
        for item in items:
            if item["url"] in processed_urls:
                continue
            processed_urls.add(item["url"])
            
            # 去重
            cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
            if cur.fetchone():
                if INCREMENTAL:
                    continue
                # 非增量模式下如果遇到已存在的，也跳过
                continue
            
            detail = fetch_detail(item["url"])
            if detail is None:
                continue
            
            content = detail["content"]
            now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            cur.execute(
                """INSERT OR IGNORE INTO gov_raw 
                (site_name, source_url, page_url, title, publish_date, summary, content, status, category)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (
                    SITE_NAME,
                    item["url"],
                    item["url"],
                    item["title"],
                    item["date"],
                    content[:200],
                    content,
                    "published",
                    "法定主动公开内容",
                )
            )
            if cur.rowcount > 0:
                total_new += 1
                if total_new % 10 == 0:
                    conn.commit()
        
        conn.commit()
        log(f"子栏目 {sub_url}: 已入库 {total_new} 条")
        time.sleep(0.3)
        
        # 检查 yfxz 分页
        if has_pagination and "yfxz" in sub_url:
            for pg in range(2, YFXZ_PAGES + 1):
                pg_url = sub_url.replace("index.html", f"index_{pg}.html")
                if pg_url in processed_urls:
                    continue
                processed_urls.add(pg_url)
                try:
                    html2 = fetch(pg_url)
                except Exception:
                    break
                items2 = parse_list_page(html2, pg_url)
                for item in items2:
                    if item["url"] in processed_urls:
                        continue
                    processed_urls.add(item["url"])
                    cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
                    if cur.fetchone():
                        if INCREMENTAL:
                            continue
                        continue
                    detail = fetch_detail(item["url"])
                    if detail is None:
                        continue
                    content = detail["content"]
                    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                    cur.execute(
                        """INSERT OR IGNORE INTO gov_raw 
                        (site_name, source_url, page_url, title, publish_date, summary, content, status, category)
                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                        (SITE_NAME, item["url"], item["url"], item["title"], item["date"],
                         content[:200], content, "published", "法定主动公开内容")
                    )
                    if cur.rowcount > 0:
                        total_new += 1
                        if total_new % 10 == 0:
                            conn.commit()
                conn.commit()
                time.sleep(0.3)
    
    conn.close()
    log(f"爬取完成，共新增 {total_new} 条")


if __name__ == "__main__":
    main()