#!/usr/bin/env python3
import os
"""昌吉回族自治州生态环境局 — 通知公告 爬虫 (静态分页)
用法:
  python3 crawl_changji.py              # 全量爬取
  python3 crawl_changji.py 5            # 爬前5页
  python3 crawl_changji.py --incremental # 增量模式
"""
import sys, re, sqlite3
import requests
from urllib.parse import urljoin
from datetime import datetime, timedelta

SITE_NAME = "昌吉州生态环境局-通知公告"
BASE = "https://www.cj.gov.cn"
LIST_PATH = "/p122/tzgg"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 50  # 约750条 (15/page), 3年约45页

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": BASE + LIST_PATH + ".html",
}

CUTOFF = (datetime.now() - timedelta(days=365 * 3)).strftime("%Y-%m-%d")


def fetch_list(session, page):
    """Fetch a list page."""
    url = f"{BASE}{LIST_PATH}.html" if page == 1 else f"{BASE}{LIST_PATH}_{page}.html"
    try:
        r = session.get(url, headers=HEADERS, timeout=30)
        if r.status_code != 200:
            return None
        r.encoding = "utf-8"
        return r.text
    except:
        return None


def parse_list(html):
    """Extract article items from list HTML.
    Format: <li class="subtitle"><a target="_blank" title="..." href="...">title</a><span>YYYY-MM-DD</span></li>
    """
    items = []
    pattern = re.compile(
        r'<a\s+target="_blank"\s+title="([^"]*)"\s+href="([^"]+)"[^>]*>.*?</a>\s*<span>(\d{4}-\d{1,2}-\d{1,2})</span>',
        re.DOTALL,
    )
    for m in pattern.finditer(html):
        title = m.group(1).strip()
        url = m.group(2)
        if not url.startswith("http"):
            url = urljoin(BASE, url)
        date_str = m.group(3).strip()
        items.append({"url": url, "title": title, "date": date_str})
    return items


def parse_detail(html, fallback_title=""):
    """Extract title, date, content from detail page."""
    # Title from <div class="title"> inside body_content
    title = ""
    m = re.search(r'<div\s+class="title"[^>]*>\s*(.*?)\s*</div>', html, re.DOTALL)
    if m:
        title = re.sub(r"<[^>]+>", "", m.group(1)).strip()
    if not title:
        m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html)
        if m:
            title = m.group(1).strip()
    if not title:
        title = fallback_title

    # Date from meta PubDate or <span class="fbrq">
    date_str = ""
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        date_str = m.group(1)
    if not date_str:
        m = re.search(r'<span\s+class="fbrq"[^>]*>\s*发布日期[：:]\s*(\d{4}-\d{1,2}-\d{1,2})', html)
        if m:
            date_str = m.group(1)

    # Content from <font id="Zoom"> inside <div class="content">
    content = ""
    cm = re.search(r'<font\s+id="Zoom"[^>]*>(.*?)</font>', html, re.DOTALL)
    if cm:
        content = cm.group(1).strip()
    else:
        # Fallback: find content inside body_content > content div
        cm = re.search(
            r'<div\s+class="body_content"[^>]*>.*?<div\s+class="content"[^>]*>(.*?)</div>\s*</div>',
            html, re.DOTALL,
        )
        if cm:
            content = cm.group(1).strip()

    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', "", content, flags=re.DOTALL | re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', "", content, flags=re.DOTALL | re.I)
        content = re.sub(r'\s*style="[^"]*"', "", content)
        content = re.sub(r"<!--.*?-->", "", content)
        content = content.strip()

    return title, date_str, content


def push_to_db(db, records):
    inserted = 0
    skipped = 0
    for item in records:
        try:
            db.execute(
                """INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags)
                VALUES (?,?,?,?,?,?,?,?,?,?)""",
                (
                    item["site_name"][:200],
                    item["source_url"],
                    item["page_url"],
                    item["title"],
                    item["publish_date"],
                    item["summary"],
                    item["content"],
                    item["status"],
                    item["category"],
                    item["tags"],
                ),
            )
            if db.total_changes > 0:
                inserted += 1
            else:
                skipped += 1
        except:
            skipped += 1
    return inserted, skipped


def run(max_pages=None, incremental=False):
    if max_pages is None:
        max_pages = MAX_PAGES

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")

    known = set(
        r[0]
        for r in db.execute(
            "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)
        ).fetchall()
    )
    print(f"  Known in DB: {len(known)}")

    session = requests.Session()
    all_items = []

    for page in range(1, max_pages + 1):
        print(f"  Page {page}...", end=" ")
        html = fetch_list(session, page)
        if not html:
            print("404/FAIL")
            break
        items = parse_list(html)
        if not items:
            print("empty")
            break
        print(f"{len(items)} items")

        if incremental:
            if all(item["url"] in known for item in items):
                print(f"    All known, stopping")
                break
            new_items = [i for i in items if i["url"] not in known]
            if not new_items:
                break
            all_items.extend(items)
        else:
            all_items.extend(items)

        # Check if last page's dates are older than cutoff
        if all(i["date"] < CUTOFF for i in items):
            print(f"    All dates before {CUTOFF}, stopping")
            break

    if not all_items:
        print("  No items to process")
        db.close()
        return

    # Filter by date
    all_items = [i for i in all_items if i["date"] >= CUTOFF or not i["date"]]
    print(f"\n  Fetching {len(all_items)} details...")

    results = []
    for i, item in enumerate(all_items):
        try:
            r = session.get(item["url"], headers=HEADERS, timeout=30)
            r.encoding = "utf-8"
            html = r.text
        except:
            continue
        title, date_str, content = parse_detail(html, item["title"])
        results.append(
            {
                "site_name": SITE_NAME,
                "source_url": item["url"][:500],
                "page_url": item["url"],
                "title": (title or item["title"])[:500],
                "publish_date": (date_str or item["date"])[:10],
                "summary": (title or item["title"])[:500],
                "content": content,
                "status": "active",
                "category": "",
                "tags": "",
            }
        )
        if (i + 1) % 20 == 0:
            print(f"    [{i+1}/{len(all_items)}]")

    inserted, skipped = push_to_db(db, results)
    db.commit()

    db.execute(
        """INSERT INTO gov_search(rowid,title,site_name,summary)
        SELECT r.id, r.title, r.site_name, r.summary
        FROM gov_raw r
        WHERE r.id NOT IN (SELECT rowid FROM gov_search)
        AND r.site_name=?""",
        (SITE_NAME,),
    )
    db.commit()
    db.close()

    print(f"  Done: new={inserted}, skip={skipped}")


if __name__ == "__main__":
    incremental = "--incremental" in sys.argv
    mp = None
    for a in sys.argv[1:]:
        if a.isdigit():
            mp = int(a)
            break
        if a == "--test":
            mp = 1
            break

    run(max_pages=mp, incremental=incremental)