#!/usr/bin/env python3
"""
crawl_hhgj.py — 个旧市人民政府·环境保护/环保信息
=============================================
TRS CMS，静态分页

栏目1: hjbh1.htm → hjbh1/{N}.htm   (环境保护, 8页, NEW→OLD)
栏目2: hbxx.htm  → hbxx/{N}.htm    (环保信息, 54页, OLD→NEW)

列表：<li><a href="../../info/XXXXX/XXXXX.htm">Title YYYY.MM.DD</a></li>
详情：<div class="content"> 正文 | <meta name="PubDate"> 日期

用法:
    python3 crawl_hhgj.py               # 全量（两个栏目）
    python3 crawl_hhgj.py 1             # 增量（只爬首页）
"""

import os
import re
import sys
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from datetime import datetime, timedelta

# ── DB ──────────────────────────────────────────────
DB_PATH = os.environ.get("SEARCH_DB", os.environ.get("GOV_DB_PATH", "/root/search.db"))

# ── URLs ────────────────────────────────────────────
BASE_URL = "https://www.hhgj.gov.cn"
SITE_NAME = "个旧市人民政府"

COLUMNS = [
    {
        "name": "环境保护",
        "source": "个旧市环境保护",
        "list_url": BASE_URL + "/ztlm/ggqsydwxxgkpt/hjbh1.htm",
        "page_dir": "/ztlm/ggqsydwxxgkpt/hjbh1",
        "n_pages": 8,      # pages 1-8
        "direction": "new_to_old",  # page 1 = newest
        "date_fmt": "dot",  # YYYY.MM.DD in list text
    },
    {
        "name": "环保信息",
        "source": "个旧市环保信息",
        "list_url": BASE_URL + "/zfxxgk/fdzdgknr/zdlyxxgk/hbxx.htm",
        "page_dir": "/zfxxgk/fdzdgknr/zdlyxxgk/hbxx",
        "n_pages": 54,     # pages 1-54
        "direction": "old_to_new",  # page 1 = oldest
        "date_fmt": "dash", # YYYY-MM-DD in list text
    },
]

# ── Headers ─────────────────────────────────────────
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/120.0.0.0 Safari/537.36",
}

# ── helpers ─────────────────────────────────────────
def log(msg):
    print(f"[hhgj] {msg}")

def clean_title(title):
    return title.replace("-个旧市人民政府", "").replace("-个旧市", "").strip()

def extract_date_from_text(text, fmt="dot"):
    """Extract date from list item text"""
    if fmt == "dot":
        m = re.search(r'(\d{4})\.(\d{2})\.(\d{2})', text)
        if m:
            return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
    else:
        m = re.search(r'(\d{4}-\d{2}-\d{2})', text)
        if m:
            return m.group(1)
    return ""

def fetch_list_page(col, page_num):
    """Fetch one list page, return [(url, title, date_str), ...]"""
    if page_num == 1:
        url = col["list_url"]
    else:
        url = f"{BASE_URL}{col['page_dir']}/{page_num}.htm"

    try:
        resp = requests.get(url, headers=HEADERS, timeout=30)
        resp.encoding = "utf-8"
        html = resp.text
    except Exception as e:
        log(f"  ERROR fetching {url}: {e}")
        return []

    # Extract <li> items
    items = []
    # Find all <li> that contain info links
    for m in re.finditer(r'<li[^>]*>(.*?)</li>', html, re.DOTALL):
        li = m.group(1)
        a = re.search(r'href=\"([^\"]*(?:info|content)/\d+/\d+\.htm)\"', li)
        if not a:
            continue
        href = a.group(1)
        # Extract text between <a> tags
        text_match = re.search(r'<a[^>]*>(.*?)</a>', li, re.DOTALL)
        text = re.sub(r'<[^>]+>', '', text_match.group(1)).strip() if text_match else ""

        # Clean title: remove date suffix
        title = re.sub(r'\d{4}\.\d{2}\.\d{2}', '', text).strip()
        title = re.sub(r'\d{4}-\d{2}-\d{2}', '', title).strip()
        title = clean_title(title)

        # Date from text
        date_str = extract_date_from_text(text, col["date_fmt"])

        # Build full URL
        if href.startswith("../../../"):
            full_url = f"{BASE_URL}/{'/'.join(href.split('/')[3:])}"
        elif href.startswith("../../"):
            full_url = f"{BASE_URL}/{href[6:]}"
        elif href.startswith("../"):
            full_url = f"{BASE_URL}/{href[3:]}"
        elif href.startswith("/"):
            full_url = f"{BASE_URL}{href}"
        else:
            full_url = urljoin(url, href)

        items.append((full_url, title, date_str))

    return items

def fetch_detail(url):
    """Return (content_html, date_str) or (None, None)"""
    try:
        resp = requests.get(url, headers=HEADERS, timeout=30)
        resp.encoding = "utf-8"
        html = resp.text
    except Exception as e:
        log(f"  ERROR fetching {url}: {e}")
        return None, None

    soup = BeautifulSoup(html, "html.parser")

    # Content: try multiple selectors
    content_html = ""
    for selector in [
        ("div", {"class": "content"}, None),
        ("div", {"class": "xxgk-con"}, None),
        ("div", {"id": re.compile(r"vsb_content")}, None),
    ]:
        div = soup.find(selector[0], selector[1])
        if div:
            content_html = str(div)
            break

    # Date: <meta name="PubDate" content="2026-06-08 11:12" />
    date_str = ""
    meta = soup.find("meta", attrs={"name": "PubDate"})
    if meta and meta.get("content"):
        raw = meta["content"].strip()
        m = re.match(r"(\d{4}-\d{2}-\d{2})", raw)
        if m:
            date_str = m.group(1)

    # Fallback
    if not date_str:
        m = re.search(r'发布日期[：:]\s*(\d{4}-\d{2}-\d{2})', html)
        if m:
            date_str = m.group(1)

    if not content_html:
        log(f"  WARNING: no content for {url}")
        return None, date_str

    return content_html, date_str

def save_to_db(items):
    import sqlite3
    if not items:
        return 0
    log(f"Opening DB: {DB_PATH}")
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for url, title, date_str, content_html, source in items:
        try:
            c.execute(
                """INSERT OR REPLACE INTO gov_raw (site_name, page_url, title, publish_date, content, category, script_name) VALUES (?, ?, ?, ?, ?, ?, 'crawl_hhgj.py')""",
                (SITE_NAME, url, title, date_str, content_html, source),
            )
            inserted += 1
        except Exception as e:
            log(f"  DB error for {url}: {e}")
    conn.commit()
    conn.close()
    return inserted

def crawl_column(col, incremental=False):
    """Crawl a single column, return [(url, title, date, html, source)]"""
    three_years_ago = (datetime.now() - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
    name = col["name"]

    log(f"\n{'='*50}")
    log(f"Column: {name}")
    log(f"3-year cutoff: {three_years_ago}")

    # Determine page range
    if incremental:
        if col["direction"] == "new_to_old":
            # Page 1 = homepage = newest
            pages = [1]
        else:
            # old_to_new: last page = newest
            pages = [col["n_pages"]]
    elif col["direction"] == "new_to_old":
        pages = range(1, col["n_pages"] + 1)
    else:
        # old_to_new: page 1 = oldest, page N = newest
        # Start from newest and go backward until before cutoff
        pages = range(col["n_pages"], 0, -1)

    all_articles = []
    for pn in pages:
        articles = fetch_list_page(col, pn)
        if not articles:
            log(f"  Page {pn}: no articles")
            continue

        # Filter 3-year
        filtered = [(u, t, d) for u, t, d in articles if d and d >= three_years_ago]
        if filtered:
            all_articles.extend(filtered)
            log(f"  Page {pn}: {len(filtered)}/{len(articles)} articles")

        # Early stop for old_to_new (starting from newest going backward)
        if col["direction"] == "old_to_new" and not incremental:
            if articles and all(d for _, _, d in articles if d):
                oldest_on_page = min(d for _, _, d in articles if d)
                if oldest_on_page < three_years_ago:
                    log(f"  Page {pn}: oldest={oldest_on_page} < cutoff, stopping")
                    break

        # For new_to_old, stop when all items before cutoff
        if col["direction"] == "new_to_old" and not incremental:
            if articles and all(d for _, _, d in articles if d):
                newest_on_page = max(d for _, _, d in articles if d)
                if newest_on_page < three_years_ago:
                    log(f"  Page {pn}: newest={newest_on_page} < cutoff, stopping")
                    break

    log(f"Total articles: {len(all_articles)}")

    # Fetch details
    results = []
    for i, (url, title, date_str) in enumerate(all_articles, 1):
        log(f"[{i}/{len(all_articles)}] {title[:50]}...")
        content_html, detail_date = fetch_detail(url)
        final_date = detail_date or date_str
        results.append((url, title, final_date, content_html or "", col["source"]))

    return results

def main():
    incremental = len(sys.argv) > 1 and sys.argv[1] == "1"

    all_results = []
    for col in COLUMNS:
        results = crawl_column(col, incremental)
        all_results.extend(results)

    # Save all
    saved = save_to_db(all_results)
    log(f"\nTotal saved: {saved}/{len(all_results)} ({len(COLUMNS)} columns)")

    if all_results:
        dates = sorted([r[2] for r in all_results if r[2]])
        log(f"Date range: {dates[0]} ~ {dates[-1]}")
        sources = set(r[4] for r in all_results)
        log(f"Sources: {', '.join(sources)}")

    # Rebuild FTS
    try:
        import sqlite3
        conn = sqlite3.connect(DB_PATH, timeout=60)
        for col in COLUMNS:
            conn.execute("""
                INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
                SELECT rowid, title, site_name, 
                       CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
                FROM gov_raw WHERE category = ?
            """, (col["source"],))
        conn.commit()
        conn.close()
        log("FTS index rebuilt")
    except Exception as e:
        log(f"FTS rebuild note: {e}")


if __name__ == "__main__":
    main()