#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Crawler for 黔南热线 - 热点关注 (qnz.com.cn)
CMS: SiteServer (STL dynamic pagination)
List: ul.list > li > p > a[href][title] + span (date)
       Pages 2+: AJAX POST https://sites.qnz.com.cn/api/stl/actions/pagecontents
Pagination: hot.html (page1), hot_{N}.html (page N), 24/page, 42 pages ~1000 items
Detail: div.show-content > p (paragraphs), span.time (date), span.source (source)

Usage:
    python3 /root/crawl_qnz_hot.py             # Full crawl (all 42 pages)
    python3 /root/crawl_qnz_hot.py --max-pages 1  # Incremental (page 1 only)
"""

import requests, re, json, sqlite3, time, os, sys
from datetime import datetime
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "黔南热线-热点关注"
CATEGORY = "热点关注"
GROUP = "贵州"
LIST_URL = "https://www.qnz.com.cn/news/hot.html"
API_URL = "https://sites.qnz.com.cn/api/stl/actions/pagecontents"
TOTAL_PAGES = 42
PER_PAGE = 24

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}


def extract_api_token():
    """Get stlPageContentsElement from page 2 HTML"""
    try:
        r = requests.get("https://www.qnz.com.cn/news/hot_2.html", headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        m = re.search(r"stlPageContentsElement:\s*\x27([^\x27]+)\x27", r.text)
        if m:
            return m.group(1)
    except Exception as e:
        print(f"  Error getting API token: {e}", file=sys.stderr)
    return None


def parse_list_html(html, base_url):
    """Parse list HTML (from inline or API response) into items"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    ul = soup.find("ul", class_="list")
    if not ul:
        lis = soup.find_all("li")
        if not lis:
            return items
    else:
        lis = ul.find_all("li", recursive=False)

    for li in lis:
        a = li.find("a")
        span = li.find("span")
        if not a or not a.get("href"):
            continue
        href = a["href"].strip()
        title = a.get("title", "").strip() or a.get_text(strip=True)
        page_url = urljoin(base_url, href)
        date_str = span.get_text(strip=True) if span else ""
        date_str = re.sub(r"\s+", "", date_str)[:10] if date_str else ""
        if title and page_url:
            items.append((title, page_url, date_str))
    return items


def get_list_page(page_num, api_token):
    """Get list items for a page number"""
    if page_num == 1:
        try:
            r = requests.get(LIST_URL, headers=HEADERS, timeout=20)
            r.encoding = "utf-8"
            if r.status_code == 200:
                return parse_list_html(r.text, LIST_URL)
        except Exception as e:
            print(f"  Error fetching page 1: {e}", file=sys.stderr)
        return []
    else:
        if not api_token:
            return []
        params = {
            "siteId": 1,
            "pageChannelId": 302,
            "templateId": 2,
            "totalNum": 1000,
            "pageCount": TOTAL_PAGES,
            "currentPageIndex": page_num,
            "stlPageContentsElement": api_token,
        }
        try:
            r = requests.post(API_URL, json=params, headers={**HEADERS, "Content-Type": "application/json"}, timeout=20)
            if r.status_code == 200:
                data = r.json()
                html = data.get("html", "")
                if html:
                    return parse_list_html(html, LIST_URL)
        except Exception as e:
            print(f"  Error fetching page {page_num} via API: {e}", file=sys.stderr)
        return []


def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  fetch_detail error {url}: {e}", file=sys.stderr)
        return None, None, None, []

    soup = BeautifulSoup(r.text, "html.parser")
    title = ""
    t = soup.find("title")
    if t:
        title = t.get_text(strip=True).replace(" - 黔南热线", "").strip()

    date_str = ""
    span = soup.find("span", class_="time")
    if span:
        date_str = span.get_text(strip=True)[:10]

    content_div = soup.find("div", class_="show-content")
    content = ""
    if content_div:
        parts = []
        for p in content_div.find_all("p"):
            txt = p.get_text(strip=True)
            if txt:
                img = p.find("img")
                if img:
                    src = img.get("src", "")
                    if src:
                        full_src = urljoin(url, src)
                        alt = img.get("alt", "")
                        parts.append(f"![{alt}]({full_src})" if alt else f"![]({full_src})")
                else:
                    parts.append(txt)
        content = "\n\n".join(parts)

    attachments = []
    seen = set()
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$", href, re.I):
            full_url = urljoin(url, href)
            name = a.get_text(strip=True) or href.split("/")[-1].split("?")[0]
            if full_url not in seen:
                seen.add(full_url)
                attachments.append({"name": name, "url": full_url})
                content += f'\n\n附件：<p><a href="{full_url}">{name}</a></p>'

    return title, date_str, content, attachments


def with_retry(fn, desc="DB op", max_attempts=3, delay=2):
    for attempt in range(1, max_attempts + 1):
        try:
            return fn()
        except sqlite3.OperationalError as e:
            if "locked" in str(e) and attempt < max_attempts:
                print(f"  {desc}: locked (attempt {attempt}/{max_attempts}), retry in {delay}s...", file=sys.stderr)
                time.sleep(delay)
            else:
                print(f"  {desc} failed: {e}", file=sys.stderr)
                return False
    return False


def init_db():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn


def save_article(conn, title, page_url, publish_date, content, attachments):
    summary = content[:200] if content else title
    summary = re.sub(r"\s+", " ", summary).strip()
    att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]"
    try:
        cur = conn.execute(
            """INSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (SITE_NAME, SITE_NAME, page_url, title.strip(), publish_date,
             summary, content, CATEGORY, att_json, GROUP),
        )
        return cur.rowcount > 0
    except Exception as e:
        print(f"  DB error: {e}", file=sys.stderr)
        return False


def main():
    max_pages = TOTAL_PAGES
    if len(sys.argv) > 1:
        try:
            max_pages = int(sys.argv[1])
        except ValueError:
            if sys.argv[1] in ("--max-pages",) and len(sys.argv) > 2:
                try:
                    max_pages = int(sys.argv[2])
                except ValueError:
                    pass

    print(f"Scraping {SITE_NAME}, max pages: {max_pages}", file=sys.stderr)

    api_token = extract_api_token()
    if api_token:
        print(f"  API token obtained ({len(api_token)} chars)", file=sys.stderr)
    else:
        print("  WARNING: No API token found, only page 1", file=sys.stderr)

    conn = init_db()
    total_new = 0
    total_found = 0

    def do_clean():
        conn.execute("DELETE FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
        conn.commit()
    with_retry(do_clean, desc=f"Clean {SITE_NAME}")
    print(f"  Cleaned old data for {SITE_NAME}", file=sys.stderr)

    for pn in range(1, max_pages + 1):
        articles = get_list_page(pn, api_token)
        if not articles:
            print(f"  Page {pn}: 0 articles (end)", file=sys.stderr)
            break
        total_found += len(articles)
        for title, page_url, date in articles:
            if date and date < "2020-01-01":
                print(f"  Skip old: {date} {title[:40]}", file=sys.stderr)
                continue
            cur = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
            if cur.fetchone():
                continue
            det_title, det_date, det_content, det_atts = fetch_detail(page_url)
            final_title = det_title or title
            final_date = det_date or date
            final_content = det_content or f"[{final_title}]({page_url})"
            final_atts = det_atts or []
            if save_article(conn, final_title, page_url, final_date, final_content, final_atts):
                total_new += 1
                if total_new <= 5:
                    print(f"  + {final_date} {final_title[:50]}", file=sys.stderr)
            time.sleep(0.5)
        print(f"  Page {pn}: {len(articles)} found, {total_new} total", file=sys.stderr)
        conn.commit()

    conn.commit()
    conn.close()
    print(f"\nDone! Found: {total_found}, New: {total_new}", file=sys.stderr)


if __name__ == "__main__":
    main()