#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Crawler for 河南濮阳工业园区 - 公告公示 (pygyyq.gov.cn)
CMS: 自定义ASP CMS, charset=gb2312
List: ul.news-list > li > a[href][title] + span(date), 15/page
Pagination: ?class=23&page=N, 3 pages ~44 items
Detail: h1.content-title, div.show>p.text or a.pdf or img content
Attachments: a.ke-insertfile or a[href*=./upload/]

Usage:
    python3 /root/gov_crawler/crawl_pygyyq_gsgg.py
"""

import requests, re, json, sqlite3, time, os, sys, urllib.parse
from datetime import datetime
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "河南濮阳工业园区-公告公示"
CATEGORY = "通知公告"
GROUP = "河南"
BASE_URL = "http://www.pygyyq.gov.cn"
LIST_URL = "http://www.pygyyq.gov.cn/zwgk/zfxx_gknr_news.asp"
MAX_PAGES = 3

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}


def fetch_gbk(url, retries=3):
    """Fetch URL with GBK encoding handling"""
    for i in range(retries):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            r.encoding = 'gbk'
            return r.text, r
        except Exception as e:
            if i < retries - 1:
                time.sleep(2)
            else:
                print(f"  [WARN] Failed to fetch {url}: {e}", file=sys.stderr)
                return None, None
    return None, None


# ---------- List ----------

def extract_list_page(page_num):
    params = {"class": 23}
    if page_num > 1:
        params["page"] = page_num
    url = LIST_URL + "?" + urllib.parse.urlencode(params)
    html, _ = fetch_gbk(url)
    if not html:
        return []

    soup = BeautifulSoup(html, "html.parser")
    items = []

    ul = soup.find("ul", class_="news-list")
    if not ul:
        return []

    for li in ul.find_all("li", recursive=False):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"]
        page_url = urljoin(url, href)
        title = a.get("title", "").strip() or a.get_text(strip=True)
        if not title:
            continue
        span = li.find("span")
        date_str = span.get_text(strip=True) if span else ""
        items.append((title, page_url, date_str))

    return items


# ---------- Detail ----------

def normalize_date(date_str):
    """Normalize YYYY-M-D to YYYY-MM-DD"""
    if not date_str:
        return ""
    try:
        parts = date_str.strip().split("-")
        if len(parts) == 3:
            return f"{int(parts[0]):04d}-{int(parts[1]):02d}-{int(parts[2]):02d}"
    except:
        pass
    return date_str


def fetch_detail(url):
    html, resp = fetch_gbk(url)
    if not html:
        return None, None, None, []

    soup = BeautifulSoup(html, "html.parser")

    # --- Title ---
    title = ""
    h1 = soup.find("h1", class_="content-title")
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        t = soup.find("title")
        if t:
            raw = t.get_text(strip=True)
            if "_" in raw:
                title = raw.split("_")[0].strip()

    # --- Date ---
    date_str = ""
    mess = soup.find("div", class_="wznr-mess")
    if mess:
        spans = mess.find_all("span")
        for span in spans:
            t = span.get_text(strip=True)
            if re.match(r"\d{4}[-\/]\d{1,2}[-\/]\d{1,2}", t):
                date_str = t.strip()
                break
    date_str = normalize_date(date_str)

    # --- Content ---
    content = ""
    show_div = soup.find("div", class_="show")
    if show_div:
        for s in show_div.find_all(["script", "style"]):
            s.decompose()
        parts = []

        # Extract text paragraphs
        for p in show_div.find_all("p"):
            # Skip the "扫一扫" footer paragraph
            if "扫一扫" in p.get_text() and "手机" in p.get_text():
                continue
            t = p.get_text(strip=True)
            if t and t not in (" ", "\xa0", "&nbsp;"):
                parts.append(t)

        # Extract tables
        for tbl in show_div.find_all("table"):
            if not tbl.find_parent("table"):
                parts.append(str(tbl))

        # Extract images (if no text content, use markdown images)
        if not parts:
            imgs = show_div.find_all("img")
            for img in imgs:
                src = img.get("src", "")
                if src:
                    full_src = urljoin(url, src)
                    alt = img.get("alt", "") or ""
                    parts.append(f"![{alt}]({full_src})")

        content = "\n\n".join(parts)

    # If still no content, try fallback: show-content div itself
    if not content:
        sc = soup.find("div", class_="show-content")
        if sc:
            for s in sc.find_all(["script", "style"]):
                s.decompose()
            for p in sc.find_all("p"):
                t = p.get_text(strip=True)
                if t and "扫一扫" not in t:
                    parts.append(t)
            content = "\n\n".join(parts)

    content = re.sub(r"\n{3,}", "\n\n", content).strip()

    # --- Attachments ---
    attachments = []
    seen = set()
    for a_tag in soup.find_all("a", href=True):
        href = a_tag["href"]
        is_attachment = (
            re.search(r"\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$", href, re.I) or
            "/upload/" in href
        )
        if is_attachment:
            full_url = urljoin(url, href)
            name = a_tag.get_text(strip=True) or href.split("/")[-1].split("?")[0]
            if full_url not in seen:
                seen.add(full_url)
                attachments.append({"name": name, "url": full_url})

    return title, date_str, content, attachments


# ---------- DB ----------

def with_retry(fn, desc="DB op", max_attempts=10, delay=5):
    for attempt in range(1, max_attempts + 1):
        try:
            return fn()
        except sqlite3.OperationalError as e:
            if "locked" in str(e) and attempt < max_attempts:
                print(f"  {desc}: locked (attempt {attempt}/{max_attempts}), retry in {delay}s...", file=sys.stderr)
                time.sleep(delay)
            else:
                print(f"  {desc} failed: {e}", file=sys.stderr)
                return False
    return False


def init_db():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn


def save_article(conn, title, page_url, publish_date, content, attachments):
    summary = content[:200] if content else title
    summary = re.sub(r"\s+", " ", summary).strip()
    att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]"
    try:
        cur = conn.execute(
            """INSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (SITE_NAME, page_url, page_url, title.strip(), publish_date,
             summary, content, CATEGORY, att_json, GROUP),
        )
        return cur.rowcount > 0
    except Exception as e:
        print(f"  DB error: {e}", file=sys.stderr)
        return False


# ---------- Main ----------

def main():
    max_pages = MAX_PAGES
    if len(sys.argv) > 1:
        try:
            max_pages = int(sys.argv[1])
        except:
            pass

    print(f"Scraping {SITE_NAME}, max pages: {max_pages}", file=sys.stderr)
    conn = init_db()
    total_new = 0
    total_found = 0
    start_time = time.time()

    def do_clean():
        conn.execute("DELETE FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
        conn.commit()
    print("  Cleaning old data...", file=sys.stderr)
    with_retry(do_clean, desc=f"Clean {SITE_NAME}")
    print(f"  Cleaned old data for {SITE_NAME}", file=sys.stderr)

    for pn in range(1, max_pages + 1):
        articles = extract_list_page(pn)
        if not articles:
            print(f"  Page {pn}: 0 items (end)", file=sys.stderr)
            break

        total_found += len(articles)
        batch_new = 0

        for title, page_url, date in articles:
            cur = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
            if cur.fetchone():
                continue

            det_title, det_date, det_content, det_atts = fetch_detail(page_url)
            final_title = det_title or title
            final_date = det_date or date
            # Always try to have meaningful content
            if det_content:
                final_content = det_content
            elif det_atts:
                # Content is just the attachment link
                att_lines = [f"[{a['name']}]({a['url']})" for a in det_atts]
                final_content = "\n\n".join(att_lines)
            else:
                final_content = f'<p><a href="{page_url}">{final_title}</a></p>'
            final_atts = det_atts or []

            if save_article(conn, final_title, page_url, final_date, final_content, final_atts):
                total_new += 1
                batch_new += 1

            time.sleep(0.3)

        elapsed = int(time.time() - start_time)
        pages_left = max_pages - pn
        eta = int(pages_left / max(1, pn / max(1, elapsed)) / 60) if elapsed > 0 and pn > 0 else 0

        print(f"  Page {pn}/{max_pages}: +{batch_new} new, "
              f"{total_new}/{total_found} total, "
              f"{elapsed//60}:{elapsed%60:02d} elapsed, "
              f"ETA {eta}min", file=sys.stderr)

        if pn % 5 == 0:
            conn.commit()

    conn.commit()
    elapsed = int(time.time() - start_time)
    conn.close()
    print(f"\nDone! Found: {total_found}, New: {total_new}, Time: {elapsed//60}m{elapsed%60:02d}s", file=sys.stderr)


if __name__ == "__main__":
    main()