#!/usr/bin/env python3
"""
ws.cq.gov.cn — 重庆市万盛经开区 → 公示公告 (gsgg)
"""
import sys, os, re, json, time, warnings
warnings.filterwarnings("ignore")
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
import requests
from bs4 import BeautifulSoup
import urllib.parse

SITE_NAME = "重庆市万盛经开区-公示公告"
BASE_URL = "https://ws.cq.gov.cn/zwgk_165/gsgg/index.html"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url, retries=3):
    for attempt in range(retries):
        try:
            r = requests.get(url, headers=HEADERS, verify=False, timeout=30)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            print("  [ERR] %s: %s" % (url[:80], e), file=sys.stderr)
            if attempt < retries - 1:
                time.sleep(2)
    return None

def parse_list(html, page_url):
    """返回 [(title, url, date), ...]"""
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for a in soup.select("a[href*='t20']"):
        href = a.get("href", "").strip()
        if not href:
            continue
        title = a.get("title") or a.get_text(" ", strip=True)
        if not title or len(title) < 5:
            continue
        title = title.strip()
        # Clean title suffix
        title = re.sub(r"[\s_]*重庆市万盛经济技术开发区管理委员会[\s_]*$", "", title).strip()

        # Resolve URL
        if href.startswith("./"):
            href = "https://ws.cq.gov.cn/zwgk_165/gsgg/" + href[2:]
        elif href.startswith("../../"):
            href = "https://ws.cq.gov.cn/" + href[6:]
        elif not href.startswith("http"):
            href = "https://ws.cq.gov.cn/zwgk_165/gsgg/" + href.lstrip("/")

        # Date from parent li span
        date = ""
        li = a.find_parent("li")
        if li:
            sp = li.find("span")
            if sp:
                date = sp.get_text().strip()
        if not date:
            m = re.search(r"(\d{4}-\d{2}-\d{2})", li.get_text() if li else "")
            if m:
                date = m.group(1)
        items.append((title, href, date))
    return items

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def parse_detail(html):
    soup = BeautifulSoup(html, "html.parser")
    title_tag = soup.find("title")
    title = title_tag.get_text().strip() if title_tag else ""
    title = re.sub(r"[\s_]*重庆市万盛经济技术开发区管理委员会[\s_]*$", "", title).strip()
    article = soup.select_one("div.zwxl-article")
    if not article:
        article = soup.find("div", class_=lambda c: c and "TRS_UEDITOR" in c)
    if not article:
        return title, ""
    parts = []
    for tag in article.find_all(["p", "table", "img"], recursive=True):
        if tag.name == "p":
            text = tag.get_text(" ", strip=True)
            if text:
                parts.append(text)
        elif tag.name == "table":
            md = html_table_to_html(tag)
            if md:
                parts.append(md)
        elif tag.name == "img":
            src = tag.get("src", "")
            if src:
                if not src.startswith("http"):
                    src = "https://ws.cq.gov.cn" + src
                parts.append("![%s](%s)" % (tag.get("alt","图片"), src))
    return title, "\n\n".join(parts)

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=3)
    args = parser.parse_args()

    all_items = []
    seen = set()
    for pn in range(1, args.pages + 1):
        url = BASE_URL if pn == 1 else BASE_URL.replace("index.html", "index_%d.html" % pn)
        print("Page %d: %s" % (pn, url))
        html = fetch(url)
        if not html:
            continue
        items = parse_list(html, url)
        print("  Found %d items" % len(items))
        for t, u, d in items:
            if u not in seen:
                seen.add(u)
                all_items.append((t, u, d))
        if not items:
            break

    print("\nTotal: %d items" % len(all_items))
    for idx, (title, url, date) in enumerate(all_items):
        print("[%d/%d] %s" % (idx+1, len(all_items), title[:50]))
        html = fetch(url)
        if not html:
            continue
        dt, content = parse_detail(html)
        if not content:
            print("  Empty, skip")
            continue
        rec = {
            "title": dt or title,
            "url": url,
            "source_url": url,
            "content": content,
            "pub_date": date,
            "site_name": SITE_NAME,
        }
        push_to_searchdb([rec])
        time.sleep(0.2)
    print("\nDone!")

if __name__ == "__main__":
    main()