#!/usr/bin/env python3
"""广安经济技术开发区 - 公示公告"""
import re, sys, os, json
import urllib.request, urllib.error
from bs4 import BeautifulSoup
import sqlite3

BASE_URL = "https://gajkq.guang-an.gov.cn"
LIST_URL = BASE_URL + "/gjjgajjjskfq/c104015/pc/list.html"
SITE_NAME = "广安经开区-公示公告"
SITE_DISPLAY = "国家级广安经济技术开发区"
GROUP_NAME = "环评平台"
DB = os.environ.get("SEARCH_DB", "/root/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

_MAX_PG = None
for i, a in enumerate(sys.argv):
    if a == "--pages" and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)
def fetch(url):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        return urllib.request.urlopen(req, timeout=30).read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr)
        return ""

def extract_items_from_json(html):
    """Extract article list from embedded JSON"""
    # Find var listData = { ... };
    start = html.find("var listData = {")
    if start < 0:
        print("[WARN] No JSON data found in page", file=sys.stderr)
        return []
    # Find the last closing } of the object before </script>
    end_marker = html.find("</script>", start)
    if end_marker < 0:
        print("[WARN] No </script> found after JSON data", file=sys.stderr)
        return []
    # Find the last } before </script>
    end = html.rfind("}", start, end_marker)
    if end < 0:
        print("[WARN] No closing } for JSON", file=sys.stderr)
        return []
    json_str = html[start:end+1]  # include }
    # Convert JS object literal to valid JSON: quote unquoted keys
    json_str = re.sub(r'(?<=[,\{])\s*(\w+)(?=\s*:)', r'"\1"', json_str)
    json_str = json_str.replace("var listData = ", "").rstrip(";")
    try:
        data = json.loads(json_str)
    except json.JSONDecodeError as e:
        print(f"[WARN] JSON decode error: {e}", file=sys.stderr)
        return []

    items = []
    for art in data.get("articleList", []):
        title = art.get("showTitle", "") or art.get("title", "")
        pub_date = art.get("pubDate", "")
        # Extract date
        date_text = ""
        m2 = re.search(r"(\d{4}-\d{2}-\d{2})", pub_date)
        if m2:
            date_text = m2.group(1)
        # Get URL
        urls = art.get("urls", "{}")
        if isinstance(urls, str):
            try:
                urls = json.loads(urls)
            except:
                urls = {}
        pc_url = urls.get("pc", "")
        if pc_url and not pc_url.startswith("http"):
            pc_url = BASE_URL + pc_url
        
        items.append({"title": title, "url": pc_url, "date": date_text})
    return items

def get_page_url(page):
    if page == 1:
        return LIST_URL
    return f"{LIST_URL}?page={page}"

def parse_detail(html, url):
    soup = BeautifulSoup(html, "html.parser")
    title = ""
    h1 = soup.select_one("h1.title")
    if h1:
        title = h1.get_text(strip=True)
    if not title and soup.title:
        title = soup.title.string.strip()
        title = re.sub(r"\s*[-_]\s*国家级广安经济技术开发区\s*$", "", title).strip()
    date_text = ""
    m = re.search(r"发布日期[：:]\s*(\d{4}-\d{2}-\d{2})", html)
    if m:
        date_text = m.group(1)
    body = soup.select_one("div.article_main")
    if not body:
        print(f"[WARN] No content: {url}", file=sys.stderr)
        return {"title": title, "date": date_text, "content": "", "attachments": []}
    attachments = []
    parts = []
    for el in body.find_all(["p", "table", "img"], recursive=True):
        if el.name == "p" and el.find_parent("table"):
            continue
        if el.name == "table" and el.find_parent("table"):
            continue
        if el.name == "p":
            text = el.get_text(" ", strip=True)
            for a in el.find_all("a", href=True):
                href = a["href"].strip()
                if re.search(r'\.(pdf|docx?|xlsx?|zip|rar)$', href, re.I):
                    atitle = a.get_text(strip=True) or a.get("download", "")
                    ahref = BASE_URL + href if href.startswith("/") else (url.rsplit("/", 1)[0] + "/" + href if not href.startswith("http") else href)
                    if not any(att["url"] == ahref for att in attachments):
                        attachments.append({"title": atitle, "url": ahref})
            if text:
                parts.append(text)
        elif el.name == 'table':
            tbl_html = html_table_to_html(el, url)
            if tbl_html:
                rows.append(tbl_html)
        elif el.name == "img":
            src = el.get("src", "")
            alt = el.get("alt", "")
            if src:
                src = BASE_URL + src if src.startswith("/") else (url.rsplit("/", 1)[0] + "/" + src if not src.startswith("http") else src)
                parts.append(f"![{alt}]({src})" if alt else f"![]({src})")
    seen = set()
    ua = []
    for att in attachments:
        if att["url"] not in seen:
            seen.add(att["url"])
            ua.append(att)
    return {"title": title, "date": date_text, "content": "\n\n".join(parts), "attachments": ua}

def main():
    html = fetch(get_page_url(1))
    if not html:
        print("[ERROR] Cannot fetch list page", file=sys.stderr)
        sys.exit(1)
    # Get total pages from JSON
    m = re.search(r'articleCount["\']?\s*[:=]\s*(\d+)', html)
    total_count = int(m.group(1)) if m else 0
    page_size = 12
    total_pages = (total_count + page_size - 1) // page_size if total_count else 1
    print(f"[INFO] Total articles: {total_count}, pages: {total_pages}", file=sys.stderr)
    if _MAX_PG and total_pages > _MAX_PG:
        print(f"[INFO] Limiting to {_MAX_PG} pages (--pages={_MAX_PG})", file=sys.stderr)
        total_pages = _MAX_PG
    all_items = []
    for pg in range(1, total_pages + 1):
        url = get_page_url(pg)
        print(f"[INFO] Fetching page {pg}/{total_pages}", file=sys.stderr)
        html = fetch(url)
        if not html:
            continue
        items = extract_items_from_json(html)
        print(f"[INFO] Found {len(items)} items", file=sys.stderr)
        all_items.extend(items)
    print(f"[INFO] Total items: {len(all_items)}", file=sys.stderr)
    conn = sqlite3.connect(DB)
    conn.execute("PRAGMA busy_timeout = 60000")
    c = conn.cursor()
    new_count = empty_content = total_attachments = 0
    for item in all_items:
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
        if c.fetchone():
            continue
        html = fetch(item["url"])
        if not html:
            continue
        detail = parse_detail(html, item["url"])
        title = detail["title"] or item["title"]
        date_text = detail["date"] or item["date"]
        content = detail["content"]
        attachments = detail["attachments"]
        plain = re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', '', content)).strip()
        summary = plain[:200]
        if not content:
            empty_content += 1
        date_rank = 0
        if date_text:
            m = re.search(r"(\d{4})-(\d{2})-(\d{2})", date_text)
            if m:
                date_rank = int(m.group(1) + m.group(2) + m.group(3))
        att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]"
        try:
            c.execute("INSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank) VALUES (?,?,?,?,?,?,?,?,?)",
                      (title, SITE_NAME, GROUP_NAME, item["url"], date_text, content, summary, att_json, date_rank))
            if c.rowcount > 0:
                new_count += 1
                total_attachments += len(attachments)
        except Exception as e:
            print(f"[ERROR] DB: {title[:30]} - {e}", file=sys.stderr)
    conn.commit()
    conn.close()
    print(f"\n[RESULT] {SITE_NAME}: {new_count} new, {empty_content} empty, {total_attachments} attachments")

if __name__ == "__main__":
    main()