#!/usr/bin/env python3
"""榆林市生态环境局 - 行政许可"""
import re, sys, os, json, time
import urllib.request, urllib.error
from bs4 import BeautifulSoup
import sqlite3

BASE_URL = "https://hb.yl.gov.cn"
LIST_PATH = "/hbyw/xzxk/"
SITE_NAME = "榆林市-行政许可"
SITE_DISPLAY = "榆林市生态环境局"
GROUP_NAME = "环评平台"
DB = os.environ.get("SEARCH_DB", "/root/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

_MAX_PG = None
for i, a in enumerate(sys.argv):
    if a == "--pages" and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)
def fetch(url):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        return urllib.request.urlopen(req, timeout=30).read().decode("utf-8", errors="replace")
    except Exception as e:
        print("[WARN] fetch failed: %s - %s" % (url, e), file=sys.stderr)
        return ""

def extract_items(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for li in soup.select("ul.list-item > li"):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"].strip()
        title = a.get("title", "") or a.get_text(strip=True)
        if "&lt;" in title:
            title = a.get_text(strip=True)
        date_span = li.find("span", class_="date")
        date_text = date_span.get_text(strip=True) if date_span else ""
        if href.startswith("./"):
            href = BASE_URL + LIST_PATH.rstrip("/") + "/" + href[2:]
        elif href.startswith("/"):
            href = BASE_URL + href
        elif not href.startswith("http"):
            href = BASE_URL + LIST_PATH.rstrip("/") + "/" + href
        items.append({"title": title.strip(), "url": href, "date": date_text})
    return items

def get_pagination_info(html):
    m = re.search(r'var countPage\s*=\s*(\d+)', html)
    if m:
        return int(m.group(1))
    return None

def parse_detail(html, url):
    soup = BeautifulSoup(html, "html.parser")

    title = ""
    h1 = soup.select_one("h1.title")
    if h1:
        title = h1.get_text(strip=True)
    if not title and soup.title:
        title = soup.title.string.strip()

    date_text = ""
    m = re.search(r"发布时间[：:]\s*(\d{4}-\d{2}-\d{2})", html)
    if m:
        date_text = m.group(1)

    content_div = soup.select_one("div.con-box div.content")
    if not content_div:
        content_div = soup.find("div", class_="con-box")

    content_parts = []
    attachments = []

    if content_div:
        for el in content_div.find_all(["p", "table", "img"], recursive=True):
            if el.name == "img":
                src = el.get("src", "").strip()
                alt = el.get("alt", "").strip()
                if src and not src.startswith("data:"):
                    if src.startswith("./"):
                        src = BASE_URL + LIST_PATH.rstrip("/") + "/" + src[2:]
                    elif src.startswith("/"):
                        src = BASE_URL + src
                    elif not src.startswith("http"):
                        src = BASE_URL + LIST_PATH.rstrip("/") + "/" + src
                    if "fileTypeImages" not in src:
                        content_parts.append("![%s](%s)" % (alt, src))
                continue

            if el.name == "p" and el.find_parent("table"):
                continue

            if el.name == 'table':
                tbl_html = html_table_to_html(el, url)
                if tbl_html:
                    cells.append(tbl_html)
            text = el.get_text(strip=True)
            if text:
                content_parts.append(text)

    # Separate pass: find attachment links
    for a in (content_div or soup).find_all("a", href=True):
        h = a["href"].lower()
        if h.endswith((".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar")):
            ah = a["href"]
            if ah.startswith("./"):
                ah = BASE_URL + LIST_PATH.rstrip("/") + "/" + ah[2:]
            elif ah.startswith("/"):
                ah = BASE_URL + ah
            elif not ah.startswith("http"):
                ah = BASE_URL + LIST_PATH.rstrip("/") + "/" + ah
            attachments.append({"name": a.get_text(strip=True) or ah.split("/")[-1], "url": ah})

    content = "\n\n".join(p for p in content_parts if p.strip())
    return {"title": title, "date": date_text, "content": content, "attachments": attachments}

def main():
    total_new = 0
    conn = sqlite3.connect(DB, timeout=60)
    c = conn.cursor()

    all_items = []
    page_num = 0

    first_html = fetch(BASE_URL + LIST_PATH)
    if not first_html:
        print("[FAIL] Cannot fetch page 0", file=sys.stderr)
        conn.close()
        return

    total_pages = get_pagination_info(first_html)
    if total_pages:
        print("[INFO] Total pages: %d" % total_pages, file=sys.stderr)

    items_page0 = extract_items(first_html)
    all_items.extend(items_page0)
    print("[INFO] Page 0: %d items" % len(items_page0), file=sys.stderr)

    page_num = 1
    while True:
        if _MAX_PG and page_num >= _MAX_PG:
            break
        if total_pages and page_num >= total_pages:
            break

        url = BASE_URL + LIST_PATH + "index_%d.html" % (page_num + 1)
        html = fetch(url)
        if not html:
            break

        items = extract_items(html)
        if not items:
            break

        print("[INFO] Page %d: %d items" % (page_num, len(items)), file=sys.stderr)
        all_items.extend(items)
        page_num += 1

    print("[INFO] Total collected: %d items" % len(all_items), file=sys.stderr)

    for idx, item in enumerate(all_items):
        if not item["url"] or "yl.gov.cn" not in item["url"]:
            continue

        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
        if c.fetchone():
            continue

        html = fetch(item["url"])
        if not html:
            continue

        detail = parse_detail(html, item["url"])
        title = detail["title"] or item["title"]
        date_text = detail["date"] or item["date"]
        content = detail["content"]
        attachments_json = json.dumps(detail["attachments"], ensure_ascii=False) if detail.get("attachments") else "[]"

        if not content.strip():
            content = title
            print("[WARN] Empty content: %s" % title[:40], file=sys.stderr)

        summary = content[:200].replace("\n", " ").strip()

        try:
            c.execute("INSERT OR IGNORE INTO gov_raw (page_url, title, site_name, group_name, publish_date, summary, content, attachments, date_rank) VALUES (?,?,?,?,?,?,?,?,?)",
                      (item["url"], title, SITE_NAME, GROUP_NAME, date_text, summary, content, attachments_json, date_text))
            conn.commit()
            if c.rowcount:
                total_new += 1
                if total_new % 10 == 0:
                    print("[INFO] Inserted %d/%d" % (total_new, len(all_items)), file=sys.stderr)
        except Exception as e:
            print("[WARN] DB insert failed: %s - %s" % (title[:30], e), file=sys.stderr)

    conn.close()
    print("[DONE] 榆林: %d new items" % total_new)

if __name__ == "__main__":
    main()