#!/usr/bin/env python3
"""
林西县 - 公告公示 (linxixian.gov.cn)
列表: zwdt/gggs/index.html → index_{N}.html (N=page-1, i=0→首页)
详情: ./202607/t20260713_XXXXXX.html
正文: div#fontzoom > div#articleContnet > div#article > div.trs_editor_view
15条/页, 34页
"""
import re, sys, os, json, time
import urllib.request, urllib.error
from bs4 import BeautifulSoup
import sqlite3

BASE_URL = "http://www.linxixian.gov.cn"
LIST_DIR = "/zwdt/gggs/"
LIST_URL = BASE_URL + LIST_DIR + "index.html"
SITE_NAME = "林西县-公告公示"
GROUP_NAME = "内蒙古"
DB = os.environ.get("SEARCH_DB", "/root/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

_MAX_PG = None
for i, a in enumerate(sys.argv):
    if a == "--pages" and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)
def fetch(url):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        return urllib.request.urlopen(req, timeout=30).read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr)
        return ""

def extract_items(html):
    """Extract items from list page - only items inside div.list > ul > li"""
    items = []
    # Find the div.list container first
    list_start = html.find('<div class="list">')
    if list_start < 0:
        return items
    list_end = html.find('</ul>', list_start)
    if list_end < 0:
        list_end = list_start + 10000
    list_html = html[list_start:list_end]

    for m in re.finditer(r'<li>.*?<a[^>]*href="([^"]+)"[^>]*>.*?<span>\[([^\]]+)\]</span>(.*?)</a>', list_html, re.I|re.S):
        href, date, title_html = m.group(1), m.group(2), m.group(3)
        # Clean title (strip HTML)
        title = re.sub(r'<[^>]+>', '', title_html).strip()
        if not title or len(title) < 6:
            continue
        # Make absolute URL
        url = href
        if href.startswith("./"):
            url = BASE_URL + LIST_DIR + href[2:]
        elif href.startswith("/"):
            url = BASE_URL + href
        elif not href.startswith("http"):
            url = BASE_URL + LIST_DIR + href
        items.append({"title": title, "url": url, "date": date.strip()})
    return items

def parse_detail(html, url):
    """Parse detail page"""
    soup = BeautifulSoup(html, "html.parser")

    # Title from h1
    title = ""
    h1 = soup.find("h1")
    if h1:
        title = h1.get_text(strip=True)

    # Date from article_info div
    date_text = ""
    info_div = soup.select_one("div.article_info")
    if info_div:
        info_text = info_div.get_text()
        m = re.search(r"(\d{4}-\d{2}-\d{2})", info_text)
        if m:
            date_text = m.group(1)

    # Content
    content = ""
    attachments = []
    content_div = soup.select_one("div#fontzoom") or soup.select_one("div.article_content_list")
    if not content_div:
        return {"title": title, "date": date_text, "content": "", "attachments": []}

    parts = []
    for el in content_div.find_all(["p", "table", "img"], recursive=True):
        if el.name == "p" and el.find_parent("table"):
            continue
        if el.name == "table" and el.find_parent("table"):
            continue

        if el.name == "img":
            src = el.get("src", "")
            # Skip decoration icons
            if "tb2.jpg" in src or "icon_" in src.lower() or "/sysimage/" in src.lower():
                continue
            alt = el.get("alt", "") or el.get("title", "")
            if src.startswith("/"):
                src = BASE_URL + src
            elif src.startswith(".."):
                src = url.rsplit("/", 1)[0] + "/" + src.lstrip("./")
            elif not src.startswith("http"):
                src = BASE_URL + LIST_DIR + src
            parts.append(f"![{alt}]({src})")
            continue

        if el.name == 'table':
            tbl_html = html_table_to_html(el, url)
            if tbl_html:
                parts.append(tbl_html)
                continue  # 表格已作富文本入正文, 避免再拍平
        text = el.get_text(" ", strip=True)
        if text:
            for a in el.find_all("a", href=True):
                ahref = a["href"]
                if re.search(r'\.(docx?|pdf|xlsx?|rar|zip)$', ahref, re.I):
                    atitle = a.get_text(strip=True) or a.get("download", "")
                    if ahref.startswith("/"):
                        ahref = BASE_URL + ahref
                    elif ahref.startswith(".."):
                        ahref = url.rsplit("/", 1)[0] + "/" + ahref.lstrip("./")
                    elif not ahref.startswith("http"):
                        ahref = BASE_URL + LIST_DIR + ahref
                    attachments.append({"title": atitle or "附件", "url": ahref})
            parts.append(text)

    content = "\n\n".join(p for p in parts if p.strip())
    content = content.replace("\u00a0", " ").replace("&nbsp;", " ")

    # Deduplicate attachments
    seen = set()
    unique_att = []
    for att in attachments:
        if att["url"] not in seen:
            seen.add(att["url"])
            unique_att.append(att)

    return {"title": title, "date": date_text, "content": content, "attachments": unique_att}


def main():
    conn = sqlite3.connect(DB, timeout=60)
    c = conn.cursor()

    all_items = []

    # Page 1: index.html
    html = fetch(LIST_URL)
    if not html:
        print("[ERROR] Cannot fetch list page", file=sys.stderr)
        sys.exit(1)

    items = extract_items(html)
    if not items:
        print("[ERROR] No items found on page 1", file=sys.stderr)
        sys.exit(1)

    print(f"[INFO] Page 1: {len(items)} items", file=sys.stderr)
    all_items.extend(items)

    # Determine total pages from the JS: countPage=34
    m = re.search(r'var countPage\s*=\s*(\d+)', html)
    total_pages = int(m.group(1)) if m else 34
    print(f"[INFO] Total pages: {total_pages}", file=sys.stderr)

    if _MAX_PG and total_pages > _MAX_PG:
        total_pages = _MAX_PG
        print(f"[INFO] Limited to {_MAX_PG} pages", file=sys.stderr)

    # Pages 2+: index_{N}.html (N=1 to total_pages-1)
    for pg in range(1, total_pages):
        url = f"{BASE_URL}{LIST_DIR}index_{pg}.html"
        html = fetch(url)
        if not html:
            break
        items = extract_items(html)
        if not items:
            break
        print(f"[INFO] Page {pg+1}: {len(items)} items", file=sys.stderr)
        all_items.extend(items)

    print(f"[INFO] Total: {len(all_items)} items from {total_pages} pages", file=sys.stderr)

    new_count = 0
    empty_count = 0
    total_att = 0

    for idx, item in enumerate(all_items):
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
        if c.fetchone():
            continue

        if idx % 5 == 0:
            print(f"[INFO] [{idx}/{len(all_items)}] {item['title'][:40]}...", file=sys.stderr, end=" ", flush=True)

        html = fetch(item["url"])
        if not html:
            if idx % 5 == 0:
                print("FETCH FAILED", file=sys.stderr)
            continue

        detail = parse_detail(html, item["url"])
        title = detail["title"] or item["title"]
        date_text = detail["date"] or item["date"]
        content = detail["content"]
        attachments = detail["attachments"]

        if not content.strip():
            empty_count += 1
            if idx % 5 == 0:
                print("CONTENT EMPTY", file=sys.stderr)
            continue

        plain = re.sub(r'\s+', ' ', content).strip()
        summary = plain[:200] if plain else title[:200]
        att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]"

        date_rank = 0
        if date_text:
            m = re.search(r"(\d{4})-(\d{2})-(\d{2})", date_text)
            if m:
                date_rank = int(m.group(1) + m.group(2) + m.group(3))

        try:
            c.execute(
                """INSERT OR IGNORE INTO gov_raw 
                (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank)
                VALUES (?,?,?,?,?,?,?,?,?)""",
                (title, SITE_NAME, GROUP_NAME, item["url"], date_text, content, summary, att_json, date_rank)
            )
            if c.rowcount > 0:
                new_count += 1
                total_att += len(attachments)
        except Exception as e:
            if idx % 5 == 0:
                print(f"DB ERROR: {e}", file=sys.stderr)

        time.sleep(0.5)

    conn.commit()
    conn.close()

    print(f"\n[RESULT] {SITE_NAME}: {new_count} new, {empty_count} empty, {total_att} attachments")


if __name__ == "__main__":
    main()