#!/usr/bin/env python3
"""
泸县双公示 - 强制重跑版 (v2.3)
重新爬取所有现有记录(不论是否有内容), 修复表格渲染
"""
import os, re, sys, json, time
from bs4 import BeautifulSoup
from urllib import request
from urllib.parse import urljoin
import ssl
import urllib.parse

SITE_NAME = "泸县人民政府-双公示"
BASE_URL = "https://www.luxian.gov.cn"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
DELAY = 3.0

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

ssl._create_default_https_context = ssl._create_unverified_context


def fetch(url, timeout=30):
    req = request.Request(url, headers=HEADERS)
    try:
        resp = request.urlopen(req, timeout=timeout)
        return resp.read().decode("utf-8", errors="ignore")
    except Exception as e:
        print("  [ERROR] %s: %s" % (url, e), file=sys.stderr)
        return None


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def _extract_elements_sorted(container):
    results = []
    for m in re.finditer(r"<(p|table|img)\b[^>]*>", container, re.DOTALL):
        start, m_end, tag = m.start(), m.end(), m.group(1)
        if tag in ("p", "table"):
            close_tag = "</" + tag + ">"
            end_pos = container.find(close_tag, m_end)
            if end_pos >= 0:
                el_html = container[start:end_pos + len(close_tag)]
            else:
                ns = container.find("<", m_end)
                el_html = container[start:ns] if ns >= 0 else container[start:]
            results.append((start, tag, el_html))
        elif tag == "img":
            cp = container.find(">", m_end)
            if cp >= 0:
                results.append((start, tag, container[start:cp + 1]))
    return results


def parse_detail(html, url):
    content_parts = []
    attachments = []

    container = None
    for pat in [
        r'<div[^>]*class="conTxt"[^>]*>(.*?)</div>\s*</div>',
        r'<div[^>]*class="conTxt"[^>]*>(.*?)</div>',
        r'<div[^>]*class="content"[^>]*>(.*?)</div>\s*</div>',
    ]:
        m = re.search(pat, html, re.DOTALL)
        if m:
            container = m.group(1)
            break
    if not container:
        body_m = re.search(r"<body[^>]*>(.*?)</body>", html, re.DOTALL)
        if body_m:
            for p in re.findall(r"<p[^>]*>(.*?)</p>", body_m.group(1), re.DOTALL):
                text = re.sub(r"<[^>]+>", "", p).strip()
                if text:
                    content_parts.append(text)
        return "\n\n".join(content_parts), []

    table_ranges = []
    for tm in re.finditer(r"<table[^>]*>.*?</table>", container, re.DOTALL):
        table_ranges.append((tm.start(), tm.end()))

    def inside_table(pos):
        return any(ts <= pos <= te for ts, te in table_ranges)

    elements = []
    for pos, tag, el_html in _extract_elements_sorted(container):
        if tag == "p":
            if inside_table(pos):
                continue
            if "<td" in el_html or "<th" in el_html:
                continue
            text = re.sub(r"<[^>]+>", "", el_html).strip()
            text = re.sub(r"\s+", " ", text)
            if text:
                elements.append((pos, text))
        elif tag == "table":
            if el_html.count("<td") > 0:
                md = html_table_to_html(el_html)
                if md:
                    elements.append((pos, md))
        elif tag == "img":
            src_m = re.search(r'src="([^"]+)"', el_html)
            alt_m = re.search(r'alt="([^"]*)"', el_html)
            if src_m:
                src = src_m.group(1)
                full_src = urljoin(url, src)
                alt = alt_m.group(1) if alt_m else ""
                elements.append((pos, "![%s](%s)" % (alt, full_src)))

    elements.sort(key=lambda x: x[0])
    content = "\n\n".join(text for _, text in elements)

    for a_href, a_text in re.findall(
        r'<a[^>]*href="([^"]+\.(?:doc|docx|pdf|xls|xlsx|ppt|pptx|zip|rar|7z|txt|wps|et))"[^>]*>([^<]+)</a>',
        html, re.DOTALL
    ):
        attachments.append({"title": a_text.strip(), "url": urljoin(url, a_href)})

    return content, attachments


def refetch_and_update():
    import sqlite3
    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()

    cur.execute("SELECT id, page_url, title, publish_date FROM gov_raw WHERE site_name=? ORDER BY id", (SITE_NAME,))
    rows = cur.fetchall()
    total = len(rows)
    print("待处理 %d 条记录" % total)

    updated = 0
    failed = 0

    for idx, (rid, page_url, title, pdate) in enumerate(rows, 1):
        print("[%d/%d] %s ... " % (idx, total, title[:40]), end="", flush=True)
        time.sleep(DELAY)

        html = fetch(page_url)
        if not html:
            print("失败, 等待10s重试...")
            time.sleep(10)
            html = fetch(page_url)
            if not html:
                print("跳过")
                failed += 1
                continue

        content, attachments = parse_detail(html, page_url)
        aj = json.dumps(attachments, ensure_ascii=False)
        summary = content[:200] if content else title
        cur.execute("UPDATE gov_raw SET content=?, attachments=?, summary=? WHERE id=?",
                   (content, aj, summary, rid))

        print("OK -> %d chars" % len(content))
        updated += 1

        if idx % 10 == 0:
            conn.commit()

    conn.commit()
    conn.close()
    print("\n[DONE] 完成: %d/%d, 失败: %d" % (updated, total, failed))


def crawl_new(max_pages=5):
    """保留普通增量爬取功能"""
    import sqlite3
    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()
    new_count = skip_count = 0

    for page in range(1, max_pages + 1):
        if page == 1:
            list_url = BASE_URL + "/zwgk/zwzt/zxzt/xylx/sgs"
        else:
            list_url = BASE_URL + "/zwgk/zwzt/zxzt/xylx/sgs_%d" % page

        print("[分页] 第%d页: %s" % (page, list_url), flush=True)
        time.sleep(DELAY)
        html = fetch(list_url)
        if not html:
            break

        items = set()
        for m in re.finditer(r'href="([^"]*content_\d+)"[^>]*>([^<]+)</a>', html):
            href = m.group(1)
            title = m.group(2).strip()
            if not title or len(title) < 5:
                continue
            pos = m.start()
            ctx = html[max(0, pos-200):pos]
            date = ""
            dm = re.search(r"(\d{4}-\d{2}-\d{2})", ctx)
            if dm:
                date = dm.group(1)
            items.add((title, BASE_URL + href if href.startswith("/") else urljoin(BASE_URL, href), date))

        if not items:
            break
        print("  找到 %d 条" % len(items), flush=True)

        for title, detail_url, date in items:
            cur.execute("SELECT id FROM gov_raw WHERE page_url=? AND site_name=?", (detail_url, SITE_NAME))
            if cur.fetchone():
                skip_count += 1
                continue

            time.sleep(DELAY)
            detail_html = fetch(detail_url)
            if not detail_html:
                skip_count += 1
                continue

            content, attachments = parse_detail(detail_html, detail_url)
            aj = json.dumps(attachments, ensure_ascii=False)
            summary = content[:200] if content else title

            cur.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name,page_url,title,content,publish_date,attachments,summary) VALUES(?,?,?,?,?,?,?)",
                (SITE_NAME, detail_url, title, content, date, aj, summary)
            )
            if cur.rowcount > 0:
                new_count += 1
                if content:
                    print("  [正文] %s -> %d chars" % (title[:30], len(content)), flush=True)
        conn.commit()

    conn.close()
    print("\n[DONE] 新增=%d, 跳过=%d" % (new_count, skip_count), flush=True)


if __name__ == "__main__":
    if len(sys.argv) > 1 and sys.argv[1] == "refetch":
        refetch_and_update()
    else:
        max_p = int(sys.argv[1]) if len(sys.argv) > 1 else 5
        crawl_new(max_p)