#!/usr/bin/env python3
"""
临邑县 - 公告公示 (linyixian.gov.cn)
列表: n30164361/n30165152/n30295960/index.html
分页: index_80988335_{N}.html (N从0开始)
详情: n30164361/n30165152/n30295960/cXXXXX/content.html
正文: div.content > p
"""
import re, sys, os, json, time
import urllib.request, urllib.error
from bs4 import BeautifulSoup
import sqlite3

BASE_URL = "http://www.linyixian.gov.cn"
LIST_DIR = "/n30164361/n30165152/n30295960/"
LIST_URL = BASE_URL + LIST_DIR + "index.html"
SITE_NAME = "临邑县-公告公示"
GROUP_NAME = "山东"
DB = os.environ.get("SEARCH_DB", "/root/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

_MAX_PG = None
for i, a in enumerate(sys.argv):
    if a == "--pages" and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break

# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)
def fetch(url):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        return urllib.request.urlopen(req, timeout=30).read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr)
        return ""

def extract_items(html):
    """Extract items from a list page"""
    items = []
    for m in re.finditer(r'<li>.*?<a[^>]*href="([^"]+)"[^>]*title="([^"]+)".*?<span class="date">([^<]+)</span>', html, re.I|re.S):
        href, title, date = m.group(1), m.group(2), m.group(3)
        # Clean title (may contain leading whitespace from the HTML)
        title = title.strip()
        if not title or len(title) < 6:
            continue
        # Make absolute URL
        url = href.replace("../../../", BASE_URL + "/")
        if not url.startswith("http"):
            url = BASE_URL + LIST_DIR + href.lstrip("./")
        items.append({"title": title, "url": url, "date": date.strip()})
    return items

def parse_detail(html, url):
    """Parse detail page"""
    soup = BeautifulSoup(html, "html.parser")

    # Title from meta (most reliable)
    title = ""
    meta = soup.find("meta", attrs={"property": "ArticleTitle"})
    if meta and meta.get("content"):
        title = meta["content"].strip()
    if not title:
        meta = soup.find("meta", attrs={"name": "ArticleTitle"})
        if meta and meta.get("content"):
            title = meta["content"].strip()
    if not title:
        title_div = soup.select_one("div.title")
        if title_div:
            title = title_div.get_text(strip=True)

    # Date from meta
    date_text = ""
    meta = soup.find("meta", attrs={"property": "PubDate"})
    if meta and meta.get("content"):
        m = re.search(r"(\d{4}-\d{2}-\d{2})", meta["content"])
        if m:
            date_text = m.group(1)
    if not date_text:
        meta = soup.find("meta", attrs={"name": "PubDate"})
        if meta and meta.get("content"):
            m = re.search(r"(\d{4}-\d{2}-\d{2})", meta["content"])
            if m:
                date_text = m.group(1)

    # Content from div.content
    content = ""
    attachments = []
    content_div = soup.select_one("div.content")
    if not content_div:
        # Fallback: try other common containers
        content_div = soup.select_one("div.TRS_Editor") or soup.select_one("div.Custom_UnionStyle") or soup.select_one("div#Zoom")

    if content_div:
        parts = []
        for el in content_div.find_all(["p", "table", "img"], recursive=True):
            # Skip elements inside tables (handled by table rendering)
            if el.name == "p" and el.find_parent("table"):
                continue

            if el.name == "img":
                src = el.get("src", "")
                # Skip decoration icons
                if "icon_" in src.lower() or "/sysimage/" in src.lower():
                    continue
                alt = el.get("alt", "") or el.get("title", "")
                # Make absolute URL
                if src.startswith("/"):
                    src = BASE_URL + src
                elif src.startswith(".."):
                    # Resolve relative path
                    src = url.rsplit("/", 1)[0] + "/" + src.lstrip("./")
                elif not src.startswith("http"):
                    src = BASE_URL + LIST_DIR + src
                parts.append(f"![{alt}]({src})")
                continue

            if el.name == 'table':
                tbl_html = html_table_to_html(el, url)
                if tbl_html:
                    parts.append(tbl_html)
                    continue  # 表格已作富文本入正文, 避免再拍平
            text = body_text(el)
            if text:
                # Check for attachment links in this paragraph
                for a in el.find_all("a", href=True):
                    ahref = a["href"]
                    if re.search(r'\.(docx?|pdf|xlsx?|rar|zip)$', ahref, re.I):
                        atitle = a.get_text(strip=True) or a.get("download", "")
                        if ahref.startswith("/"):
                            ahref = BASE_URL + ahref
                        elif ahref.startswith(".."):
                            ahref = url.rsplit("/", 1)[0] + "/" + ahref.lstrip("./")
                        elif not ahref.startswith("http"):
                            ahref = BASE_URL + LIST_DIR + ahref
                        attachments.append({"title": atitle or "附件", "url": ahref})
                parts.append(text)

        content = "\n\n".join(parts)
        # Clean &nbsp;
        content = content.replace("\u00a0", " ").replace("&nbsp;", " ")

    # Deduplicate attachments
    seen = set()
    unique_att = []
    for att in attachments:
        if att["url"] not in seen:
            seen.add(att["url"])
            unique_att.append(att)

    return {"title": title, "date": date_text, "content": content, "attachments": unique_att}


def main():
    conn = sqlite3.connect(DB, timeout=60)
    c = conn.cursor()

    all_items = []
    total_pages = 0

    # Page 1: index.html
    html = fetch(LIST_URL)
    if html:
        items = extract_items(html)
        print(f"[INFO] Page 1: {len(items)} items", file=sys.stderr)
        all_items.extend(items)
        total_pages = 1

    # Pages 2+: index_80988335_{N}.html
    for p in range(1, 100):
        if _MAX_PG and total_pages >= _MAX_PG:
            break
        url = f"{BASE_URL}{LIST_DIR}index_80988335_{p}.html"
        html = fetch(url)
        if not html:
            break
        items = extract_items(html)
        if not items:
            break
        print(f"[INFO] Page {total_pages+1}: {len(items)} items", file=sys.stderr)
        all_items.extend(items)
        total_pages += 1
        if len(items) < 14:
            # Short or last page - try one more page then stop
            # (some site patterns have a partial page in the middle)
            pass

    print(f"[INFO] Total pages: {total_pages}, total items: {len(all_items)}", file=sys.stderr)

    new_count = 0
    empty_count = 0
    total_att = 0

    for idx, item in enumerate(all_items):
        # Check if exists
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
        if c.fetchone():
            continue

        print(f"[INFO] [{idx+1}/{len(all_items)}] {item['title'][:40]}...", file=sys.stderr, end=" ")

        html = fetch(item["url"])
        if not html:
            print("FETCH FAILED", file=sys.stderr)
            continue

        detail = parse_detail(html, item["url"])
        title = detail["title"] or item["title"]
        date_text = detail["date"] or item["date"]
        content = detail["content"]
        attachments = detail["attachments"]

        if not content:
            empty_count += 1
            print("CONTENT EMPTY", file=sys.stderr)
            continue

        # Summary
        plain = re.sub(r'\s+', ' ', content).strip()
        summary = plain[:200] if plain else title[:200]

        att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]"

        # Date rank
        date_rank = 0
        if date_text:
            m = re.search(r"(\d{4})-(\d{2})-(\d{2})", date_text)
            if m:
                date_rank = int(m.group(1) + m.group(2) + m.group(3))

        try:
            c.execute(
                """INSERT OR IGNORE INTO gov_raw 
                (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank)
                VALUES (?,?,?,?,?,?,?,?,?)""",
                (title, SITE_NAME, GROUP_NAME, item["url"], date_text, content, summary, att_json, date_rank)
            )
            if c.rowcount > 0:
                new_count += 1
                total_att += len(attachments)
                print(f"OK ({len(content)}字) att:{len(attachments)}", file=sys.stderr)
            else:
                print("DUPLICATE", file=sys.stderr)
        except Exception as e:
            print(f"DB ERROR: {e}", file=sys.stderr)

        time.sleep(0.3)

    conn.commit()
    conn.close()

    print(f"\n[RESULT] 临邑县-公告公示: {new_count} new, {empty_count} empty, {total_att} attachments")


if __name__ == "__main__":
    main()