#!/usr/bin/env python3
"""
融水苗族自治县人民政府 - 通知公告 爬虫
TRS CMS, createPageHTML(128, 0, "index", "shtml", "1919")
15条/页, 128页共约1920条
"""

import sys, re, time, hashlib, sqlite3, subprocess
from datetime import datetime
from urllib.parse import urljoin
import requests

BASE_URL = "http://www.rongshui.gov.cn/tzgg/"
SESSION = requests.Session()
SESSION.headers.update({
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
})
SITE_NAME = "融水苗族自治县通知公告"
GROUP = "县区"
DB_PATH = "/mnt/data/search.db"

def fetch(url, max_retries=3):
    for attempt in range(max_retries):
        try:
            resp = SESSION.get(url, timeout=30)
            resp.encoding = 'utf-8'
            if resp.status_code == 200:
                return resp.text
        except Exception as e:
            pass
        time.sleep(2)
    return None

def parse_list_page(html, base_url):
    items = []
    for quote in ['"', "'"]:
        qesc = re.escape(quote)
        pattern = re.compile(
            rf'<li\s+class="list-group-item\s+hidden-xs"[^>]*>.*?'
            rf'<a[^>]*href={qesc}([^{quote}]+?){qesc}[^>]*title={qesc}([^{quote}]*?){qesc}[^>]*>.*?</a>.*?'
            rf'class="layout-fixed"[^>]*>([^<]+)</div>',
            re.DOTALL
        )
        for m in pattern.finditer(html):
            href = m.group(1).strip()
            title = m.group(2).replace('&nbsp;', ' ').replace('\u00a0', ' ').strip()
            date = m.group(3).strip()
            if title and href:
                items.append((title, urljoin(base_url, href), date))
    return items

def extract_pagination(html):
    m = re.search(r'createPageHTML\s*\(\s*(\d+)', html)
    return int(m.group(1)) if m else 1

def parse_detail(html, page_url):
    title = ""
    m = re.search(r'<meta\s+name="ArticleTitle"[^>]*content="([^"]*)"', html, re.IGNORECASE)
    if m: title = m.group(1).strip()
    if not title:
        m = re.search(r'<h3[^>]*class="marginB15"[^>]*>(.*?)</h3>', html, re.DOTALL)
        if m: title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    title = title.replace('&nbsp;', ' ').replace('\u00a0', ' ').strip()

    publish_date = ""
    m = re.search(r'<meta\s+name="PubDate"[^>]*content="([^"]*)"', html, re.IGNORECASE)
    if m:
        m2 = re.match(r'(\d{4}-\d{2}-\d{2})', m.group(1).strip())
        if m2: publish_date = m2.group(1)
    if not publish_date:
        m = re.search(r'发布日期[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', html)
        if m: publish_date = m.group(1).replace('/', '-')

    # Content body
    content_html = ""
    idx = html.find('class="trs_editor_view')
    if idx == -1:
        idx = html.find('class=\"trs_editor_view')
    if idx == -1:
        idx = html.find('class="contentTextBox"')
    if idx == -1:
        idx = html.find('class=\"contentTextBox\"')

    if idx != -1:
        start = html.find('>', idx) + 1
        end_marker = html.find('<!-- 正文e -->', start)
        if end_marker == -1:
            end_marker = html.find('id="downloadfile"', start)
        if end_marker == -1:
            end_marker = html.find('class="downloadfile"', start)
        if end_marker == -1:
            end_marker = html.find('</div>', html.rfind('<div', 0, start))
        if end_marker != -1:
            content_html = html[start:end_marker].strip()

    # Attachments
    attachments = []
    aidx = html.find('id="downloadfile"')
    if aidx == -1:
        aidx = html.find('id=\"downloadfile\"')
    if aidx != -1:
        list_start = html.find('<ul', aidx)
        list_end = html.find('</ul>', list_start) if list_start != -1 else -1
        if list_start != -1 and list_end != -1:
            ul_html = html[list_start:list_end + 5]
            for m in re.finditer(r'<a\s+href="([^"]+)"[^>]*>([^<]+)</a>', ul_html):
                attachments.append((m.group(2).strip(), urljoin(page_url, m.group(1).strip())))

    return title, publish_date, content_html, attachments

def render_content(content_html, attachments, page_url):
    if not content_html.strip():
        if attachments:
            return '\n'.join([f'[{name}]({href})' for name, href in attachments])
        return ""

    content_html = re.sub(r'<br\s*/?>', '\n', content_html)
    parts = []
    pos = 0

    while pos < len(content_html):
        tags = [('p', content_html.find('<p', pos)),
                ('table', content_html.find('<table', pos)),
                ('div', content_html.find('<div', pos)),
                ('img', content_html.find('<img', pos))]
        tags = [(t, i) for t, i in tags if i != -1]
        if not tags:
            break
        tags.sort(key=lambda x: x[1])
        tag, idx = tags[0]

        if tag == 'p':
            p_end = content_html.find('</p>', idx)
            if p_end == -1:
                pos = idx + 2
                continue
            inner = content_html[idx:p_end + 4]
            text = re.sub(r'<[^>]+>', '', inner).strip()
            text = text.replace('&nbsp;', ' ').replace('\u00a0', ' ')
            text = re.sub(r'[ \t]+', ' ', text)
            if text:
                parts.append(text)
            pos = p_end + 4

        elif tag == 'table':
            t_depth = 1
            i = content_html.find('>', idx) + 1
            while i < len(content_html) and t_depth > 0:
                if content_html[i:i+7] == '<table ' or content_html[i:i+7] == '<table>' or content_html[i:i+8] == '<table\t' or content_html[i:i+7] == '<table\n':
                    t_depth += 1
                    i += 7
                elif content_html[i:i+8] == '</table>':
                    t_depth -= 1
                    if t_depth == 0:
                        md_table = table_to_markdown(content_html[idx:i + 8])
                        if md_table:
                            parts.append(md_table)
                        i += 8
                    else:
                        i += 8
                else:
                    i += 1
            pos = i

        elif tag == 'div':
            d_end = content_html.find('>', idx) + 1
            if 'ue_table' in content_html[idx:d_end]:
                t_start = content_html.find('<table', d_end)
                if t_start != -1 and t_start < content_html.find('</div>', d_end):
                    t_depth = 1
                    i = content_html.find('>', t_start) + 1
                    while i < len(content_html) and t_depth > 0:
                        if content_html[i:i+7] == '<table ' or content_html[i:i+7] == '<table>' or content_html[i:i+8] == '<table\t' or content_html[i:i+7] == '<table\n':
                            t_depth += 1
                            i += 7
                        elif content_html[i:i+8] == '</table>':
                            t_depth -= 1
                            if t_depth == 0:
                                md_table = table_to_markdown(content_html[t_start:i + 8])
                                if md_table:
                                    parts.append(md_table)
                                i += 8
                            else:
                                i += 8
                        else:
                            i += 1
                    pos = i
                else:
                    # Skip non-table div
                    dd = 1
                    ci = d_end
                    while ci < len(content_html) and dd > 0:
                        if content_html[ci:ci+4] == '<div' and content_html[ci+4:ci+5] in '> \t\n':
                            dd += 1
                            ci += 4
                        elif content_html[ci:ci+6] == '</div>':
                            dd -= 1
                            if dd == 0:
                                pos = ci + 6
                            ci += 6
                        else:
                            ci += 1
                    if dd != 0:
                        pos = d_end + 1
            else:
                dd = 1
                ci = d_end
                while ci < len(content_html) and dd > 0:
                    if content_html[ci:ci+4] == '<div' and content_html[ci+4:ci+5] in '> \t\n':
                        dd += 1
                        ci += 4
                    elif content_html[ci:ci+6] == '</div>':
                        dd -= 1
                        if dd == 0:
                            pos = ci + 6
                        ci += 6
                    else:
                        ci += 1
                if dd != 0:
                    pos = d_end + 1

        elif tag == 'img':
            src_m = re.search(r'src="([^"]+)"', content_html[idx:idx+200])
            alt_m = re.search(r'alt="([^"]*)"', content_html[idx:idx+200])
            if src_m:
                src = urljoin(page_url, src_m.group(1))
                alt = alt_m.group(1) if alt_m else ''
                parts.append(f'![{alt}]({src})')
            gt = content_html.find('>', idx)
            pos = gt + 1 if gt != -1 else idx + 50

    if attachments:
        parts.append('\n**相关附件：**')
        for name, href in attachments:
            parts.append(f'[{name}]({href})')

    return '\n\n'.join(p.strip() for p in parts if p.strip())

def table_to_markdown(table, *args, **kwargs):
    """保留 HTML 表格结构（不转 md）"""
    return str(table)

def main(max_pages=5):
    # Fetch first page
    html = fetch(BASE_URL)
    if not html:
        print("ERROR: Failed to fetch list page", file=sys.stderr)
        sys.exit(1)

    total_pages = extract_pagination(html)
    pages_to_fetch = min(max_pages, total_pages)
    print(f"Total pages: {total_pages}, fetching: {pages_to_fetch}")

    # Parse all pages
    all_items = []
    all_items.extend(parse_list_page(html, BASE_URL))
    print(f"  Page 0: {len(all_items)} items")

    for page in range(1, pages_to_fetch):
        page_html = fetch(f"{BASE_URL}index_{page}.shtml")
        if not page_html:
            continue
        items = parse_list_page(page_html, f"{BASE_URL}index_{page}.shtml")
        all_items.extend(items)
        print(f"  Page {page}: {len(items)} items")
        time.sleep(1)

    # Deduplicate by URL
    seen = set()
    unique = []
    for t, u, d in all_items:
        if u not in seen:
            seen.add(u)
            unique.append((t, u, d))

    print(f"\nTotal unique items: {len(unique)}")

    # Check existing in DB
    existing = set()
    try:
        res = subprocess.run(['sqlite3', "-cmd", ".timeout 60000", DB_PATH,
            "SELECT page_url FROM gov_raw WHERE page_url LIKE '%rongshui.gov.cn/tzgg/%'"],
            capture_output=True, text=True, timeout=10)
        if res.stdout.strip():
            existing = set(res.stdout.strip().split('\n'))
    except Exception:
        pass

    new_items = [(t, u, d) for t, u, d in unique if u not in existing] if existing else unique
    print(f"New to crawl: {len(new_items)}")

    if not new_items:
        print("Nothing new.")
        return

    # Fetch details and insert
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    inserted = 0
    errors = 0
    fts_sqls = []

    for title, url, list_date in new_items:
        detail_html = fetch(url)
        if not detail_html:
            errors += 1
            continue

        det_title, det_date, content_html, attachments = parse_detail(detail_html, url)
        final_title = (det_title or title).replace('\xa0', ' ').strip()
        final_date = det_date or list_date

        content = render_content(content_html, attachments, url)
        if not content.strip():
            if attachments:
                content = '\n'.join([f'<p><a href="{href}">{name}</a></p>' for name, href in attachments])
            else:
                content = final_title

        summary = content[:200].replace('\n', ' ').strip() if len(content) > 200 else content.replace('\n', ' ')
        source_url = url  # same as page_url for this site
        attachments_str = '; '.join([f'{name}|{href}' for name, href in attachments]) if attachments else ''

        try:
            cur = conn.execute('''
                INSERT INTO gov_raw (title, content, summary, site_name,
                page_url, publish_date, source_url, attachments, category, group_name)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ''', (final_title, content, summary, SITE_NAME,
                  url, final_date, source_url, attachments_str, '通知公告', GROUP))
            if cur.rowcount > 0:
                inserted += 1
                rid = cur.lastrowid
                fts_sqls.append(
                    "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) "
                    f"VALUES ({rid},'{final_title.replace(chr(39),chr(39)+chr(39))}','{SITE_NAME.replace(chr(39),chr(39)+chr(39))}','{summary.replace(chr(39),chr(39)+chr(39))}');"
                )
                print(f"  [{inserted}] {final_title[:40]}...")
        except Exception as e:
            print(f"  ERROR: {e}", file=sys.stderr)
            errors += 1

        time.sleep(0.3)

    conn.commit()

    # FTS sync
    if fts_sqls:
        try:
            fts_sqls.append("SELECT 1 /* noop: gov_search 由触发器维护, 无需 rebuild */;")
            subprocess.run(['sqlite3', "-cmd", ".timeout 60000", DB_PATH],
                input='\n'.join(fts_sqls), capture_output=True, text=True, timeout=30)
            print(f"FTS: {len(fts_sqls)-1} rows + rebuild")
        except Exception as e:
            print(f"FTS error: {e}", file=sys.stderr)

    conn.close()
    print(f"\nDone. Inserted: {inserted}, Errors: {errors}")


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='融水苗族自治县通知公告爬虫')
    parser.add_argument('--max-pages', type=int, default=5, help='最大爬取页数')
    args = parser.parse_args()
    main(max_pages=args.max_pages)