#!/usr/bin/env python3
"""
利辛县生态环境分局 - 建设项目环评文件（报告书、报告表）审批
列表: POST XxgkTarget/showTree (columnId=109934, page=N)
详情: /XxgkContent/show/{id}.html → div#zoom
"""
import sys, re, json, time, requests, sqlite3
from bs4 import BeautifulSoup
import urllib3
urllib3.disable_warnings()

DB_PATH = "/root/search.db"
SITE_NAME = "利辛县生态环境分局-环评审批"
CATEGORY = "环评公示"
GROUP = "安徽"
BASE = "https://www.lixin.gov.cn"
COLUMN_ID = "109934"
BRANCH_ID = "490"
MAX_PAGES = 50  # ~10 per page

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def init_db():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn


def save_article(conn, title, page_url, publish_date, content):
    summary = content[:200] if content else title
    summary = re.sub(r"\s+", " ", summary).strip()
    try:
        cur = conn.execute(
            """INSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date, summary, content, category, group_name)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (SITE_NAME, page_url, page_url, title.strip(), publish_date,
             summary, content, CATEGORY, GROUP),
        )
        return cur.rowcount > 0
    except Exception as e:
        print(f"  DB error: {e}", file=sys.stderr)
        return False


def fetch_detail(url):
    """Get title, date, content from detail page."""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = 'utf-8'
    except Exception:
        return None, None, None
    if r.status_code not in (200, 201):
        return None, None, None
    # Follow meta refresh redirect (OpennessContent → XxgkContent)
    if 'http-equiv=\"refresh\"' in r.text:
        m = re.search(r'url=([^\"]+)', r.text, re.I)
        if m:
            redirect_url = m.group(1)
            if redirect_url.startswith('/'):
                redirect_url = BASE + redirect_url
            try:
                r = requests.get(redirect_url, headers=HEADERS, timeout=30, verify=False)
                r.encoding = 'utf-8'
            except Exception:
                pass

    soup = BeautifulSoup(r.text, 'html.parser')

    # Title
    title = None
    meta = soup.find('meta', {'name': 'ArticleTitle'})
    if meta and meta.get('content'):
        title = meta['content'].strip()
    if not title:
        title_tag = soup.find('title')
        if title_tag:
            parts = title_tag.get_text(strip=True).split('-')
            if parts and parts[0].strip():
                title = parts[0].strip()

    # Date
    date_str = ''
    meta_date = soup.find('meta', {'name': 'PubDate'})
    if meta_date and meta_date.get('content'):
        m = re.search(r'(\d{4}-\d{2}-\d{2})', meta_date['content'])
        if m:
            date_str = m.group(1)

    # Content from div#zoom
    content = None
    zoom = soup.find('div', id='zoom')
    if zoom:
        parts = []
        for elem in zoom.find_all(['p', 'table'], recursive=True):
            if elem.name == 'p':
                if elem.find_parent('table'):
                    continue
                text = elem.get_text('', strip=True)
                if text:
                    parts.append(text)
            elif elem.name == 'table':
                parts.append(str(elem))
        c = '\n\n'.join(parts) if parts else body_text(zoom)
        if len(c.strip()) >= 20:
            content = c

    if not content:
        content = f'<p><a href="{url}">{title}</a></p>'

    return title, date_str, content


def main():
    max_pages = MAX_PAGES
    if len(sys.argv) > 1:
        try:
            max_pages = int(sys.argv[1])
        except ValueError:
            if sys.argv[1] in ('--incremental',):
                max_pages = 1

    print(f"[{SITE_NAME}] max pages: {max_pages}", file=sys.stderr)
    conn = init_db()
    total_new = 0

    # Clean old data
    conn.execute("DELETE FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    conn.commit()
    print(f"  Cleaned old data", file=sys.stderr)

    session = requests.Session()
    session.headers.update(HEADERS)

    for page in range(1, max_pages + 1):
        print(f"[LIST] Page {page}", file=sys.stderr)

        try:
            r = session.post(f"{BASE}/XxgkTarget/showTree", data={
                'columnId': COLUMN_ID,
                'contentDivId': 'opennessContents_109934',
                'branchId': BRANCH_ID,
                'keyword': '',
                'page': str(page),
            }, timeout=30)
            if r.status_code != 200:
                print(f"  HTTP {r.status_code}", file=sys.stderr)
                break
        except Exception as e:
            print(f"  Error: {e}", file=sys.stderr)
            break

        soup = BeautifulSoup(r.text, 'html.parser')
        table = soup.find('table')
        if not table:
            print(f"  No table, stopping", file=sys.stderr)
            break

        rows = table.find_all('tr')
        if not rows:
            print(f"  No rows, stopping", file=sys.stderr)
            break

        print(f"  {len(rows)} articles", file=sys.stderr)

        for row in rows:
            tds = row.find_all('td')
            if len(tds) < 3:
                continue

            serial = tds[0].get_text(strip=True)
            a_tag = tds[1].find('a', href=True)
            if not a_tag:
                continue

            title = a_tag.get_text(strip=True)
            href = a_tag['href']
            if not href.startswith('http'):
                href = BASE + href
            date_str = tds[2].get_text(strip=True)

            print(f"  #{serial} [{date_str}] {title[:40]}...", file=sys.stderr)

            # Check if exists
            cur = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (href,))
            if cur.fetchone():
                continue

            det_title, det_date, det_content = fetch_detail(href)
            final_title = det_title or title
            final_date = det_date or date_str
            final_content = det_content or f"[{final_title}]({href})"

            if save_article(conn, final_title, href, final_date, final_content):
                total_new += 1
                if total_new <= 5:
                    print(f"    + {final_date} {final_title[:50]}", file=sys.stderr)

            time.sleep(0.5)

        conn.commit()
        time.sleep(1)

    conn.commit()
    conn.close()
    print(f"\nDone: {total_new} new articles inserted", file=sys.stderr)
    print(total_new)


if __name__ == '__main__':
    main()