#!/usr/bin/env python3
"""yjgx.gov.cn 阳江滨海新区（阳江高新区）- 环境保护信息公开 爬虫"""
import requests, re, os, sys, json, time, urllib3
from bs4 import BeautifulSoup
import urllib.parse

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "阳江滨海新区（阳江高新区）-环境保护信息公开"
BASE_URL = "http://www.yjgx.gov.cn"
LIST_URL = BASE_URL + "/zwgk/zdlyxxgk/hjbhxxgk/index.html"
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)

def log(msg):
    print(msg, flush=True)

def get_soup(url, timeout=20):
    r = requests.get(url, headers=HEADERS, verify=False, timeout=timeout)
    r.encoding = 'utf-8'
    return BeautifulSoup(r.text, 'html.parser')

def extract_date(text):
    m = re.search(r'(\d{4})-(\d{1,2})-(\d{1,2})', text)
    if m:
        return f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    m = re.search(r'(\d{4})年(\d{1,2})月(\d{1,2})日', text)
    if m:
        return f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    return ""

def extract_list_items(soup):
    """从列表页提取 (title, url, date)"""
    items = []
    # xwlist -> ul[1] (第二个ul)
    xwlist = soup.find('div', class_='xwlist')
    if not xwlist:
        log("  [WARN] 未找到 div.xwlist")
        return items
    uls = xwlist.find_all('ul')
    if len(uls) < 2:
        log("  [WARN] xwlist 中没有第二个ul")
        return items
    ul = uls[1]  # 第二个ul是文章列表
    for li in ul.find_all('li'):
        a = li.find('a')
        if not a:
            continue
        href = a.get('href', '')
        title = a.get_text(strip=True)
        if not href or not title:
            continue
        # 补全URL
        if href.startswith('/'):
            href = BASE_URL + href
        elif not href.startswith('http'):
            href = BASE_URL + '/' + href.lstrip('/')
        # 日期
        span = li.find('span', class_=re.compile(r'date', re.I))
        date_str = extract_date(span.get_text() if span else '')
        items.append((title, href, date_str))
    return items

def get_pagination_urls():
    """生成前5页的URL"""
    urls = [LIST_URL]  # index.html = page 1
    for p in range(2, MAX_PAGES + 1):
        urls.append(BASE_URL + f"/zwgk/zdlyxxgk/hjbhxxgk/index_{p}.html")
    return urls

def fetch_detail(url):
    """抓取详情页：返回 (content_text, attachments_list, pub_date)"""
    try:
        soup = get_soup(url, timeout=25)
    except Exception as e:
        log(f"  [WARN] 详情页请求失败: {url} -> {e}")
        return "", [], ""

    # 正文 - 优先 div.Content, 再 div.content_nr
    content_div = soup.find('div', class_='Content')
    if not content_div:
        content_div = soup.find('div', class_='content_nr')
    if not content_div:
        log(f"  [WARN] 未找到内容容器: {url}")
        return "", [], ""

    # 提取段落+表格
    parts = []
    for tag in content_div.find_all(['p', 'table', 'div', 'img'], recursive=True):
        if tag.name == 'p':
            text = tag.get_text(strip=True)
            if text:
                parts.append(text)
        if tag.name == 'table':
            parts.append(html_table_to_html(tag, url))
        elif tag.name == 'img':
            src = tag.get('src', '')
            alt = tag.get('alt', '')
            if src:
                if src.startswith('http'):
                    parts.append(f"![{alt}]({src})")
                elif src.startswith('/'):
                    parts.append(f"![{alt}]({BASE_URL}{src})")

    content = '\n\n'.join(parts)
    if not content:
        content = body_text(content_div)

    # 附件
    attachments = []
    for a in content_div.find_all('a', href=True):
        ahref = a['href']
        if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', ahref.lower()):
            aname = a.get_text(strip=True) or ahref.split('/')[-1]
            if ahref.startswith('/'):
                ahref = BASE_URL + ahref
            elif not ahref.startswith('http'):
                ahref = BASE_URL + '/' + ahref.lstrip('/')
            attachments.append({'name': aname, 'url': ahref})

    # 日期（从详情页提取）
    body_text = soup.get_text()
    pub_date = extract_date(body_text)
    m = re.search(r'发布日期[：:]\s*(\d{4}-\d{1,2}-\d{1,2})', body_text)
    if m:
        pub_date = m.group(1)

    if not content or len(content.strip()) < 20:
        _t = ''
        _h = soup.find('h1')
        if _h:
            _t = _h.get_text(strip=True)
        if not _t and soup.title and soup.title.string:
            _t = soup.title.string.strip()
        content = f'<p><a href="{url}">{_t or url}</a></p>'

    return content, attachments, pub_date


def crawl():
    import sqlite3

    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute('PRAGMA busy_timeout=5000')

    list_urls = get_pagination_urls()

    all_items = []
    for pi, purl in enumerate(list_urls):
        log(f"\n列表页 {pi+1}/{len(list_urls)}: {purl}")
        try:
            soup = get_soup(purl, timeout=20)
            items = extract_list_items(soup)
            log(f"  找到 {len(items)} 条")
            all_items.extend(items)
        except Exception as e:
            log(f"  [ERROR] 列表页失败: {e}")

    log(f"\n共获取 {len(all_items)} 条文章链接")

    # 去重
    seen_urls = set()
    unique_items = []
    for title, url, date in all_items:
        if url not in seen_urls:
            seen_urls.add(url)
            unique_items.append((title, url, date))
    log(f"去重后: {len(unique_items)} 条")

    # 检查已有URL
    existing = set()
    try:
        for row in conn.execute("SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)):
            existing.add(row[0])
    except:
        pass

    new_count = 0
    skip_count = 0
    error_count = 0

    for idx, (title, url, pub_date) in enumerate(unique_items):
        if url in existing:
            skip_count += 1
            continue

        log(f"\n[{idx+1}/{len(unique_items)}] {title[:50]}...")

        content, attachments, detail_date = fetch_detail(url)
        if detail_date and not pub_date:
            pub_date = detail_date

        # 正文为空回退
        if not content or len(content.strip()) < 20:
            content = f'<p><a href="{url}">{title}</a></p>'
            if attachments:
                attach_links = '\n'.join(f"- [{a['name']}]({a['url']})" for a in attachments)
                content += '\n\n' + attach_links

        summary = content[:200] if content else title

        retries = 3
        for attempt in range(retries):
            try:
                conn.execute("""
                    INSERT OR IGNORE INTO gov_raw(title, content, publish_date, source_url, page_url, site_name, summary)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                """, (title, content, pub_date, url, url, SITE_NAME, summary))

                if conn.total_changes > 0:
                    new_count += 1
                    log(f"  ✅ 新增: {title[:40]} | date={pub_date}")
                    if attachments:
                        conn.execute("UPDATE gov_raw SET attachments=? WHERE page_url=?", (json.dumps(attachments, ensure_ascii=False), url))
                else:
                    skip_count += 1
                    log(f"  ⏭️ 已存在")
                conn.commit()
                break
            except Exception as e:
                if 'database is locked' in str(e) and attempt < retries - 1:
                    log(f"  ⏳ DB锁住, 等待 ({attempt+1}/{retries})...")
                    time.sleep(2)
                    conn.rollback()
                else:
                    raise e

        time.sleep(0.5)

    conn.close()
    log(f"\n{'='*50}")
    log(f"  完成! 新增: {new_count}  跳过: {skip_count}  错误: {error_count}")
    log(f"{'='*50}")


if __name__ == '__main__':
    pages_arg = 5
    if len(sys.argv) > 1:
        for i, arg in enumerate(sys.argv):
            if arg in ('--pages', '-p', '--max-pages'):
                if i + 1 < len(sys.argv):
                    try:
                        pages_arg = int(sys.argv[i+1])
                    except:
                        pass
                    break
            else:
                try:
                    pages_arg = int(arg)
                except:
                    pass
    pages_arg = max(1, min(pages_arg, 20))
    MAX_PAGES = pages_arg
    crawl()