#!/usr/bin/env python3
"""ynnh.gov.cn 南华县人民政府 - 通知公告 爬虫"""
import requests, re, os, sys, json, time, urllib3
from bs4 import BeautifulSoup
import urllib.parse

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "南华县人民政府-通知公告"
BASE_URL = "https://www.ynnh.gov.cn"
LIST_URL = BASE_URL + "/zfxxgk/fdzdgknr/tzgg.htm"
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)

def log(msg):
    print(msg, flush=True)

def get_soup(url, timeout=20):
    r = requests.get(url, headers=HEADERS, verify=False, timeout=timeout)
    r.encoding = 'utf-8'
    return BeautifulSoup(r.text, 'html.parser')

def extract_date(text):
    """从 '2026年07月07日' 格式提取 YYYY-MM-DD"""
    m = re.search(r'(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日', text)
    if m:
        return f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    m = re.search(r'(\d{4})-(\d{1,2})-(\d{1,2})', text)
    if m:
        return f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    return ""

def extract_list_items(soup):
    """从列表页提取 (title, url, date)"""
    items = []
    # 先找scroll_main容器，再找ul
    scroll = soup.find('div', class_=re.compile(r'scroll_main'))
    if not scroll:
        scroll = soup
    ul = scroll.find('ul')
    if not ul:
        log("  [WARN] 未找到<ul>列表容器")
        return items
    for li in ul.find_all('li', id=re.compile(r'line_u6')):
        a = li.find('a')
        if not a:
            continue
        # 优先取 title 属性（完整标题），fallback到显示文本
        title = a.get('title', '').strip() or a.get_text(strip=True)
        href = a.get('href', '')
        if not href or not title:
            continue
        # 补全URL
        if href.startswith('../../'):
            href = BASE_URL + href[5:]
        elif href.startswith('../'):
            href = BASE_URL + href[2:]
        elif href.startswith('/'):
            href = BASE_URL + href
        elif not href.startswith('http'):
            href = BASE_URL + '/' + href.lstrip('/')
        
        # 日期
        span = li.find('span')
        date_str = extract_date(span.get_text() if span else '')
        
        items.append((title.strip(), href, date_str))
    return items

def get_pagination_urls():
    """生成前5页的URL列表"""
    urls = [LIST_URL]
    for page in range(2, MAX_PAGES + 1):
        # 分页规律: tzgg/(9-N).htm
        file_num = 9 - page
        urls.append(f"{BASE_URL}/zfxxgk/fdzdgknr/tzgg/{file_num}.htm")
    return urls

def fetch_detail(url):
    """抓取详情页：返回 (content_text, attachments_list)"""
    try:
        soup = get_soup(url, timeout=25)
    except Exception as e:
        log(f"  [WARN] 详情页请求失败: {url} -> {e}")
        return "", []
    
    # 正文
    content_div = soup.find('div', class_='v_news_content')
    if not content_div:
        log(f"  [WARN] 未找到内容: {url}")
        return "", []
    
    # 提取段落+表格
    parts = []
    for tag in content_div.find_all(['p', 'table', 'div', 'span', 'img', 'br'], recursive=True):
        if tag.name == 'p':
            text = tag.get_text(strip=True)
            if text:
                parts.append(text)
        if tag.name == 'table':
            parts.append(html_table_to_html(tag, url))
        elif tag.name == 'img':
            src = tag.get('src', '')
            alt = tag.get('alt', '')
            if src:
                if src.startswith('/'):
                    parts.append(f'<p><a href="{BASE_URL}{src}">查看图片</a></p>')
                elif src.startswith('http'):
                    parts.append(f'<p><a href="{src}">查看图片</a></p>')
                else:
                    parts.append(f'<p><a href="{BASE_URL}/{src.lstrip("/")}">查看图片</a></p>')
        elif tag.name == 'br':
            parts.append('')
    
    # 如果没有通过标签提取到内容，fallback到纯文本
    text_content = body_text(content_div)
    if not parts:
        parts = [text_content]
    
    content = '\n\n'.join(parts)
    
    # 附件
    attachments = []
    for a in content_div.find_all('a'):
        ahref = a.get('href', '')
        if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', ahref.lower()):
            aname = a.get_text(strip=True) or ahref.split('/')[-1]
            if ahref.startswith('/'):
                ahref = BASE_URL + ahref
            elif ahref.startswith('..'):
                ahref = BASE_URL + re.sub(r'^\.\./+', '/', ahref)
            elif not ahref.startswith('http'):
                ahref = BASE_URL + '/' + ahref.lstrip('/')
            attachments.append({'name': aname, 'url': ahref})
    
    # 如果正文为空但有附件，fallback
    if len(content.strip()) < 20 and attachments:
        content = ""
    
    return content, attachments

def crawl():
    import sqlite3
    
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute('PRAGMA busy_timeout=5000')
    
    list_urls = get_pagination_urls()
    
    all_items = []
    for pi, purl in enumerate(list_urls):
        log(f"\n列表页 {pi+1}/{len(list_urls)}: {purl}")
        try:
            soup = get_soup(purl, timeout=20)
            items = extract_list_items(soup)
            log(f"  找到 {len(items)} 条")
            all_items.extend(items)
        except Exception as e:
            log(f"  [ERROR] 列表页失败: {e}")
    
    log(f"\n共获取 {len(all_items)} 条文章链接")
    
    # 去重
    seen_urls = set()
    unique_items = []
    for title, url, date in all_items:
        if url not in seen_urls:
            seen_urls.add(url)
            unique_items.append((title, url, date))
    log(f"去重后: {len(unique_items)} 条")
    
    # 检查已有URL
    existing = set()
    try:
        for row in conn.execute("SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)):
            existing.add(row[0])
    except:
        pass
    
    new_count = 0
    skip_count = 0
    error_count = 0
    
    for idx, (title, url, pub_date) in enumerate(unique_items):
        if url in existing:
            skip_count += 1
            continue
        
        log(f"\n[{idx+1}/{len(unique_items)}] {title[:50]}...")
        
        # 抓详情
        content, attachments = fetch_detail(url)
        
        # 正文为空回退
        if not content or len(content.strip()) < 20:
            content = f'<p><a href="{url}">{title}</a></p>'
            if attachments:
                attach_links = '\n'.join(f"- [{a['name']}]({a['url']})" for a in attachments)
                content += '\n\n' + attach_links
        
        # 确保日期
        if not pub_date:
            # 从详情页提取
            try:
                soup = get_soup(url, timeout=15)
                body_text = soup.get_text()
                pub_date = extract_date(body_text)
            except:
                pass
        
        summary = content[:200] if content else title
        
        try:
            retries = 3
            for attempt in range(retries):
                try:
                    conn.execute("""
                        INSERT OR IGNORE INTO gov_raw(title, content, publish_date, source_url, page_url, site_name, summary)
                        VALUES (?, ?, ?, ?, ?, ?, ?)
                    """, (title, content, pub_date, url, url, SITE_NAME, summary))
                    
                    if conn.total_changes > 0:
                        new_count += 1
                        log(f"  ✅ 新增: {title[:40]} | date={pub_date}")
                        if attachments:
                            attach_text = json.dumps(attachments, ensure_ascii=False)
                            conn.execute("UPDATE gov_raw SET attachments=? WHERE page_url=?", (attach_text, url))
                    else:
                        skip_count += 1
                        log(f"  ⏭️ 已存在 (INSERT OR IGNORE)")
                    
                    conn.commit()
                    break  # success, exit retry loop
                except Exception as e:
                    if 'database is locked' in str(e) and attempt < retries - 1:
                        log(f"  ⏳ DB锁住, 等待重试 ({attempt+1}/{retries})...")
                        time.sleep(2)
                        conn.rollback()
                    else:
                        raise e
        except Exception as e:
            log(f"  ❌ 入库失败: {e}")
            error_count += 1
            conn.rollback()
        
        # 间隔，避免触发反爬
        time.sleep(0.5)
    
    conn.close()
    
    log(f"\n{'='*50}")
    log(f"  完成!")
    log(f"  新增: {new_count}")
    log(f"  跳过: {skip_count}")
    log(f"  错误: {error_count}")
    log(f"{'='*50}")

if __name__ == '__main__':
    crawl()