#!/usr/bin/env python3
"""
阜新蒙古族自治县人民政府 - 通知通告 爬虫
CMS: 自定义政府CMS
栏目: /channel/list/12821.html - 通知通告 (12条/页)
仅全量前5页，日跑仅第1页
"""
import requests
import re
import json
import time
import sys
import os
from bs4 import BeautifulSoup

BASE_URL = "https://www.fmx.gov.cn"
LIST_URL = BASE_URL + "/channel/list/12821.html"
SITE_NAME = "阜新蒙古族自治县-通知通告"
GROUP = "企业"
DB_PATH = os.environ.get('DB_PATH', '/root/search.db')

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}

def fetch(url):
    r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
    r.encoding = 'utf-8'
    return r.text

def parse_list(html):
    """解析列表页 - 通知通告栏目 div.new_lists > ul.new_content_ul"""
    soup = BeautifulSoup(html, "html.parser")
    items = []
    
    new_lists = soup.find("div", class_="new_lists")
    if new_lists:
        ul = new_lists.find("ul", class_="new_content_ul")
        if ul:
            for li in ul.find_all("li"):
                a = li.find("a", href=True)
                if a and a.get("href"):
                    url = a["href"]
                    title = a.get("title", "") or a.get_text(strip=True)
                    if not title:
                        continue
                    if not url.startswith("http"):
                        url = BASE_URL + url
                    span = li.find("span")
                    date = span.get_text(strip=True) if span else ""
                    if date and re.match(r"^\d{2}-\d{2}-\d{2}$", date):
                        date = "20" + date
                    items.append((title, url, date))
    
    return items

def parse_detail(html, url):
    """解析详情页"""
    title = ''
    m = re.search(r'<meta[^>]*ArticleTitle[^>]*content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()
    if not title:
        m = re.search(r'<h2[^>]*class="DetailcontentTile2"[^>]*>(.*?)</h2>', html, re.DOTALL)
        if m:
            title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    
    date = ''
    m = re.search(r'日期[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', html)
    if m:
        date = m.group(1).strip().replace('/', '-')
    
    content = ''
    attachments = []
    
    m = re.search(r'<div[^>]*id="Detailcontent"[^>]*class="Detailcontent_inner[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        raw = m.group(1)
        
        # Extract attachment links
        for a in re.finditer(r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>', raw):
            href = a.group(1)
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', href, re.I):
                name = re.sub(r'<[^>]+>', '', a.group(2)).strip() or href.split('/')[-1]
                if href.startswith('/'):
                    href = BASE_URL + href
                elif not href.startswith('http'):
                    href = BASE_URL + '/' + href.lstrip('/')
                attachments.append({"url": href, "name": name})
        
        # Convert tables to readable format
        for table in re.finditer(r'<table[^>]*>(.*?)</table>', raw, re.DOTALL):
            table_html = table.group(0)
            rows = re.findall(r'<tr[^>]*>(.*?)</tr>', table_html, re.DOTALL)
            table_lines = []
            for row in rows:
                cells = re.findall(r'<t[dh][^>]*>(.*?)</t[dh]>', row, re.DOTALL)
                cell_texts = []
                for c in cells:
                    ct = re.sub(r'<[^>]+>', '', c).strip()
                    cell_texts.append(ct)
                table_lines.append(' | '.join(cell_texts))
            raw = raw.replace(table_html, '\n' + '\n'.join(table_lines) + '\n')
        
        raw = re.sub(r'<script[^>]*>.*?</script>', '', raw, flags=re.DOTALL)
        raw = re.sub(r'<style[^>]*>.*?</style>', '', raw, flags=re.DOTALL)
        raw = raw.replace('<br />', '\n').replace('<br/>', '\n').replace('<br>', '\n')
        raw = raw.replace('</p>', '\n\n')
        raw = raw.replace('</div>', '\n')
        
        raw = re.sub(r'<[^>]+>', '', raw)
        raw = raw.replace('&nbsp;', ' ').replace('&amp;', '&')
        raw = raw.replace('&lt;', '<').replace('&gt;', '>')
        
        raw = re.sub(r'[ \t]+', ' ', raw)
        raw = re.sub(r'\n{3,}', '\n\n', raw).strip()
        
        if raw:
            content = raw
    
    return title, date, content, attachments

def crawl_all(max_pages=5):
    """全量爬取（默认前5页）"""
    all_items = []
    
    for page in range(1, max_pages + 1):
        if page == 1:
            url = LIST_URL
        else:
            url = "%s/channel/list/12821_%d.html" % (BASE_URL, page)
        
        print("Fetching page %d: %s" % (page, url))
        try:
            html = fetch(url)
            items = parse_list(html)
            print("  -> %d items" % len(items))
            all_items.extend(items)
            time.sleep(0.3)
        except Exception as e:
            print("  ERROR: %s" % e)
            break
    
    seen = set()
    unique = []
    for item in all_items:
        item_url = item[1]
        if item_url not in seen:
            seen.add(item_url)
            unique.append(item)
    
    print("\nTotal unique items: %d" % len(unique))
    return unique

def crawl_incremental():
    """增量爬取（仅第1页）"""
    print("Fetching page 1 (incremental): %s" % LIST_URL)
    html = fetch(LIST_URL)
    items = parse_list(html)
    print("  -> %d items" % len(items))
    return items

def save_to_db(items):
    """保存到DB"""
    import sqlite3
    
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute('PRAGMA busy_timeout=30000')
    c = conn.cursor()
    
    new_count = 0
    for i, (title, item_url, date) in enumerate(items):
        for attempt in range(3):
            try:
                detail_html = fetch(item_url)
                t, d, content, attachments = parse_detail(detail_html, item_url)
                final_title = t or title
                final_date = d or date
                summary = content[:200] if content else final_title
                attrs_json = json.dumps(attachments, ensure_ascii=False) if attachments else '[]'
                
                c.execute('''INSERT OR IGNORE INTO gov_raw 
                    (site_name, source_url, page_url, title, publish_date,
                     content, summary, category, status, attachments)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'published', ?)''',
                    (SITE_NAME, '', item_url,
                     final_title, final_date, content or '', summary, GROUP, attrs_json))
                if c.rowcount > 0:
                    new_count += 1
                    if new_count <= 5 or (i+1) % 10 == 0:
                        print("  [%d/%d] %s +%d" % (i+1, len(items), final_title[:40], new_count))
                break
            except Exception as e:
                if attempt < 2:
                    time.sleep(3)
                    continue
                print("  ERROR [%s]: %s" % (item_url, e))
                break
        
        time.sleep(0.3)
    
    conn.commit()
    conn.close()
    print("\nInserted %d new records (trigger auto-syncs FTS)" % new_count)
    return new_count

if __name__ == '__main__':
    mode = sys.argv[1] if len(sys.argv) > 1 else 'full'
    
    if mode == 'incremental':
        print("=== 阜新蒙古族自治县-通知通告 - 增量爬取 ===")
        items = crawl_incremental()
        save_to_db(items)
    else:
        print("=== 阜新蒙古族自治县-通知通告 - 全量爬取 ===")
        items = crawl_all(max_pages=5)
        save_to_db(items)