#!/usr/bin/env python3
import sys, os, re, time, sqlite3
sys.path.insert(0, '/root/gov_crawler')

SITE_NAME = "东乡区-环境保护"
LIST_URL = "https://www.jxfzdx.gov.cn/col/col21311/index.html?number=L00004L00004L00010"

from playwright.sync_api import sync_playwright
import requests
from bs4 import BeautifulSoup

ENV_PATTERNS = [
    r'环[境保评]', r'环保', r'环境影响评价', r'环评',
    r'审批[决公]', r'拟批准', r'拟受理',
    r'土壤污染', r'污染',
    r'水质[状报]', r'饮用水源',
    r'环境质量', r'环境[监检]测',
    r'排污[许证]', r'应急预案',
    r'辐射安[全装]',
    r'噪声', r'废气', r'废水', r'废[气弃]', r'固[体废]',
    r'危险废物', r'危废',
    r'生态[环修保]',
    r'绿色', r'低碳', r'碳中和',
    r'林[业地木]', r'林木采伐', r'林地[公使]',
    r'建设[项工]', r'项目[公拟环]',
    r'新能源', r'光伏',
    r'污水[处治]', r'垃圾[处分]', r'整治',
    r'公示', r'公告',
    r'报告[书表]', r'征求意见', r'公众参与',
    r'备案[通确]', r'批复', r'决定书',
    r'有限公司', r'责任公司', r'合作社',
    r'生产线', r'技改', r'扩建', r'新建',
    r'年产', r'年出栏', r'养殖', r'养猪',
    r'调查[报评]', r'风险评估',
]
SKIP_TITLES = [
    r'^.{0,8}(局|中心|委|办|会|镇|乡|场|街道)$',
    r'公开指南$', r'公开年报', r'公开制度',
    r'政府信息公开', r'政府网站工作',
    r'网站年度', r'年度工作报表',
    r'信息公[开]', r'主动公开',
    r'东乡区人民政府网站',
    r'东乡区人民政府信息公开',
    r'政策解读', r'政策文件解读',
    r'解读工作',
]

def is_environmental(title):
    if not title or len(title) < 8: return False
    for p in SKIP_TITLES:
        if re.search(p, title): return False
    title_clean = title.replace(' ', '').replace('\u200b', '')
    for pat in ENV_PATTERNS:
        if re.search(pat, title_clean): return True
    return False

def scrape(pages_limit):
    all_items = []
    seen_urls = set()
    
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(
            user_agent="Mozilla/5.0",
            viewport={"width": 1920, "height": 1080}
        )
        page = context.new_page()
        page.goto(LIST_URL, wait_until="domcontentloaded", timeout=30000)
        time.sleep(5)
        
        for pg in range(1, pages_limit + 1):
            print(f"\n=== Page {pg} ===")
            
            all_links = page.evaluate("""
                () => {
                    const results = [];
                    const seen = new Set();
                    document.querySelectorAll('a[href*="art_"]').forEach(a => {
                        const text = a.textContent.trim();
                        if (!text || text.length < 5) return;
                        if (seen.has(a.href)) return;
                        seen.add(a.href);
                        const li = a.closest('li');
                        let date = '';
                        if (li) { const b = li.querySelector('b'); if (b) date = b.textContent.trim(); }
                        results.push({title: text, href: a.href, date: date});
                    });
                    return results;
                }
            """)
            
            env_items = [it for it in all_links if is_environmental(it['title'])]
            print(f"  Total: {len(all_links)}, Environmental: {len(env_items)}")
            
            for item in env_items:
                if item['href'] not in seen_urls:
                    seen_urls.add(item['href'])
                    all_items.append(item)
                    print(f"    + {item['title'][:50]}")
            
            if pg < pages_limit:
                np = pg + 1
                result = page.evaluate(f"""
                    () => {{
                        if (typeof funGoPage === 'function') {{
                            funGoPage('/module/xxgk/search.jsp', {np});
                            return 'called funGoPage page=' + {np};
                        }}
                        return 'funGoPage not found';
                    }}
                """)
                print(f"  Navigate: {result}")
                time.sleep(4)
                page.wait_for_load_state("domcontentloaded", timeout=30000)
                time.sleep(2)
        
        browser.close()
    
    return all_items

def get_detail(url):
    try:
        r = requests.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0"})
        r.encoding = 'utf-8'
    except:
        return '', ''
    soup = BeautifulSoup(r.text, 'html.parser')
    page_title = ''
    t = soup.find('title')
    if t:
        page_title = t.get_text(strip=True)
        page_title = re.sub(r'^[^-—]*?(?:——|-)\s*', '', page_title)
    zoom = soup.find('div', id='zoom')
    if not zoom:
        zoom = soup.find('div', class_=re.compile(r'content|article|nrzw|zoom|main-word', re.I))
    if not zoom:
        return page_title, ''
    parts = []
    for child in zoom.children:
        if child.name == 'p':
            text = child.get_text(' ', strip=True)
            if text: parts.append(text)
        elif child.name == 'table':
            md = html_table_to_html(str(child))
            if md: parts.append(md)
        elif child.name in ('div', 'section', 'article'):
            _extract_nodes(child, parts)
    if not parts:
        text = zoom.get_text('\n', strip=True)
        if text: parts = [text]
    body = '\n\n'.join(parts)
    body = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', body)
    return page_title, body

def _extract_nodes(node, parts):
    for child in node.children:
        if child.name == 'p':
            text = child.get_text(' ', strip=True)
            if text: parts.append(text)
        elif child.name == 'table':
            md = html_table_to_html(str(child))
            if md: parts.append(md)
        elif child.name in ('div', 'section', 'article'):
            _extract_nodes(child, parts)

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def push_to_db(items):
    """Insert into DB, replacing any existing rows with same page_url"""
    db = sqlite3.connect('/root/search.db')
    
    for item in items:
        try:
            # First delete any existing row with this page_url
            db.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE page_url=?)", (item['url'],))
            db.execute("DELETE FROM gov_raw WHERE page_url=?", (item['url'],))
            
            # Then insert new row
            db.execute(
                "INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags, attachments) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                (item['site_name'], item['source_url'], item['url'],
                 item['title'][:500], item.get('pub_date',''), (item.get('summary','') or '')[:500],
                 item.get('content',''), 'active', '', '', '')
            )
        except Exception as e:
            print(f"  ERROR on {item['title'][:30]}: {e}")
    
    db.commit()
    db.close()

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--pages', type=int, default=5)
    args = parser.parse_args()
    print(f"[{SITE_NAME}] 开始（最多{args.pages}页）")
    items = scrape(args.pages)
    if not items:
        print("  ⚠️ 无结果")
        return
    print(f"\n[{SITE_NAME}] 总计: {len(items)} 条")
    db_items = []
    success = failed = 0
    for i, item in enumerate(items):
        url = item['href']
        title = item['title']
        date_str = item.get('date', '')
        print(f"  [{i+1}] {title[:40]}...", end=' ', flush=True)
        page_title, body = get_detail(url)
        if not body:
            print("⚠️ 空")
            failed += 1
        else:
            print(f"✅ {len(body)}字")
            success += 1
        db_items.append({
            'site_name': SITE_NAME, 'source_url': url, 'url': url,
            'title': page_title or title, 'pub_date': date_str,
            'summary': body[:500] if body else '', 'content': body,
        })
    
    # Push: delete-then-insert (replaces any existing rows)
    push_to_db(db_items)
    
    # Verify
    vdb = sqlite3.connect('/root/search.db')
    cnt = vdb.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchone()[0]
    cnt2 = vdb.execute("SELECT COUNT(*) FROM gov_search WHERE site_name=?", (SITE_NAME,)).fetchone()[0]
    vdb.close()
    print(f"\n[{SITE_NAME}] 完成: {len(db_items)}条, 正文成功{success}, 空{failed}")
    print(f"  DB: raw={cnt}, search={cnt2}")

if __name__ == '__main__':
    main()