#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""crawl_rongcheng_eia.py — 荣成市人民政府-建设项目环境影响评价信息"""

import os, re, sqlite3, time, sys, random, urllib.parse
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = os.environ.get('SEARCH_DB', '/root/search.db')
SITE_NAME = '荣成市人民政府-建设项目环境影响评价信息'
LIST_URL = 'http://www.rongcheng.gov.cn/module/xxgk/search.jsp?infotypeId=RCA241904&vc_title=&vc_number=&area='
PAGE_SIZE = 20
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')
MAX_WORKERS = 5

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
}

import requests
sess = requests.Session()
sess.headers.update(HEADERS)


def fetch_list(page):
    """Fetch one page via POST"""
    data = {
        'infotypeId': 'RCA241904',
        'jdid': '121',
        'area': '',
        'divid': 'div56364',
        'vc_title': '',
        'vc_number': '',
        'currpage': str(page),
        'vc_filenumber': '',
        'vc_all': '',
        'texttype': '',
        'fbtime': '',
    }
    for _ in range(3):
        try:
            r = sess.post(LIST_URL, data=data, timeout=15)
            r.encoding = 'utf-8'
            return r.text
        except Exception as e:
            time.sleep(1)
    return None


def parse_list(html):
    """Parse articles from list HTML"""
    items = []
    for tr in re.finditer(r'<tr class="tr_main_value_\w+"[^>]*>(.*?)</tr>', html, re.DOTALL):
        tr_html = tr.group(1)
        a = re.search(r"<a\s+href='([^']+)'[^>]*title='([^']*)'", tr_html)
        date_td = re.search(r"<td align='center' width='80'>([^<]+)</td>", tr_html)
        if a and date_td:
            items.append({
                'url': a.group(1),
                'title': a.group(2).strip(),
                'date': date_td.group(1).strip(),
            })
    return items


def extract_content(html):
    """Extract content from detail page"""
    # Content is between <!--<$[信息内容]>begin--> and <!--<$[信息内容]>end-->
    # With ZJEG_RSS markers
    m = re.search(r'<!--<\$\[信息内容\]>begin-->(.*?)<!--<\$\[信息内容\]>end-->', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    else:
        # Fallback: try ContentStart/ContentEnd
        m = re.search(r'<meta name="ContentStart">(.*?)<meta name="ContentEnd">', html, re.DOTALL)
        content = m.group(1).strip() if m else ''
    
    # Clean scripts and styles
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL)
    return content.strip()


def fetch_detail(url):
    """Fetch detail page"""
    for _ in range(3):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=15)
            resp.encoding = 'utf-8'
            return resp.text
        except Exception as e:
            time.sleep(1)
    return None


def process_item(item):
    """Process one article: fetch detail, extract content"""
    html = fetch_detail(item['url'])
    if not html:
        return None, 'FETCH_ERR'
    
    title = ''
    pub_date = ''
    
    # Title from meta tag
    m = re.search(r'<meta name="ArticleTitle"\s+content="([^"]+)"', html)
    if m:
        title = m.group(1).strip()
    
    # Date from meta tag
    m = re.search(r'<meta name="PubDate"\s+content="([^"]+)"', html)
    if m:
        pub_date = m.group(1).strip()[:10]  # YYYY-MM-DD only
    
    content = extract_content(html)
    
    return {
        'site_name': SITE_NAME,
        'title': title or item['title'],
        'page_url': item['url'],
        'publish_date': pub_date or item['date'],
        'source_url': item['url'],
        'content': content,
    }, None


def main():
    print(f'[{datetime.now().strftime("%H:%M:%S")}] {SITE_NAME}')
    
    # Fetch first page to get total
    html = fetch_list(1)
    if not html:
        print('[ERROR] Failed page 1')
        return
    
    # Find total records and pages
    total = 0
    m = re.search(r'共(\d+)条记录', html)
    if m:
        total = int(m.group(1))
    else:
        print('[ERROR] Cannot find total')
        return
    
    total_pages = 0
    m = re.search(r'共(\d+)页', html)
    if m:
        total_pages = int(m.group(1))
    
    print(f'Total: {total} records, {total_pages} pages')
    
    # Fetch all pages
    all_items = parse_list(html)
    for p in range(2, total_pages + 1):
        h = fetch_list(p)
        if h:
            items = parse_list(h)
            all_items.extend(items)
            print(f'  Page {p}: {len(items)} items')
        else:
            print(f'  Page {p}: FAILED')
        time.sleep(random.uniform(0.3, 0.6))
    
    print(f'Collected: {len(all_items)} items')
    
    recent = [i for i in all_items if i['date'] >= CUTOFF]
    print(f'Within 3 years: {len(recent)} (filtered: {len(all_items)-len(recent)})')
    
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    existing = set(r[0] for r in c.execute('SELECT page_url FROM gov_raw WHERE site_name=?', (SITE_NAME,)))
    
    to_fetch = [i for i in recent if i['url'] not in existing]
    print(f'New items: {len(to_fetch)}')
    
    new = err = 0
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {executor.submit(process_item, item): item for item in to_fetch}
        for i, future in enumerate(as_completed(futures), 1):
            item = futures[future]
            result, error = future.result()
            if error or not result:
                print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... {error or "FAIL"}')
                err += 1
                continue
            try:
                c.execute('''INSERT OR IGNORE INTO gov_raw
                    (site_name, title, page_url, publish_date, source_url, content)
                    VALUES (?,?,?,?,?,?)''', (
                        result['site_name'], result['title'], result['page_url'],
                        result['publish_date'], result['source_url'], result['content']))
                if c.rowcount:
                    new += 1
                    print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... OK ({len(result["content"])}B)')
                else:
                    print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... DUP')
            except Exception as e:
                print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... ERR: {e}')
                err += 1
            conn.commit()
    
    conn.close()
    print(f'\n=== Done === New: {new}, Errors: {err}')


if __name__ == '__main__':
    main()