#!/usr/bin/env python3
"""兴安盟生态环境局-建设项目环评 爬虫"""
import re, sys, os, json, argparse
sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import fetch_page

BASE = 'http://sthjj.xam.gov.cn'
SITE_NAME = '兴安盟生态环境局-建设项目环评'
LIST_URL = BASE + '/hygq/zfxxgk/zfxxgkml/hjzdlyxxgk/jsxmhp/index.html'

def parse_list(html):
    items = []
    for m in re.finditer(r'<li><b></b><a href="([^"]+)"[^>]*title="([^"]*)"\s*istitle="true">', html):
        url = m.group(1)
        title = m.group(2)
        # 从 match 后面找 date
        rest = html[m.end():]
        dm = re.search(r'\s*<span>([^<]+)</span>', rest)
        date = dm.group(1) if dm else ''
        title = title.lstrip(':').strip()
        items.append({'title': title, 'url': BASE + url, 'date': date})
    return items

def get_total_pages(html):
    m = re.search(r'd71f1e60-(\d+)\.html[^>]*>尾页', html)
    if m:
        return int(m.group(1))
    m = re.search(r'totalpage="(\d+)"', html)
    if m:
        return int(m.group(1))
    return 1

def parse_detail(html, url):
    m = re.search(r'<div class="detail_txt" id="mainText">(.*?)</div>\s*<div class="sliblings', html, re.S)
    if not m:
        m = re.search(r'<div class="detail_txt" id="mainText">(.*?)</div>\s*<div class="detail_foot', html, re.S)
    if not m:
        m = re.search(r'<div class="detail_txt" id="mainText">(.*?)</div>', html, re.S)
    if not m:
        return None, []

    content_html = m.group(1)
    attachments = []
    for am in re.finditer(r'<a\s+href="([^"]*\.(?:doc|docx|pdf|xls|xlsx|rar|zip))"[^>]*>([^<]+)</a>', content_html, re.I):
        attach_url = am.group(1)
        attach_title = am.group(2).strip()
        if attach_url.startswith('/'):
            attach_url = BASE + attach_url
        attachments.append(f'[{attach_title}]({attach_url})')

    tables = []
    for tm in re.finditer(r'(<table[^>]*>.*?</table>)', content_html, re.I | re.S):
        tables.append(tm.group(1))

    content_html = re.sub(r'<br\s*/?>', '\n', content_html)
    text = re.sub(r'<[^>]+>', '', content_html)
    text = text.replace('&nbsp;', ' ').replace('&lt;', '<').replace('&gt;', '>')
    text = re.sub(r'&[a-z]+;', '', text)
    text = re.sub(r'\n\s*\n', '\n\n', text)
    text = text.strip()
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=[\u4e00-\u9fff])', '', text)

    parts = []
    if text:
        parts.append(text)
    if tables:
        parts.append('\n\n### 表格\n')
        for t in tables:
            parts.append(t)
    content = '\n'.join(parts)
    return content, attachments

def insert_db(data_list):
    """直接写入 search.db"""
    import sqlite3
    db = sqlite3.connect('/root/search.db', timeout=30)
    db.execute('PRAGMA journal_mode=WAL')
    db.execute('PRAGMA synchronous=NORMAL')

    ok, skip = 0, 0
    for d in data_list:
        try:
            pub_date = d.get('pub_date', '')
            if pub_date:
                md = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', str(pub_date))
                if md:
                    y, mo, day = int(md.group(1)), int(md.group(2)), int(md.group(3))
                    pub_date = f'{y:04d}-{mo:02d}-{day:02d}'

            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, category, tags, attachments)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (
                d.get('site_name', ''),
                d.get('source_url', d.get('url', '')),
                d.get('url', ''),
                (d.get('title') or '')[:500],
                pub_date,
                (d.get('summary') or '')[:500],
                d.get('content', ''),
                'active',
                d.get('category', ''),
                d.get('tags', ''),
                d.get('attachments', ''),
            ))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except Exception as e:
            print(f'  DB error: {e}')
            skip += 1

    db.commit()

    try:
        db.execute("""INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r
            WHERE r.id NOT IN (SELECT rowid FROM gov_search)""")
        db.commit()
    except Exception as e:
        print(f'  FTS ERROR: {e}')

    print(f'  [DB] new:{ok} skip:{skip}')
    db.close()

def main():
    parser = argparse.ArgumentParser(description='兴安盟-建设项目环评')
    parser.add_argument('--pages', type=int, default=1, help='爬取页数')
    parser.add_argument('--max-articles', type=int, default=0, help='最大记录数')
    args = parser.parse_args()

    max_pages = args.pages if args.pages > 0 else 1

    html = fetch_page(LIST_URL)
    if not html:
        print('获取首页失败')
        return

    total_pages = get_total_pages(html)
    pages_to_fetch = min(max_pages, total_pages)
    print(f'总页数: {total_pages}, 将爬取: {pages_to_fetch} 页')

    all_items = []
    seen_urls = set()

    items = parse_list(html)
    for item in items:
        if item['url'] not in seen_urls:
            seen_urls.add(item['url'])
            all_items.append(item)

    for page in range(2, pages_to_fetch + 1):
        page_url = f'{BASE}/hygq/zfxxgk/zfxxgkml/hjzdlyxxgk/jsxmhp/d71f1e60-{page}.html'
        html = fetch_page(page_url)
        if not html:
            print(f'第{page}页获取失败')
            break
        items = parse_list(html)
        if not items:
            print(f'第{page}页无数据，停止')
            break
        for item in items:
            if item['url'] not in seen_urls:
                seen_urls.add(item['url'])
                all_items.append(item)
        print(f'第{page}页: {len(items)} 条')

    count = 0
    limit = args.max_articles if args.max_articles > 0 else len(all_items)
    batch = []

    for item in all_items[:limit]:
        html = fetch_page(item['url'])
        if not html:
            print(f"详情获取失败: {item['url']}")
            continue

        content, attachments = parse_detail(html, item['url'])
        if content is None:
            print(f'正文解析失败: {item["title"][:30]}')
            continue

        if attachments:
            content += '\n\n### 附件\n' + '\n'.join(attachments)

        batch.append({
            'title': item['title'],
            'url': item['url'],
            'pub_date': item['date'],
            'site_name': SITE_NAME,
            'content': content,
            'summary': item['title'][:200],
            'attachments': json.dumps(attachments, ensure_ascii=False),
        })
        count += 1

    if batch:
        insert_db(batch)

    print(f'完成: 入库 {count} 条 (共 {len(all_items)} 条)')

if __name__ == '__main__':
    main()