#!/usr/bin/env python3
"""
新疆准东经济技术开发区 — 环评公示 爬虫
==============================
  python3 crawl_xjzdkfq.py              # 增量（默认：只爬第1页最新）
  python3 crawl_xjzdkfq.py --full       # 全量：15页
  python3 crawl_xjzdkfq.py --pages=3    # 指定页数
  python3 crawl_xjzdkfq.py --sync       # 同步已有数据到服务器

"""

import sys, os, re, sqlite3, hashlib, time, subprocess, html as html_mod, ssl, urllib.request
from datetime import datetime
from urllib.parse import urljoin

# ─── 配置 ───
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

SITE_NAME = "新疆准东经济技术开发区"
DOMAIN = "www.xjzdkfq.cn"
BASE_URL = "https://www.xjzdkfq.cn"
LIST_TEMPLATE = "https://www.xjzdkfq.cn/p163/sthj1.html"       # page 1
LIST_TEMPLATE_N = "https://www.xjzdkfq.cn/p163/sthj1_{page}.html"  # pages 2-N
TOTAL_PAGES = 15

# ─── HTTP 工具 ───
CTX = ssl._create_unverified_context()
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "Chrome/125.0.0.0 Safari/537.36",
}

def http_get(url, timeout=25):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=timeout, context=CTX)
        raw = resp.read()
        text = raw.decode("utf-8", errors="replace")
        return text
    except Exception as e:
        return None

# ─── 列表页解析 ───
def parse_list_page(html):
    """提取 xxgkzd 区域的文章链接"""
    items = []
    seen_urls = set()

    # 只取 xxgkzd 区域内的链接
    area_m = re.search(r'class="xxgkzd"[^>]*>(.*?)<div[^>]*id="myPage"', html, re.DOTALL)
    if not area_m:
        area_m = re.search(r'class="xxgkzd"[^>]*>(.*?)</div>', html, re.DOTALL)
    area = area_m.group(1) if area_m else html

    # 匹配文章链接: /p163/sthj1/YYYYMMDD/NNNNNN.html
    for m in re.finditer(
        r'<a[^>]*href="(/p163/sthj1/\d+/\d+\.html)"[^>]*>(.*?)</a>',
        area, re.DOTALL
    ):
        href = m.group(1).strip()
        title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        if not title or not href:
            continue
        if not href.startswith('http'):
            href = urljoin(BASE_URL, href)
        if href in seen_urls:
            continue
        seen_urls.add(href)
        items.append({'title': title, 'url': href})

    return items

def get_detail(url):
    """获取详情页：标题、正文HTML、日期"""
    html = http_get(url)
    if not html:
        return None

    # 标题：<title> 去后缀
    title = ''
    tm = re.search(r'<title>([^<]+)</title>', html)
    if tm:
        title = re.sub(r'_国家级新疆准东经济技术开发区\s*$', '', tm.group(1)).strip()

    # 日期：从URL路径提取 YYYYMMDD
    date = ''
    dm = re.search(r'/(\d{8})/\d+\.html', url)
    if dm:
        d = dm.group(1)
        date = f"{d[:4]}-{d[4:6]}-{d[6:8]}"

    # 正文：<font id="Zoom"> ... </font>
    content = ''
    cm = re.search(
        r'<font[^>]*id="?Zoom"?[^>]*>(.*?)</font>',
        html, re.DOTALL
    )
    if cm:
        content = cm.group(1).strip()
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<p>\s*<br\s*/?>\s*</p>', '', content)
        if len(content) > 10000:
            content = content[:10000] + '...'

    return {
        'title': title,
        'content': content,
        'date': date,
    }

def extract_plain_text(html_content):
    if not html_content:
        return ''
    text = re.sub(r'</(p|div|li|h[1-6]|tr|td|th)>', r'</\1>\n\n', html_content)
    text = re.sub(r'<br\s*/?>', '\n', text)
    text = re.sub(r'<[^>]+>', '', text)
    text = html_mod.unescape(text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()

def categorize(title, text):
    t = (title + ' ' + text).lower()
    tags = []
    for stage, kws in [
        ('环评阶段', ['环评', '环境影响评价', '征求意见', '第一次公示']),
        ('审批报批', ['审批', '批复', '拟批复', '报批', '许可', '批准']),
        ('竣工验收', ['验收', '竣工', '试运行']),
        ('招标采购', ['招标', '投标', '采购']),
        ('规划设计', ['规划', '设计', '可研', '可行性研究', '选址']),
        ('安全评价', ['安全评价', '安评', '安全预评价', '安全验收']),
    ]:
        if any(kw in t for kw in kws):
            tags.append(stage)
    return ','.join(tags)

# ─── DB 操作 ───
def ensure_table():
    """确保search.db有gov_raw和gov_search表"""
    conn = sqlite3.connect(SEARCH_DB)
    conn.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT, page_url TEXT,
        title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0,
        summary TEXT, status TEXT, category TEXT DEFAULT '',
        visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT ''
    )''')
    try:
        conn.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
            title, content, site_name,
            content='gov_raw', content_rowid='id', tokenize='unicode61'
        )''')
    except sqlite3.OperationalError:
        pass
    conn.commit()
    conn.close()

def is_url_crawled(url):
    conn = sqlite3.connect(SEARCH_DB)
    cur = conn.execute("SELECT id FROM gov_raw WHERE page_url=? AND site_name=?", (url, SITE_NAME))
    r = cur.fetchone()
    conn.close()
    return r is not None

def store_item(item):
    url = item['url']
    content = item.get('content', '')
    summary = item.get('summary', extract_plain_text(content)[:300])
    date = item.get('date', '')
    title = item['title']
    cat = categorize(title, summary)

    conn = sqlite3.connect(SEARCH_DB)
    try:
        conn.execute(
            "INSERT OR IGNORE INTO gov_raw "
            "(title, content, publish_date, page_url, source_url, site_name, summary, tags) "
            "VALUES (?,?,?,?,?,?,?,?)",
            (title, content, date, url, DOMAIN, SITE_NAME, summary, cat)
        )
        if conn.total_changes > 0:
            row_id = conn.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,)).fetchone()
            if row_id:
                try:
                    conn.execute(
                        "INSERT INTO gov_search(rowid, title, content, site_name) VALUES (?,?,?,?)",
                        (row_id[0], title, content, SITE_NAME)
                    )
                except sqlite3.IntegrityError:
                    pass
        conn.commit()
        return conn.total_changes > 0
    finally:
        conn.close()


# ─── 主逻辑 ───
def main():
    args = sys.argv[1:]
    full_mode = '--full' in args

    if full_mode:
        pages_to_crawl = TOTAL_PAGES
        mode = "全量"
    else:
        pages_match = [a for a in args if a.startswith('--pages=')]
        pages_to_crawl = int(pages_match[0].split('=')[1]) if pages_match else 1
        mode = f"增量（{pages_to_crawl}页）"

    print(f"🔍 {SITE_NAME}爬虫 — {mode}模式")
    print(f"  总页数: {TOTAL_PAGES}, 本次爬取: {pages_to_crawl}页")
    print()

    ensure_table()

    new_total = 0
    skip_total = 0
    for page in range(1, pages_to_crawl + 1):
        if page == 1:
            url = LIST_TEMPLATE
        else:
            url = LIST_TEMPLATE_N.format(page=page)

        html = http_get(url)
        if not html:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ 获取列表页失败")
            continue

        items = parse_list_page(html)
        if not items:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ 未解析到条目")
            continue

        page_new = 0
        page_skip = 0
        for item in items:
            if is_url_crawled(item['url']):
                page_skip += 1
                skip_total += 1
                continue

            detail = get_detail(item['url'])
            if detail:
                item['title'] = detail['title'] or item['title']
                item['content'] = detail['content'] or ''
                item['date'] = detail['date'] or ''
                item['summary'] = extract_plain_text(detail.get('content', ''))[:300]

            store_item(item)
            page_new += 1
            new_total += 1

        print(f"  [p{page:2d}/{pages_to_crawl}] ✓ {len(items)}条 (新增{page_new} 跳过{page_skip})" + ' ' * 20, end='\r')
        time.sleep(0.5 if full_mode else 0.3)

    print()
    print(f"\n📊 完成！新增: {new_total} | 跳过: {skip_total}")

if __name__ == '__main__':
    main()