#!/usr/bin/env python3
import os
"""庆城县人民政府-生态环境 爬虫"""
import os, re, sqlite3, time
from datetime import datetime, timezone, timedelta
from urllib.parse import urljoin

BASE = 'https://www.chinaqingcheng.gov.cn'
SITE_NAME = '庆城县-生态环境'
DB = os.environ.get('SEARCH_DB', '/root/search.db')
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'

tz = timezone(timedelta(hours=8))
cutoff = datetime.now(tz) - timedelta(days=365*3)
cutoff_str = cutoff.strftime('%Y-%m-%d')

def fetch(url):
    import urllib.request
    req = urllib.request.Request(url, headers={'User-Agent': UA})
    try:
        with urllib.request.urlopen(req, timeout=20) as r:
            data = r.read()
            enc = r.headers.get_content_charset() or 'utf-8'
            return data.decode(enc, errors='replace')
    except Exception as e:
        print(f'  [WARN] fetch fail: {e}')
        return ''

def extract_content(html):
    """从详情页提取正文"""
    m = re.search(r'<div class="conTxt"[^>]*>(.*?)</div>\s*<div class="userControl"', html, re.DOTALL)
    if m:
        return m.group(1).strip()
    return ''

def extract_date(html):
    """提取发布时间"""
    m = re.search(r'发布时间[：:](\d{4}-\d{2}-\d{2})', html)
    if m:
        return m.group(1)
    return ''

def extract_title(html):
    """从标题标签提取"""
    m = re.search(r'<title>(.*?)</title>', html)
    if m:
        title = m.group(1).strip()
        # 去掉站点后缀
        for suffix in ['_生态环境_重大民生信息_ 法定主动公开内容_政府信息公开_庆城县人民政府',
                       '_庆城县人民政府']:
            if title.endswith(suffix):
                title = title[:-len(suffix)]
                break
        return title
    return ''

def list_articles(page=1):
    """返回该页所有文章 (full_url, title, date_str)"""
    if page == 1:
        url = f'{BASE}/zwgk/xxgkml/zdmsxx/sthjing/'
    else:
        url = f'{BASE}/zwgk/xxgkml/zdmsxx/sthjing_{page}/'

    html = fetch(url)
    if not html:
        return []

    articles = []
    # 每篇文章: <li><span class="date">2026-06-11</span><a href="/zwgk/.../content_38332">Title</a></li>
    pattern = re.compile(
        r'<span class="date">(\d{4}-\d{2}-\d{2})</span>.*?'
        r'<a[^>]*href="([^"]*content_\d+)"[^>]*title="[^"]*">\s*(.*?)\s*</a>',
        re.DOTALL,
    )
    for m in pattern.finditer(html):
        date_str = m.group(1)
        url_path = m.group(2)
        title = re.sub(r'<[^>]+>', '', m.group(3)).strip()
        full_url = BASE + url_path if url_path.startswith('/') else urljoin(BASE, url_path)
        articles.append((full_url, title, date_str))

    return articles

def get_total_pages(html):
    pages = re.findall(r"href='/zwgk/xxgkml/zdmsxx/sthjing_(\d+)'", html)
    if pages:
        return max(int(p) for p in pages)
    return 1

def crawl(limit_pages=None):
    """主爬取逻辑"""
    html = fetch(f'{BASE}/zwgk/xxgkml/zdmsxx/sthjing/')
    if not html:
        print('[ERROR] 首页打不开')
        return

    total_pages = get_total_pages(html)
    if limit_pages:
        total_pages = min(total_pages, limit_pages)

    print(f'总页数: {total_pages}')

    conn = sqlite3.connect(DB, timeout=60)
    c = conn.cursor()

    new_count = 0
    for page in range(1, total_pages + 1):
        articles = list_articles(page)
        if not articles:
            print(f'  第{page}页: 0 条')
            continue

        filtered = [(u, t, d) for u, t, d in articles if d >= cutoff_str]
        print(f'  第{page}页: {len(articles)} 条 (近3年: {len(filtered)})')

        for url, title, list_date in filtered:
            c.execute('SELECT id FROM gov_raw WHERE page_url = ?', (url,))
            row = c.fetchone()

            if row:
                continue  # 已有正文

            detail_html = fetch(url)
            if not detail_html:
                print(f'  ✗ {title[:30]}... 下载失败')
                continue

            detail_title = extract_title(detail_html) or title
            detail_date = extract_date(detail_html) or list_date
            content = extract_content(detail_html)

            if content:
                if row:
                    c.execute(
                        'UPDATE gov_raw SET title=?, publish_date=?, content=? WHERE page_url=?',
                        (detail_title, detail_date, content, url),
                    )
                else:
                    c.execute(
                        'INSERT OR IGNORE INTO gov_raw (site_name, title, publish_date, page_url, source_url, content) VALUES (?,?,?,?,?,?)',
                        (SITE_NAME, detail_title, detail_date, url, url, content),
                    )
                new_count += 1
                print(f'  ✓ {detail_title[:30]}... (+1)')
            else:
                print(f'  - {title[:30]}... (无正文)')

            time.sleep(0.2)

        if len(filtered) < len(articles):
            break

        time.sleep(0.5)

    conn.commit()

    # 更新 FTS
    # QC20260926 去掉手写 gov_search 整站删除(抢锁源; FTS 由 gov_raw 触发器维护) 
    # c.execute("DELETE FROM gov_search WHERE site_name=?", (SITE_NAME,))
    rows = c.execute("SELECT id, title, content FROM gov_raw WHERE site_name=? AND content IS NOT NULL AND content != ''", (SITE_NAME,)).fetchall()
    for rid, title, content in rows:
        # 2026-09-22: 先提交 gov_raw —— 下面手动写 FTS 会因触发器已写过同一
        #   rowid 而 IntegrityError，若不先 commit，这条记录会被一并回滚（静默丢数据）
        conn.commit()
        c.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?,?,?,?)",
                  (rid, title, SITE_NAME, content or ''))
    conn.commit()

    c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    total = c.fetchone()[0]
    conn.close()

    print(f'\n新增: {new_count} | 总计: {total}')

if __name__ == '__main__':
    import sys
    full = '--full' in sys.argv
    limit_pages = None
    if full:
        limit_pages = 100
    elif len(sys.argv) > 1 and sys.argv[1].isdigit():
        limit_pages = int(sys.argv[1])
    if not full:
        print('增量模式: 只爬第1页')
        crawl(limit_pages=1)
    else:
        crawl(limit_pages=limit_pages)