#!/usr/bin/env python3
"""杨凌示范区 - 环评公示 爬虫
URL: https://www.yangling.gov.cn/zwgk/fdzdgknr/hjbh/hpgs/
列表: <li><span>date</span><a href="...">title</a></li>
分页: 1.html → 4.html (共4页, ~80条)
详情: <h1>标题, <div class="m-txt-article">正文
"""

import sys, os, re, time, json, logging, argparse, sqlite3
from datetime import datetime, timedelta
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup

BASE_URL = "https://www.yangling.gov.cn/zwgk/fdzdgknr/hjbh/hpgs"
SITE_NAME = "杨凌示范区-环评公示"
MAX_PAGES = 10
DATE_CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
REQUEST_DELAY = 0.3
DB_PATH = "/root/search.db"

logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(levelname)s %(message)s', datefmt='%H:%M:%S')
log = logging.getLogger(__name__)

def init_db():
    db = sqlite3.connect(DB_PATH)
    db.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT NOT NULL,
        title TEXT NOT NULL,
        page_url TEXT NOT NULL UNIQUE,
        publish_date TEXT,
        source_url TEXT,
        content TEXT,
        crawl_time TEXT DEFAULT (datetime('now','localtime'))
    )""")
    db.commit()
    return db

def record_exists(db, page_url):
    return db.execute("SELECT 1 FROM gov_raw WHERE page_url=?", (page_url,)).fetchone() is not None

def save_record(db, item):
    db.execute("INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, source_url, content) VALUES (?,?,?,?,?,?)",
               (item['site_name'], item['title'], item['page_url'], item.get('publish_date',''), item.get('source_url',''), item.get('content','')))
    return db.cursor().rowcount

def fetch(url):
    try:
        r = requests.get(url, timeout=20, headers={'User-Agent': 'Mozilla/5.0'})
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        log.error(f"请求失败 {url[:60]}: {e}")
        return None

def get_total_pages(html):
    """从分页区域检测总页数"""
    m = re.search(r'共(\d+)页', html)
    if m:
        return int(m.group(1))
    # fallback: count page links
    pages = re.findall(r'href="(\d+)\.html"', html)
    if pages:
        return max(int(p) for p in pages)
    return 1

def parse_list_items(html):
    """从列表页提取文章"""
    soup = BeautifulSoup(html, 'html.parser')
    items = []
    for li in soup.find_all('li'):
        span = li.find('span')
        a = li.find('a', href=True)
        if not (span and a):
            continue
        date = span.get_text(strip=True)
        if not re.match(r'\d{4}-\d{2}-\d{2}', date):
            continue
        href = a['href']
        if not href.startswith('http'):
            href = urljoin(BASE_URL + '/', href)
        title = a.get_text(strip=True)
        items.append({'title': title, 'page_url': href, 'publish_date': date})
    return items

def parse_detail(html, page_url):
    """解析详情页"""
    soup = BeautifulSoup(html, 'html.parser')
    h1 = soup.find('h1')
    title = h1.get_text(strip=True) if h1 else ''
    m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
    pub_date = m.group(1) if m else ''
    content_div = soup.select_one('.m-txt-article')
    content_html = ''
    if content_div:
        for s in content_div.select('script,style'):
            s.decompose()
        content_html = str(content_div).strip()
    return title, pub_date, content_html

def crawl(full=True):
    db = init_db()
    total_saved = 0
    total_skipped = 0
    total_count = 0

    # 第1页
    html1 = fetch(f"{BASE_URL}/1.html")
    if not html1:
        log.error("无法获取第1页")
        return {"total": 0, "saved": 0, "skipped": 0}

    total_pages = get_total_pages(html1)
    pages = min(total_pages, MAX_PAGES) if full else 1
    log.info(f"共{total_pages}页, 爬{pages}页")

    all_items = []
    for page in range(1, pages + 1):
        if page == 1:
            html = html1
        else:
            html = fetch(f"{BASE_URL}/{page}.html")
            if not html:
                continue
        items = parse_list_items(html)
        log.info(f"第{page}页: {len(items)} 条")
        all_items.extend(items)
        time.sleep(REQUEST_DELAY)

    # 去重
    seen = set()
    unique_items = []
    for item in all_items:
        if item['page_url'] not in seen:
            seen.add(item['page_url'])
            unique_items.append(item)

    log.info(f"去重后共 {len(unique_items)} 条")

    # 详情
    for i, item in enumerate(unique_items):
        if item['publish_date'] and item['publish_date'] < DATE_CUTOFF:
            log.info(f"  跳过(日期过旧): {item['title'][:40]} ({item['publish_date']})")
            continue

        if record_exists(db, item['page_url']):
            total_skipped += 1
            continue

        total_count += 1
        log.info(f"  详情 [{total_count}/{len(unique_items)}]: {item['title'][:50]}...")
        html = fetch(item['page_url'])
        if html:
            title, pub_date, content = parse_detail(html, item['page_url'])
            item['title'] = title or item['title']
            item['publish_date'] = pub_date or item['publish_date']
            item['content'] = content
            item['source_url'] = "杨凌示范区管委会"
            item['site_name'] = SITE_NAME

            if save_record(db, item):
                total_saved += 1
            else:
                total_skipped += 1
            db.commit()
        time.sleep(REQUEST_DELAY)

    log.info(f"完成: total={total_count}, saved={total_saved}, skipped={total_skipped}")
    return {"total": total_count, "saved": total_saved, "skipped": total_skipped}

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--full', action='store_true')
    args = parser.parse_args()
    result = crawl(full=args.full)
    print(json.dumps(result, ensure_ascii=False))