#!/usr/bin/env python3
"""凌海市人民政府-生态环境-行政管理 爬虫 (Visual SiteBuilder 静态站)"""
import re, requests, sqlite3, os, sys, time
from bs4 import BeautifulSoup
from urllib.parse import urljoin

SITE_NAME = '凌海市-生态环境-行政管理'
BASE = 'http://www.lnlh.gov.cn'
LIST = '/xxgk/zdlyxxgk/sthj1/xzgl.htm'
DB_PATH = os.environ.get('DB_PATH', '/root/search.db')

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
}

def get_total_pages():
    """从分页组件获取总页数"""
    r = requests.get(BASE + LIST, headers=HEADERS, timeout=30)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text, 'lxml')
    paging = soup.find('div', class_=lambda c: c and 'pb_sys' in str(c))
    if paging:
        links = paging.find_all('a')
        max_n = 0
        for a in links:
            txt = a.get_text(strip=True)
            if txt.isdigit():
                max_n = max(max_n, int(txt))
        if max_n > 0:
            return max_n
    return 2  # fallback

def parse_list(page=1):
    """解析列表页"""
    if page == 1:
        url = BASE + LIST
    else:
        # page 2 → xzgl/1.htm, page 3 → xzgl/2.htm
        url = BASE + f'/xxgk/zdlyxxgk/sthj1/xzgl/{page-1}.htm'
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = 'utf-8'
    except Exception as e:
        print(f'  ⚠️ 请求失败 {url}: {e}')
        return []
    soup = BeautifulSoup(r.text, 'lxml')
    items = []
    for li in soup.select('ul li'):
        a = li.find('a')
        span = li.find('span')
        if a and span:
            href = a.get('href', '')
            if not href.startswith('http'):
                href = urljoin(url, href)
            items.append({
                'title': a.get_text(strip=True),
                'url': href,
                'date': span.get_text(strip=True).strip(),
            })
    return items

def parse_detail(url):
    """解析详情页正文"""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = 'utf-8'
        soup = BeautifulSoup(r.text, 'lxml')
        # 正文在 div.v_news_content 内层
        div = soup.find('div', class_='v_news_content')
        if not div:
            # 备用: id=vsb_content
            div = soup.find(id='vsb_content')
        if div:
            for tag in div(['script', 'style']):
                tag.decompose()
            return str(div)
        # 备用: div.list-page
        lp = soup.find('div', class_='list-page')
        if lp:
            for tag in lp(['script', 'style']):
                tag.decompose()
            return str(lp)
    except Exception as e:
        print(f'  ⚠️ 详情解析失败 {url}: {e}')
    return ''

def run(max_pages=None):
    total_pages = get_total_pages()
    if max_pages and max_pages < total_pages:
        total_pages = max_pages

    print(f'📋 共 {total_pages} 页')

    conn = sqlite3.connect(DB_PATH)
    new, skip = 0, 0

    for pg in range(1, total_pages + 1):
        items = parse_list(pg)
        if not items:
            print(f'  ⚠️ 第{pg}页无数据')
            continue
        print(f'📄 第{pg}/{total_pages}页 ({len(items)}条)')
        for item in items:
            body = parse_detail(item['url'])
            try:
                conn.execute("""
                    INSERT OR IGNORE INTO gov_raw
                    (site_name, source_url, page_url, title, publish_date, content, summary)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                """, (SITE_NAME, item['url'], item['url'], item['title'],
                      item['date'], body, item['title']))
                if conn.total_changes > 0:
                    new += 1
                else:
                    skip += 1
            except Exception as e:
                print(f'  ❌ - {e}')
            time.sleep(0.3)
        conn.commit()

    conn.close()
    print(f'\n✅ {SITE_NAME}: 新增{new}, 跳过{skip}, 共{total_pages}页')

if __name__ == '__main__':
    max_p = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else None
    run(max_p)