#!/usr/bin/env python3
import sys, os, re, json, sqlite3, requests
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = 'https://117.187.129.149'
HOST = 'www.gzchishui.gov.cn'
LIST_PATH = '/xwzx/tzgg/zw/zxxx/'
SITE_NAME = '赤水市-通知公告-最新信息'
HEADERS = {'Host': HOST, 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
CUT_DATE = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')

def fetch(url, label=''):
    try:
        r = requests.get(url, headers=HEADERS, verify=False, timeout=20)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f'[ERROR] {label}: {e}')
        return ''

def parse_list(html):
    soup = BeautifulSoup(html, 'lxml')
    content = soup.find('div', class_='content01')
    if not content: return []
    items = []
    for a in content.find_all('a', href=True):
        title = a.get('title', '') or a.get_text(strip=True)
        href = a['href'].strip()
        date_span = a.find_next('span')
        date = date_span.get_text(strip=True) if date_span else ''
        if date: items.append((title, href, date))
    return items

def parse_detail(html, url):
    soup = BeautifulSoup(html, 'lxml')
    t = soup.find(id='NewsArticleTitle')
    title = t.get_text(strip=True) if t else ''
    p = soup.find(id='NewsArticlePubDay')
    pub_date = p.get_text(strip=True)[:10] if p else ''
    content_div = soup.find('div', class_='neirong')
    if not content_div:
        content_div = soup.find('div', class_=lambda c: c and 'TRS_UEDITOR' in str(c))
    if not content_div:
        font = soup.find('div', class_='font')
        if font:
            detail = font.find('div', class_='detail_main_xx')
            if detail:
                parts = [str(sib) for sib in detail.find_next_siblings() if sib.name in ('p','div','table','ul','ol')]
                content_html = ''.join(parts)
            else: content_html = str(font)
        else: content_html = ''
    else: content_html = str(content_div)
    return title, pub_date, content_html

def save_article(title, pub_date, content_html, page_url):
    summary = BeautifulSoup(content_html, 'lxml').get_text(strip=True)[:200] if content_html else title
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    c.execute('INSERT OR IGNORE INTO gov_raw (title, page_url, site_name, summary, content, publish_date) VALUES (?,?,?,?,?,?)',
              (title, page_url, SITE_NAME, summary, content_html, pub_date))
    n = c.rowcount
    conn.commit(); conn.close()
    return n

def get_total_pages(html):
    soup = BeautifulSoup(html, 'lxml')
    tail = soup.find('a', href=re.compile(r'index_\d+\.html'), string='尾页')
    if tail:
        m = re.search(r'index_(\d+)', tail['href'])
        if m: return int(m.group(1)) + 1
    nums = set()
    for a in soup.find_all('a', href=re.compile(r'index_(\d+)\.html')):
        m = re.search(r'index_(\d+)', a['href'])
        if m: nums.add(int(m.group(1)))
    return max(nums) + 1 if nums else 1

def resolve_url(href):
    if href.startswith('http'): return href.replace('https://www.gzchishui.gov.cn', BASE_URL)
    if href.startswith('/'): return BASE_URL + href
    if href.startswith('./'): return BASE_URL + LIST_PATH + href[2:]
    return BASE_URL + LIST_PATH + href

def main(mode='full'):
    start = datetime.now()
    print(f'[赤水] mode={mode}, cut_date={CUT_DATE}')
    html = fetch(BASE_URL + LIST_PATH, '首页')
    if not html: return
    total_pages = get_total_pages(html)
    print(f'[赤水] 总页数: {total_pages}')
    all_items = []
    for page in range(total_pages):
        url = BASE_URL + LIST_PATH + ('' if page == 0 else f'index_{page}.html')
        html = fetch(url, f'第{page+1}页')
        if not html: continue
        items = parse_list(html)
        all_items.extend(items)
        print(f'[赤水] 第{page+1}/{total_pages}页: {len(items)}条')
        if items and items[-1][2] < CUT_DATE: break
    print(f'[赤水] 列表共 {len(all_items)} 条')
    new, exists, old = 0, 0, 0
    for idx, (title, href, date) in enumerate(all_items):
        if date and date < CUT_DATE: old += 1; continue
        html = fetch(resolve_url(href), f'详情{idx+1}')
        if not html: continue
        dt, pd, ch = parse_detail(html, resolve_url(href))
        n = save_article(dt or title, pd or date, ch, resolve_url(href))
        if n: new += 1; exists += 0
        else: exists += 1
        if (idx+1) % 20 == 0:
            el = (datetime.now()-start).total_seconds()
            print(f'[赤水] {idx+1}/{len(all_items)}, 新增{new}, 已存在{exists}, {el:.0f}s')
    el = (datetime.now()-start).total_seconds()
    print(f'[赤水] 完成! 新增{new}, 已存在{exists}, 跳过旧{old}, 耗时{el:.0f}s')

if __name__ == '__main__':
    main(sys.argv[1] if len(sys.argv)>1 else 'full')