#!/usr/bin/env python3
import os
"""Crawl 沂水县 - 公告公示 (5个子栏目)
https://www.yishui.gov.cn/xzwgk1/...

Usage:
  python3 crawl_yishui.py <site_key> [incremental]

Keys:
  ysxycjd  -- 沂水县沂城街道公告公示
  ysxyzz   -- 沂水县沂州镇公告公示
  ysxgqz   -- 沂水县功能区公告公示
  fdzdgknr -- 沂水县法定主动公开公告公示
  ysxzgz   -- 沂水县县直公告公示
"""
import requests, re, sqlite3, sys, time, hashlib
from datetime import datetime, timedelta
from bs4 import BeautifulSoup

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = 'https://www.yishui.gov.cn'
CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')

SUB_SITES = {
    'ysxycjd': {'path': 'xzwgk1/bm_xz_jd_xxgk/xz_jd_/ysxycjd/fdzdgknr1', 'name': '沂水县 - 沂城街道公告公示'},
    'ysxyzz':  {'path': 'xzwgk1/bm_xz_jd_xxgk/xz_jd_/ysxyzz/fdzdgknr1',   'name': '沂水县 - 沂州镇公告公示'},
    'ysxgqz':  {'path': 'xzwgk1/bm_xz_jd_xxgk/xz_jd_/ysxgqz/fdzdgknr1',   'name': '沂水县 - 功能区公告公示'},
    'fdzdgknr': {'path': 'xzwgk1/fdzdgknr',                                 'name': '沂水县 - 法定主动公开公告公示'},
    'ysxzgz':  {'path': 'xzwgk1/bm_xz_jd_xxgk/xz_jd_/ysxzgz/fdzdgknr1',   'name': '沂水县 - 县直公告公示'},
}

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
}
session = requests.Session()
session.headers.update(HEADERS)

def resolve_url(href, base_path):
    """Resolve relative URL like ../../info/xxx.htm against base_path"""
    if href.startswith('http'):
        return href
    if href.startswith('/'):
        return BASE_URL + href
    # Count ../ levels
    levels = 0
    while href.startswith('../'):
        levels += 1
        href = href[3:]
    parts = base_path.strip('/').split('/')
    if levels >= len(parts):
        return BASE_URL + '/' + href
    resolved = '/'.join(parts[:-levels]) + '/' + href
    return BASE_URL + '/' + resolved.replace('//', '/')

def parse_list(html, base_path):
    """Parse article list from page HTML"""
    items = []
    soup = BeautifulSoup(html, 'html.parser')
    for ul in soup.find_all('ul'):
        lis = ul.find_all('li', recursive=False)
        if len(lis) < 5:
            continue
        articles = []
        for li in lis:
            a = li.find('a', href=True)
            span = li.find('span')
            if not a or not span:
                continue
            date = span.get_text(strip=True)
            href = a['href']
            # Must be a date like 2026-06-12 AND link should go to /info/ or /zwgk/
            if not re.match(r'\d{4}-\d{2}-\d{2}', date):
                continue
            if '/info/' not in href and '/zwgk/' not in href:
                continue  # skip navigation items
            title = a.get('title') or a.get_text(strip=True)
            if not title or len(title) < 10:
                continue
            url = resolve_url(href, base_path)
            articles.append({'title': title.strip(), 'url': url, 'date': date})
        
        if len(articles) >= 3:
            return articles
    return items

def get_pagination_info(html):
    """Get next page URL and total page count"""
    soup = BeautifulSoup(html, 'html.parser')
    page_div = soup.find('span', class_='p_pages')
    if not page_div:
        return None, 1
    
    next_link = None
    max_href_num = 0
    for a in page_div.find_all('a', href=True):
        href = a['href']
        txt = a.get_text(strip=True)
        if '下页' in txt or '下一页' in txt:
            if 'javascript' not in href:
                next_link = href
        # Find highest number in href like gggs/6.htm
        m = re.search(r'/(\d+)\.htm', href)
        if m:
            n = int(m.group(1))
            if n > max_href_num:
                max_href_num = n
    
    return next_link, max_href_num + 1  # total pages = max_href_num + 1

def fetch_detail(url):
    try:
        resp = session.get(url, timeout=30)
        resp.encoding = 'utf-8'
        html = resp.text
    except:
        return '', '', ''
    
    tm = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
    title = tm.group(1).strip() if tm else ''
    
    dm = re.search(r'<meta name="PubDate" content="([^"]*)"', html)
    date_str = dm.group(1)[:10] if dm else ''
    
    soup = BeautifulSoup(html, 'html.parser')
    con = soup.find('div', class_='newscontent_s')
    if con:
        for tag in con.find_all(['script', 'style']):
            tag.decompose()
        return title, date_str, str(con).strip()
    
    return title, date_str, ''

def main():
    if len(sys.argv) < 2:
        keys = ', '.join(SUB_SITES.keys())
        print(f"Usage: python3 crawl_yishui.py <site_key> [incremental]\nKeys: {keys}")
        return
    
    site_key = sys.argv[1]
    if site_key not in SUB_SITES:
        print(f"Unknown key: {site_key}")
        return
    
    info = SUB_SITES[site_key]
    base_path = info['path']
    site_name = info['name']
    is_incremental = 'incremental' in sys.argv
    list_url = f'{BASE_URL}/{base_path}/gggs.htm'
    
    print(f'=== {site_name} ===')
    
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted, skipped, old_total = 0, 0, 0
    consecutive_old = 0
    min_pages = 5
    start_time = time.time()
    
    pg = 1
    current_url = list_url
    has_articles = True
    
    while has_articles:
        try:
            resp = session.get(current_url, timeout=30)
            resp.encoding = 'utf-8'
            html = resp.text
        except Exception as e:
            print(f'  [ERROR] Page {pg}: {e}')
            time.sleep(2)
            if pg > 1:
                break
            current_url = list_url
            pg += 1
            continue
        
        items = parse_list(html, base_path)
        if not items:
            print(f'  Page {pg}: 0 items (stop)')
            break
        
        all_old = all(item['date'] < CUTOFF_DATE for item in items if item['date'])
        if all_old:
            consecutive_old += 1
            if consecutive_old >= 2 and pg >= min_pages:
                print(f'  Page {pg}: all old, stop')
                break
        else:
            consecutive_old = 0
        
        for item in items:
            if item['date'] and item['date'] < CUTOFF_DATE:
                old_total += 1
                continue
            
            has = c.execute("SELECT 1 FROM gov_raw WHERE page_url=? AND content IS NOT NULL AND content!=''", (item['url'],)).fetchone()
            if has:
                skipped += 1
                continue
            
            try:
                title, date_str, content = fetch_detail(item['url'])
            except Exception as e:
                print(f'  [ERROR] Detail: {e}')
                time.sleep(1)
                skipped += 1
                continue
            
            final_title = title or item['title']
            final_date = date_str or item['date']
            int_id = int(hashlib.md5(item['url'].encode()).hexdigest()[:15], 16) % (2**63)
            date_rank = int(final_date.replace('-', '')) if final_date else 0
            summary = re.sub(r'<[^>]+>', '', content)[:200] if content else final_title
            summary = re.sub(r'\s+', ' ', summary).strip()
            
            try:
                c.execute(
                    'INSERT OR IGNORE INTO gov_raw(id, site_name, source_url, page_url, title, publish_date, content, summary, date_rank) VALUES(?,?,?,?,?,?,?,?,?)',
                    (int_id, site_name, item['url'], item['url'], final_title, final_date, content, summary, date_rank)
                )
                if c.rowcount > 0:
                    inserted += 1
            except Exception as e:
                print(f'  [DB] {e}')
                skipped += 1
            time.sleep(0.3)
        
        elapsed = time.time() - start_time
        print(f'  Page {pg}: +{inserted} (skipped {skipped}, >3y {old_total}) [{elapsed:.0f}s]')
        
        if is_incremental:
            break
        
        # Navigate to next page
        next_link, total_pages = get_pagination_info(html)
        if not next_link:
            print(f'  Page {pg}: no next page')
            break
        
        # Resolve next page URL
        # For pg=1: current_url = .../gggs.htm, next_link = gggs/6.htm
        # Resolve: replace gggs.htm with gggs/6.htm in the directory
        base_dir = current_url.rsplit('/', 1)[0]
        if pg == 1:
            current_url = base_dir + '/' + next_link
        else:
            current_url = base_dir.rsplit('/', 1)[0] + '/' + next_link
        
        pg += 1
    
    conn.commit()
    # FTS rebuild
    try:
        c2 = conn.cursor()
        c2.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE site_name=?)", (site_name,))
        rows = c2.execute("SELECT id, title, site_name, summary FROM gov_raw WHERE site_name=?", (site_name,)).fetchall()
        for r in rows:
            c2.execute("INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES(?,?,?,?)", r)
        conn.commit()
        print(f'  FTS: {len(rows)} records rebuilt')
    except Exception as e:
        print(f'  FTS error: {e}')
    conn.close()
    elapsed = time.time() - start_time
    print(f'\nDone ({elapsed:.0f}s). Inserted {inserted}, Skipped {skipped}, Old {old_total}')

if __name__ == '__main__':
    main()