#!/usr/bin/env python3
"""
宁夏汇晟环保 - 项目公示 (nxhshb.com)
CMS: 508sys/JZ
列表API: nr.jsp?_np=0_884_{offset}  (offset=0,10)
详情: nd.jsp?id={id}
正文: div.richContent
日期: 发表时间：YYYY-MM-DD
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb, clean_html
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "宁夏汇晟环保-项目公示"
BASE_URL = "https://nxhshb.com"
LIST_TPL = "https://nxhshb.com/nr.jsp?_np=0_884_{offset}"
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
MAX_OFFSETS = [0, 10]
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except: return None

def parse_list(html):
    items = []
    # Find all nd.jsp?id=N links with titles
    seen = set()
    for m in re.finditer(r'<a[^>]*href="(?:https://nxhshb\.com/)?(nd\.jsp\?id=\d+)"[^>]*>(.*?)</a>', html, re.I|re.S):
        href = m.group(1)
        title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        if title and len(title) > 10 and href not in seen:
            seen.add(href)
            items.append((title, BASE_URL + '/' + href))
    return items

def fetch_detail(url):
    html = fetch(url)
    if not html: return "", "", ""
    
    title = ""
    m = re.search(r'<title>(.*?)\s*[-|]\s*.*?</title>', html, re.I|re.S)
    if m: title = m.group(1).strip()
    if not title:
        m = re.search(r'<div[^>]*class="[^"]*newsDetail[^"]*"[^>]*>\s*<h[1-4][^>]*>(.*?)</h[1-4]>', html, re.I|re.S)
        if m: title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    
    pub_date = ""
    for pat in [r'发表时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})',
                 r'发布时间[：:\s]*(\d{4}-\d{1,2}-\d{1,2})']:
        m = re.search(pat, html)
        if m:
            pub_date = m.group(1).replace(' ','')
            break
    
    content = ""
    idx = html.find('richContent')
    if idx > 0:
        div_start = html.rfind('<div', 0, idx)
        if div_start >= 0:
            depth = 0
            pos = div_start
            while pos < len(html):
                if html[pos:pos+4] == '<div' and html[pos+4] in ' >':
                    depth += 1
                    pos += 4
                elif html[pos:pos+6] == '</div>':
                    depth -= 1
                    if depth == 0:
                        inner = html[html.find('>',div_start)+1:pos]
                        if len(inner) > 50:
                            content = inner.strip()
                            content = clean_html(content)
                        break
                    pos += 6
                else:
                    pos += 1
    
    return title, pub_date, content

def main():
    incremental = len(sys.argv) > 1 and sys.argv[1] in ('1', '--incremental', '-i')
    print(f"爬取: {SITE_NAME}" + (" [增量]" if incremental else ""))
    
    all_list = []
    offsets = [0] if incremental else MAX_OFFSETS
    for offset in offsets:
        url = LIST_TPL.format(offset=offset)
        html = fetch(url)
        if not html or len(html) < 500:
            print(f"  offset={offset}: 无内容")
            break
        items = parse_list(html)
        print(f"  offset={offset}: {len(items)} 条")
        all_list.extend(items)
    print(f"  总计: {len(all_list)} 条")
    
    results = []
    for i, (title, url) in enumerate(all_list):
        print(f"  [{i+1}/{len(all_list)}] {title[:45]}...", end=" ", flush=True)
        dt, date, content = fetch_detail(url)
        if date and date < CUTOFF:
            print("过旧")
            if not incremental: continue
        if not content or len(content) < 50:
            print("无正文")
            continue
        
        final_title = dt or title
        summary = re.sub(r'<[^>]+>', '', content)[:200].strip()
        results.append({
            "site_name": SITE_NAME, "title": final_title, "url": url,
            "content": content, "summary": summary, "pub_date": date or ""
        })
        print(f"OK ({len(content)}字)")
        time.sleep(0.3)
    
    if results:
        push_to_searchdb(results, "nxhshb")
    print(f"完成! 共 {len(results)} 条")

if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"耗时: {time.time()-t0:.1f}s")