#!/usr/bin/env python3
"""
江苏国正检测有限公司 - 新闻动态
http://www.gztesting.com/list_news.aspx?cid=216
ASP.NET WebForms + AspNetPager 翻页
"""
import sys, os, re, time, json
sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import fetch_page, clean_html
import sqlite3, requests
import os

SITE_NAME = "江苏国正检测-新闻动态"
BASE = "http://www.gztesting.com"
LIST_URL = f"{BASE}/list_news.aspx?cid=216"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
THREE_YEARS_AGO = "2023-06-01"
TOTAL_PAGES = 3

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}

def fetch_list_page(page_no, sess=None):
    """Get a page. Page 1 via GET, pages 2+ via ASP.NET postback."""
    if sess is None:
        sess = requests.Session()
        sess.headers.update(HEADERS)
    
    if page_no == 1:
        r = sess.get(LIST_URL, timeout=30)
        return r.text if r.status_code == 200 else None
    
    # For page 2+, first ensure we have page 1's viewstate
    # Extract hidden fields (exclude non-hidden inputs like text/button)
    html_for_vs = getattr(sess, '_last_html', None)
    if not html_for_vs:
        r = sess.get(LIST_URL, timeout=30)
        if r.status_code != 200:
            return None
        html_for_vs = r.text
    
    data = {}
    for m in re.finditer(
        r'<input\s+type="hidden"\s+name="([^"]+)"[^>]*value="([^"]*)"',
        html_for_vs
    ):
        data[m.group(1)] = m.group(2)
    
    data["__EVENTTARGET"] = "_ctl0$ContentPlaceHolder1$AspNetPager1"
    data["__EVENTARGUMENT"] = str(page_no)
    
    r = sess.post(LIST_URL, data=data, timeout=30)
    if r.status_code == 200:
        sess._last_html = r.text
        return r.text
    return None

def parse_list(html):
    """从列表页提取 (title, url, date_str, summary)"""
    items = []
    for m in re.finditer(r'<div class="new">(.*?)</div>', html, re.DOTALL):
        item_html = m.group(1)
        # URL + Title
        link_match = re.search(r'href="(/show\.aspx\?id=\d+)"[^>]*class="tit"[^>]*>(.*?)</a>', item_html, re.DOTALL)
        if not link_match:
            continue
        href = BASE + link_match.group(1)
        title = re.sub(r'<[^>]+>', '', link_match.group(2)).strip()
        
        # Summary (from <p class="con">)
        summary = ""
        con_match = re.search(r'<p class="con">(.*?)</p>', item_html, re.DOTALL)
        if con_match:
            summary = re.sub(r'<[^>]+>', '', con_match.group(1)).strip()
            summary = re.sub(r'\s+', ' ', summary)[:300]
        
        items.append((title, href, "", summary))
    return items

def fetch_detail(url):
    """获取详情页正文"""
    html = fetch_page(url, encoding='utf-8', timeout=20)
    if not html:
        return None, None, None
    
    # 标题
    title = ""
    m = re.search(r'<h3[^>]*id="title_info"[^>]*>(.*?)</h3>', html, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    
    # 正文
    content = ""
    m = re.search(r'<div class="detail">(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    
    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = clean_html(content)
    
    # 发布日期（本网站文章页没显示日期，留空）
    pub_date = ""
    
    return title or None, content or None, pub_date or None

def insert_to_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA busy_timeout=60000")
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            db.execute("INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?,?,?,?,?,?,?,?,?,?)", (
                item["site_name"][:200], item["url"], item["url"],
                item["title"][:500], item["pub_date"][:10],
                item["summary"][:500], item["content"],
                "active", "", item["tags"][:100],
            ))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    db.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (SITE_NAME,))
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")

def main():
    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"{'='*50}")

    all_items = []
    seen_urls = set()

    MAX_PAGES = TOTAL_PAGES
    if len(sys.argv) > 1 and sys.argv[1] == "inc":
        MAX_PAGES = 1
        print("📋 增量模式：仅第1页")

    sess = requests.Session()
    sess.headers.update(HEADERS)
    
    for page_no in range(1, MAX_PAGES + 1):
        print(f"  📄 第 {page_no}/{MAX_PAGES} 页...", end=" ", flush=True)
        html = fetch_list_page(page_no, sess)
        if not html:
            print("❌ 无响应")
            break
        items = parse_list(html)
        if not items:
            print("0 条")
            break
        print(f"✅ {len(items)} 条")

        for title, href, date_str, summary in items:
            if href in seen_urls:
                continue
            seen_urls.add(href)
            
            print(f"    {title[:55]}...", end=" ", flush=True)
            
            # 获取详情页
            content = ""
            dt, content, pub_date = fetch_detail(href)
            if dt:
                title = dt
            if pub_date:
                date_str = pub_date
            
            if not summary and content:
                summary = re.sub(r'<[^>]+>', ' ', content).strip()[:300]
                summary = re.sub(r'\s+', ' ', summary)

            all_items.append({
                "site_name": SITE_NAME,
                "title": title,
                "url": href,
                "content": content or "",
                "pub_date": date_str or "",
                "summary": summary or title[:300],
                "tags": "环评验收",
            })
            print(f"✅ {len(content or '')}字")
            time.sleep(0.5)

        time.sleep(1)

    if all_items:
        # Clean old + re-insert
        db = sqlite3.connect(SEARCH_DB, timeout=60)
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)         db.execute("DELETE FROM gov_raw WHERE site_name=?", (SITE_NAME,))
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)         db.execute("DELETE FROM gov_search WHERE site_name=?", (SITE_NAME,))
        db.commit()
        db.close()
        insert_to_db(all_items)
    print(f"\n✅ 完成! 共 {len(all_items)} 条")

if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")