from urllib.parse import urlencode
#!/usr/bin/env python3
import os
"""泰和县-建设项目环境影响评价 (API动态加载)"""
import requests, re, sqlite3, time, json
from datetime import datetime, timedelta

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
NAME = "泰和县"
BASE = "https://www.jxth.gov.cn"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
H = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url):
    for retry in range(2):
        try:
            r = requests.get(url, headers=H, timeout=30)
            if r.status_code == 200: r.encoding = "utf-8"; return r.text
        except: time.sleep(2)
    return None

def fetch_api(page):
    # QC20260926 重写：原实现请求失败即返回 None → 上层打印"第1页无数据"
    url = f"{BASE}/api-ajax_list-{page}.html"
    body = [("ajax_type[]", "9_xxgk"), ("ajax_type[]", "167445"), ("ajax_type[]", "9"),
            ("ajax_type[]", "xxgk"), ("ajax_type[]", "Y-m-d"), ("ajax_type[]", "50"),
            ("ajax_type[]", "20"), ("ajax_type[]", "is_top DESC"),
            ("ajax_type[]", "displayorder DESC"), ("ajax_type[]", "inputtime DESC"),
            ("ajax_type[]", ""), ("is_ds", "1")]
    hh = dict(H)
    hh["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"  # QC20260926
    hh.update({"X-Requested-With": "XMLHttpRequest",
               "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
               "Referer": BASE + "/"})
    for _try in range(3):
        try:
            r = requests.post(url, data=urlencode(body), headers=hh, timeout=30)
            if r.status_code == 200:
                r.encoding = 'utf-8'
                return json.loads(r.text)
        except Exception as e:
            print('  [RETRY %d/3] %s' % (_try + 1, e), flush=True)
        time.sleep(3)
    return None


def extract_content(html):
    patterns = [
        'class="xxgk_content"', 'class="zwxxgk_box"',
        'class="zwxxgk_bd"', 'class="content"',
        'id="zoom"', 'class="nr_content"', 'class="articlecon"'
    ]
    for p in patterns:
        i = html.find(p)
        if i < 0: continue
        ds = html.rfind("<div", 0, i)
        if ds < 0: continue
        s = html[ds:]; d = 0
        for j in range(len(s)):
            if s[j:j+4] == "<div" and (j+4 >= len(s) or s[j+4] in " >\n\r\t"): d += 1
            elif s[j:j+6] == "</div>":
                d -= 1
                if d == 0:
                    gt = s.find(">", 0, j)
                    c = s[gt+1:j] if gt > 0 else s[7:j]
                    c = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', c, flags=re.DOTALL|re.I)
                    c = re.sub(r'\s*(style|class|align|lang|dir)="[^"]*"', '', c)
                    c = re.sub(r'\n\s*\n+', '\n', c)
                    if c.strip(): return c.strip()
    return ""

def extract(html):
    t = (re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html) or [None, ""])[1]
    if not t:
        t = re.sub(r'^泰和县政府信息公开-', '', (re.search(r'<title>(.*?)<', html) or [None,""])[1]).strip()
    pd = (re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{2}-\d{2})', html) or [None, ""])[1]
    if not pd:
        pd = (re.findall(r'(\d{4}-\d{2}-\d{2})', html) or [""])[0]
    c = extract_content(html)
    return t, pd, c

def run(max_pages=5):
    print(f"\n{'='*50}\n{NAME}\n{'='*50}")
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute("PRAGMA busy_timeout=60000")
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)
    # conn.execute("DELETE FROM gov_raw WHERE site_name=?", (NAME,))  # QC20260926 已去除
    conn.commit()
    print("清理旧数据")

    items = []
    for pg in range(1, max_pages+1):
        d = fetch_api(pg)
        if not d or not d.get("data"):
            print(f"  -> 第{pg}页无数据"); break
        data = d["data"]
        total = d.get("total", 0)
        print(f"第{pg}页: {len(data)} 条 (共{total}条)")
        for item in data:
            inp = item.get("inputtime", "")
            if inp and inp < CUTOFF:
                print(f"  {item['title'][:40]}... ({inp}) 超过3年")
                continue
            items.append({
                "url": item["url"],
                "title": item["title"],
                "date": inp,
                "summary": item.get("description", "")
            })
        if len(data) < 20:
            print("  -> 已到最后一页")
            break

    print(f"\n共 {len(items)} 条")
    new = skip = no_body = 0
    for i, item in enumerate(items):
        title = item["title"]
        date = item["date"]
        html = fetch(item["url"])
        if not html:
            print(f"  ? 取不到详情: {title[:40]}..."); skip += 1; continue
        dt, detail_date, content = extract(html)
        if dt: title = dt
        if detail_date: date = detail_date
        dr = 0
        try: dr = int(date.replace("-",""))
        except: pass
        if not content:
            no_body += 1
            content = item.get("summary", "")
            if not content:
                print(f"  ? 空正文: {title[:40]}...")
        try:
            conn.execute("""INSERT OR IGNORE INTO gov_raw(site_name,source_url,page_url,title,publish_date,content,summary,date_rank,category)
                VALUES(?,?,?,?,?,?,?,?,?)""",
                (NAME, item["url"], item["url"], title, date, content, title, dr, "政府公告"))
            if conn.total_changes: new+=1
            else: skip+=1
        except Exception as e: print(f"  ? DB: {e}"); skip+=1
        if i%10==0: conn.commit(); print(f"  ...{i}/{len(items)}")
        time.sleep(0.3)
    conn.commit()
    try:
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)         conn.execute("DELETE FROM gov_search WHERE site_name=?",(NAME,))
        conn.execute("""INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary)
            SELECT rowid,title,site_name,summary FROM gov_raw WHERE site_name=?""",(NAME,))
        conn.commit()
    except Exception as e: print(f"? FTS: {e}")
    conn.close()
    print(f"\n{NAME}: 新增{new}, 空正文{no_body}, 跳过{skip}")

if __name__=="__main__":
    import sys; mp=int(sys.argv[1]) if len(sys.argv)>1 else 5; run(mp)