#!/usr/bin/env python3
"""
crawl_linqing_tzgg.py - 临清市人民政府-通知公告
"""
import requests, re, sqlite3, sys, time

BASE_URL = "http://www.linqing.gov.cn"
LIST_URL = BASE_URL + "/channel_t_269_15185/"
CDN_IP = "120.224.28.134"
DB_PATH = "/root/search.db"
SITE_NAME = "临清市人民政府-通知公告"
SLEEP = 0.5

session = requests.Session()
session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"})

def get_page(url):
    real_url = url.replace("www.linqing.gov.cn", CDN_IP)
    try:
        resp = session.get(real_url, headers={"Host": "www.linqing.gov.cn"}, timeout=30)
        resp.encoding = "utf-8"
        return resp
    except Exception as e:
        print(f"  [NET ERR] {e}")
        return None

def parse_list(html):
    items = []
    for m in re.finditer(r'<a[^>]*title="([^"]*)"[^>]*href="([^"]*doc_[a-f0-9]+\.html)"[^>]*>', html):
        title = m.group(1).strip()
        href = m.group(2)
        ctx = html[m.start():m.start()+500]
        date_m = re.search(r"发布时间[：:](\d{4}-\d{2}-\d{2})", ctx)
        date_str = date_m.group(1) if date_m else ""
        url = BASE_URL + href if href.startswith("/") else href
        if "linqing.gov.cn" in url:
            items.append({"title": title, "url": url, "date": date_str})
    return items

def parse_detail(html):
    result = {"title": "", "date": "", "content": ""}
    m_title = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
    if m_title: result["title"] = m_title.group(1).strip()
    m_date = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', html)
    if m_date:
        dm = re.search(r"\d{4}-\d{2}-\d{2}", m_date.group(1))
        if dm: result["date"] = dm.group()
    m_content = re.search(r'<div\s+class="Details_content"[^>]*>(.*?)</div>', html, re.DOTALL)
    if not m_content: return result
    html_content = m_content.group(1)
    parts = []; pos = 0
    while pos < len(html_content):
        p = re.match(r"<p[^>]*>(.*?)</p>", html_content[pos:], re.DOTALL)
        if p:
            t = re.sub(r"<[^>]+>", "", p.group(1)).strip()
            t = re.sub(r"\s+", " ", t).strip()
            t = t.replace("&nbsp;"," ").replace("&mdash;","—").replace("&amp;","&")
            if t: parts.append(t)
            pos += p.end(); continue
        img = re.match(r'<img[^>]*src="([^"]+)"[^>]*>', html_content[pos:], re.DOTALL)
        if img:
            s = img.group(1)
            if not s.startswith("http"): s = BASE_URL + s
            parts.append(f"![图片]({s})")
            pos += img.end(); continue
        tag = re.match(r"<[^>]+>", html_content[pos:])
        if tag: pos += tag.end(); continue
        ws = re.match(r"\s+", html_content[pos:])
        if ws: pos += ws.end(); continue
        pos += 1
    result["content"] = "\n\n".join(parts).strip()
    return result

def save(url, title, date, content):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    try:
        c.execute("SELECT id FROM gov_raw WHERE page_url=? AND site_name=?", (url, SITE_NAME))
        if c.fetchone(): conn.close(); return "skip"
        summary = content[:200]
        has_table = 1 if "| ---" in content else 0
        c.execute("""INSERT OR REPLACE INTO gov_raw (page_url, title, publish_date, site_name, content, attachments, summary, has_table, script_name) VALUES (?,?,?,?,?,?,?,?, 'crawl_linqing_tzgg.py')""", (url, title, date, SITE_NAME, content, "", summary, has_table))
        conn.commit(); conn.close(); return "new"
    except Exception as e:
        conn.close(); return f"err:{e}"

def crawl(pages=1):
    new=skip=0
    resp = get_page(LIST_URL)
    if not resp: return new,skip
    items = parse_list(resp.text)
    print(f"[LIST] {len(items)} items")
    for item in items:
        resp2 = get_page(item["url"])
        if not resp2: skip+=1; continue
        d = parse_detail(resp2.text)
        if not d["content"]: skip+=1; print(f"  [EMPTY] {item['title'][:50]}"); continue
        r = save(item["url"], d.get("title") or item["title"], d.get("date") or item["date"], d["content"])
        if r=="new": new+=1; print(f"  [OK] {d['title'][:60]}")
        elif r=="skip": skip+=1; print(f"  [SKIP] {item['title'][:50]}")
        time.sleep(SLEEP)
    return new,skip

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=1)
    args = parser.parse_args()
    n,s = crawl(args.pages)
    print(f"\n[RESULT] {SITE_NAME}: {n} new, {s} skip")