#!/usr/bin/env python3
"""太谷区-通知公告（生态环境局太谷分局）爬虫
站点: http://www.taigu.gov.cn
栏目: 晋中市生态环境局太谷分局-通知公告
分页: tzgg50jzssthjjtgfj_{N} (第1页: 无后缀), 40页×20条
CMS: PowerCMS (ASP.NET MVC)
"""
import re, sys, os, time, json, urllib.request, urllib.error, sqlite3
from datetime import datetime

DB_PATH = "/root/search.db"
BASE_URL = "http://www.taigu.gov.cn"
# Page 1 URL - long path to the column
PAGE1_PATH = "/zwgk/bmxxgkml/qtdwml/50jzssthjjtgfj/fdzdgknrsthjfj/zcwjsthjfj/tzgg50jzssthjjtgfj"
MAX_PAGES = 40
SITE_NAME = "太谷区-通知公告"

def fetch(url, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
                "Accept-Language": "zh-CN,zh;q=0.9",
            })
            resp = urllib.request.urlopen(req, timeout=30)
            return resp.read().decode("utf-8", errors="replace")
        except Exception as e:
            if i < retries - 1:
                time.sleep(2)
            else:
                print(f"  [ERROR] {url}: {e}", file=sys.stderr)
                return None

def resolve_url(rel_url, page_url=""):
    if rel_url.startswith("//"):
        return "https:" + rel_url
    if rel_url.startswith("http"):
        return rel_url
    if rel_url.startswith("/"):
        return BASE_URL + rel_url
    # Relative to page directory
    if page_url:
        page_dir = "/".join(page_url.split("/")[:-1])
        return page_dir + "/" + rel_url
    return BASE_URL + "/" + rel_url

def parse_list(html):
    items = []
    idx = html.find('class="infoList"')
    if idx < 0:
        return items
    end = html.find("</ul>", idx)
    section = html[idx:end]
    
    lis = re.findall(r'<li[^>]*>(.*?)</li>', section, re.DOTALL)
    for li in lis:
        href_m = re.search(r'href=[\"\']([^\"\']+)[\"\']', li)
        if not href_m:
            continue
        url = href_m.group(1)
        
        # Title from title attribute (parse out "标题：" prefix)
        title_m = re.search(r'title=[\"\']([^\"\']+)[\"\']', li)
        title = ""
        if title_m:
            raw_title = title_m.group(1)
            # Format: "标题：TITLE\n点击数：N\n发表时间：DATE"
            t = re.sub(r'标题[：:]\s*', '', raw_title)
            t = re.split(r'\n', t)[0].strip()
            title = t
        if not title:
            # Fallback to display text
            a_m = re.search(r'<a[^>]*>(.*?)</a>', li, re.DOTALL)
            if a_m:
                title = re.sub(r'<[^>]+>', '', a_m.group(1)).strip()
        
        # Date from <span class="date">
        date_m = re.search(r'<span[^>]*class=\"date\"[^>]*>(.*?)</span>', li)
        date_str = date_m.group(1).strip() if date_m else ""
        dm = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', date_str)
        publish_date = dm.group(1) if dm else ""
        
        items.append({
            "page_url": url,
            "title": title,
            "date": publish_date,
        })
    return items

def parse_detail(html, url):
    title = ""
    title_m = re.search(r'<h2[^>]*class="title"[^>]*>(.*?)</h2>', html, re.DOTALL)
    if title_m:
        title = re.sub(r'<[^>]+>', '', title_m.group(1)).strip()
    
    # Publish date
    publish_date = ""
    pub_m = re.search(r'发布时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})', html)
    if pub_m:
        publish_date = pub_m.group(1)
    if not publish_date:
        dm = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', url)
        if dm:
            publish_date = dm.group(1)
    
    # Content from conTxt div
    content_html = ""
    c = re.search(r'class="conTxt"[^>]*>(.*?)</div>', html, re.DOTALL)
    if c:
        content_html = c.group(1).strip()
    
    # Build content: tables as raw HTML, p as text
    tables = re.findall(r'<table[^>]*>.*?</table>', content_html, re.DOTALL)
    no_table_html = re.sub(r'<table[^>]*>.*?</table>', '', content_html, flags=re.DOTALL)
    no_table_html = re.sub(r'</?tbody[^>]*>', '', no_table_html)
    
    parts = []
    for p in re.findall(r'<p[^>]*>(.*?)</p>', no_table_html, re.DOTALL):
        text = re.sub(r'<[^>]+>', '', p).strip()
        if text:
            parts.append(text)
    for t in tables:
        parts.append(t)
    
    # Images
    for img in re.findall(r'<img[^>]+src=[\"\']([^\"\']+)[\"\'][^>]*>', content_html):
        img_url = resolve_url(img, url)
        alt_m = re.search(r'alt=[\"\']([^\"\']*)[\"\']', content_html[content_html.find(img)-100:content_html.find(img)+len(img)+50] if content_html.find(img) >= 100 else html)
        alt = alt_m.group(1) if alt_m else ""
        parts.append(f"![{alt}]({img_url})")
    
    # Attachments
    attachments = []
    seen = set()
    for m in re.finditer(r'href=[\"\']([^\"\']*\.(?:pdf|doc|docx|xls|xlsx|zip|rar))[\"\']', html, re.I):
        att_url = resolve_url(m.group(1), url)
        if att_url not in seen:
            seen.add(att_url)
            attachments.append(att_url)
    
    rendered = "\n\n".join(parts)
    # Clean up HTML entities
    rendered = rendered.replace("&nbsp;", " ").replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&")
    rendered = re.sub(r' {2,}', ' ', rendered)  # collapse multiple spaces
    if attachments:
        att_lines = []
        for att_url in attachments:
            att_title = att_url.split("/")[-1]
            att_lines.append(f"- [{att_title}]({att_url})")
        rendered += "\n\n**附件：**\n" + "\n".join(att_lines)
    
    return {
        "title": title or url.split("/")[-1],
        "publish_date": publish_date,
        "content_rendered": rendered,
        "attachments": json.dumps(attachments, ensure_ascii=False),
    }

def save_to_db(items, details):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for item in items:
        pu = item["page_url"]
        det = details.get(pu, {})
        # Ensure URL is absolute
        full_url = pu if pu.startswith("http") else resolve_url(pu)
        c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=? AND site_name=?", (full_url, SITE_NAME))
        if c.fetchone()[0] > 0:
            continue
        content = det.get("content_rendered", "")
        c.execute("""INSERT OR REPLACE INTO gov_raw (page_url, title, site_name, publish_date, content, attachments, category, date_rank, script_name) VALUES (?,?,?,?,?,?,?,?, 'crawl_taigu.py')""", (
            full_url,
            det.get("title", item.get("title", "")),
            SITE_NAME,
            det.get("publish_date", item.get("date", "")),
            content,
            det.get("attachments", "[]"),
            "通知公告",
            int(datetime.now().timestamp()),
        ))
        inserted += 1
    conn.commit()
    conn.close()
    return inserted

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--incremental", action="store_true")
    parser.add_argument("--max-pages", type=int, default=5, help="Max pages to crawl")
    args = parser.parse_args()
    
    pages_to_crawl = min(args.max_pages, MAX_PAGES)
    all_items = []
    for page_num in range(1, pages_to_crawl + 1):
        if page_num == 1:
            url = f"{BASE_URL}{PAGE1_PATH}"
        else:
            url = f"{BASE_URL}{PAGE1_PATH}_{page_num}"
        print(f"[LIST] Page {page_num}/{pages_to_crawl}...", end=" ", flush=True)
        html = fetch(url)
        if not html:
            print("FAILED")
            continue
        items = parse_list(html)
        print(f"{len(items)} items")
        all_items.extend(items)
    
    print(f"\nTotal items: {len(all_items)}")
    
    details = {}
    for i, item in enumerate(all_items):
        pu = item["page_url"]
        full_url = pu if pu.startswith("http") else resolve_url(pu)
        print(f"  [{i+1}/{len(all_items)}] {item['title'][:40]}...", end=" ", flush=True)
        html = fetch(full_url)
        if not html:
            print("SKIP")
            continue
        det = parse_detail(html, full_url)
        details[pu] = det
        print(f"OK ({len(det.get('content_rendered',''))} chars)")
        time.sleep(1.5)  # WAF rate limit: ≤1 req/s
    
    inserted = save_to_db(all_items, details)
    print(f"\nInserted: {inserted} new items")
    print(f"Skipped (existing): {len(all_items) - inserted}")

if __name__ == "__main__":
    main()