#!/usr/bin/env python3
"""台儿庄区-通知公告爬虫
站点: http://www.tez.gov.cn/xwzx/tzgg/
栏目: 通知公告
分页: index_N.html (第1页: index.html), 52页×15条
CMS: Custom
"""
import re, sys, os, time, json, urllib.request, urllib.error, sqlite3
from datetime import datetime

DB_PATH = "/root/search.db"
BASE_URL = "http://www.tez.gov.cn"
LIST_PATH = "/xwzx/tzgg"
MAX_PAGES = 52
SITE_NAME = "台儿庄区-通知公告"

def fetch(url, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
                "Accept-Language": "zh-CN,zh;q=0.9",
            })
            resp = urllib.request.urlopen(req, timeout=30)
            return resp.read().decode("utf-8", errors="replace")
        except Exception as e:
            if i < retries - 1:
                time.sleep(2)
            else:
                print(f"  [ERROR] {url}: {e}", file=sys.stderr)
                return None

def parse_list(html):
    items = []
    idx = html.find('class="news-list"')
    if idx < 0:
        return items
    end = html.find("</ul>", idx)
    section = html[idx:end]

    lis = re.findall(r'<li[^>]*>(.*?)</li>', section, re.DOTALL)
    for li in lis:
        href_m = re.search(r'href=[\"\']([^\"\']+)[\"\']', li)
        if not href_m:
            continue
        url = href_m.group(1)
        if url.startswith("./"):
            url = f"{BASE_URL}{LIST_PATH}/{url[2:]}"
        elif not url.startswith("http"):
            url = BASE_URL + url
        
        # Title from <a> tag content, strip &middot; prefix
        a_m = re.search(r'<a[^>]*>(.*?)</a>', li, re.DOTALL)
        title = ""
        if a_m:
            raw_title = a_m.group(1)
            # Remove &middot;&nbsp; prefix and HTML entities
            title = raw_title.replace("&middot;", "").replace("&nbsp;", "").strip()
            title = re.sub(r'\s+', ' ', title)
        
        # Date from <span>
        date_m = re.search(r'<span>(.*?)</span>', li)
        date_str = date_m.group(1).strip() if date_m else ""
        dm = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', date_str)
        publish_date = dm.group(1) if dm else ""

        items.append({
            "page_url": url,
            "title": title,
            "date": publish_date,
        })
    return items

def parse_detail(html, url):
    title = ""
    title_m = re.search(r'<h2>(.*?)</h2>', html, re.DOTALL)
    if title_m:
        title = title_m.group(1).strip()
    
    # Publish date
    publish_date = ""
    date_m = re.search(r'时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})', html)
    if date_m:
        publish_date = date_m.group(1)
    if not publish_date:
        dm2 = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', url)
        if dm2:
            publish_date = dm2.group(1)
    
    # Content - find the view/TRS_UEDITOR div
    content_html = ""
    c = re.search(r'class="view[^"]*trs_editor_view[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
    if c:
        content_html = c.group(1).strip()
    # Fallback: any div with TRS_UEDITOR class
    if not content_html:
        c2 = re.search(r'class="[^"]*TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
        if c2:
            content_html = c2.group(1).strip()
    # Fallback: inside news-cont
    if not content_html:
        c3 = re.search(r'class="news-cont"[^>]*>(.*?)</div>', html, re.DOTALL)
        if c3:
            c4 = re.search(r'<p>(.*?)</p>\s*$', c3.group(1), re.DOTALL)
            if c4:
                content_html = c4.group(1).strip()
    
    # Build content: tables as raw HTML, p as text
    tables = re.findall(r'<table[^>]*>.*?</table>', content_html, re.DOTALL)
    no_table_html = re.sub(r'<table[^>]*>.*?</table>', '', content_html, flags=re.DOTALL)
    no_table_html = re.sub(r'</?tbody[^>]*>', '', no_table_html)
    
    parts = []
    for p in re.findall(r'<p[^>]*>(.*?)</p>', no_table_html, re.DOTALL):
        text = re.sub(r'<[^>]+>', '', p).strip()
        if text:
            parts.append(text)
    for t in tables:
        parts.append(t)
    
    # Helper to resolve relative URLs against page URL
    def resolve_url(rel_url):
        if rel_url.startswith("//"):
            return "https:" + rel_url
        if rel_url.startswith("http"):
            return rel_url
        # Relative to the current page's directory
        page_dir = "/".join(url.split("/")[:-1])
        if rel_url.startswith("./"):
            return page_dir + "/" + rel_url[2:]
        if rel_url.startswith("/"):
            return BASE_URL + rel_url
        return page_dir + "/" + rel_url
    
    # Images
    for img in re.findall(r'<img[^>]+src=[\"\']([^\"\']+)[\"\'][^>]*>', content_html):
        img_url = resolve_url(img)
        alt_m = re.search(r'alt=[\"\']([^\"\']*)[\"\']', content_html[content_html.find(img)-100:content_html.find(img)+len(img)+50] if content_html.find(img) >= 100 else html)
        alt = alt_m.group(1) if alt_m else ""
        parts.append(f"![{alt}]({img_url})")
    
    # Attachments
    attachments = []
    seen = set()
    for m in re.finditer(r'href=[\"\']([^\"\']*\.(?:pdf|doc|docx|xls|xlsx|zip|rar))[\"\']', html, re.I):
        att_url = resolve_url(m.group(1))
        if att_url not in seen:
            seen.add(att_url)
            attachments.append(att_url)
    
    rendered = "\n\n".join(parts)
    if attachments:
        att_lines = []
        for att_url in attachments:
            att_title = att_url.split("/")[-1]
            att_lines.append(f"- [{att_title}]({att_url})")
        rendered += "\n\n**附件：**\n" + "\n".join(att_lines)
    
    return {
        "title": title or url.split("/")[-1].replace(".html", ""),
        "publish_date": publish_date,
        "content_rendered": rendered,
        "attachments": json.dumps(attachments, ensure_ascii=False),
    }

def save_to_db(items, details):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for item in items:
        pu = item["page_url"]
        det = details.get(pu, {})
        c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=? AND site_name=?", (pu, SITE_NAME))
        if c.fetchone()[0] > 0:
            continue
        content = det.get("content_rendered", "")
        c.execute("""INSERT OR REPLACE INTO gov_raw 
            (page_url, title, site_name, publish_date, content, attachments, category, date_rank)
            VALUES (?,?,?,?,?,?,?,?)""", (
            pu,
            det.get("title", item.get("title", "")),
            SITE_NAME,
            det.get("publish_date", item.get("date", "")),
            content,
            det.get("attachments", "[]"),
            "通知公告",
            int(datetime.now().timestamp()),
        ))
        inserted += 1
    conn.commit()
    conn.close()
    return inserted

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--incremental", action="store_true")
    parser.add_argument("--max-pages", type=int, default=5, help="Max pages to crawl")
    args = parser.parse_args()

    pages_to_crawl = min(args.max_pages, MAX_PAGES)
    all_items = []
    for page_num in range(1, pages_to_crawl + 1):
        if page_num == 1:
            url = f"{BASE_URL}{LIST_PATH}/"
        else:
            url = f"{BASE_URL}{LIST_PATH}/index_{page_num}.html"
        print(f"[LIST] Page {page_num}/{pages_to_crawl}...", end=" ", flush=True)
        html = fetch(url)
        if not html:
            print("FAILED")
            continue
        items = parse_list(html)
        print(f"{len(items)} items")
        all_items.extend(items)

    print(f"\nTotal items: {len(all_items)}")

    details = {}
    for i, item in enumerate(all_items):
        pu = item["page_url"]
        print(f"  [{i+1}/{len(all_items)}] {item['title'][:40]}...", end=" ", flush=True)
        html = fetch(pu)
        if not html:
            print("SKIP")
            continue
        det = parse_detail(html, pu)
        details[pu] = det
        print(f"OK ({len(det.get('content_rendered',''))} chars)")

    inserted = save_to_db(all_items, details)
    print(f"\nInserted: {inserted} new items")
    print(f"Skipped (existing): {len(all_items) - inserted}")

if __name__ == "__main__":
    main()