#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
浙江政务服务网-舟山市岱山县-建设项目环境影响评价信息公示
http://zsds.zjzwfw.gov.cn/col/col1460404/index.html
"""
import re, sys, os, json, time, requests

DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
SITE_NAME = "岱山县-建设项目环评信息公示"
GROUP = "岱山"
MAX_PAGES = 5

LIST_URL = "http://zsds.zjzwfw.gov.cn/col/col1460404/index.html"


def fetch_page(url):
    r = requests.get(url, headers=HEADERS, timeout=30, allow_redirects=True)
    r.encoding = "utf-8"
    return r.text


def extract_list_items(html):
    """Parse list page: <a href='/art/...' class='bt_link' title='TITLE'>TITLE</a> with date in div below."""
    items = []
    # Match each article block - look for a tags with bt_link class inside tr
    # Find all bt_link entries with title attrs, extract href and title separately
    items_map = {}
    for m in re.finditer(r"""class='bt_link'\s+title='([^']+)'""", html):
        title = m.group(1).strip()
        items_map[m.start()] = {"title": title, "url": "", "date": ""}
    
    # Find corresponding href for each item - look backward from bt_link
    for pos in sorted(items_map.keys()):
        before = html[max(0, pos-200):pos]
        href_m = re.search(r"href='(/art/\d+/\d+/\d+/art_\d+_\d+\.html)'", before)
        if href_m:
            items_map[pos]["url"] = "http://zsds.zjzwfw.gov.cn" + href_m.group(1)
    
    # Build items list
    items = [v for v in items_map.values() if v["url"]]

    # Extract dates for each item - find the date divs
    date_pattern = r"""<div style="color: #B4B4B4;font-size: 12px;margin-right:2px;">(\d{4}-\d{2}-\d{2})</div>"""
    dates = re.findall(date_pattern, html)
    
    # Assign dates to items
    for i, item in enumerate(items):
        if i < len(dates):
            item["date"] = dates[i]
    
    return items


def extract_content(html, source_url):
    """Extract title, content, date from detail page."""
    # Title from the banner table
    title = ""
    m = re.search(r"style='color:#fff;font-size:30px;height:40px'>(.*?)</td>", html, re.DOTALL)
    if m:
        title = m.group(1).strip()
    else:
        m = re.search(r"<title>(.*?)</title>", html, re.DOTALL)
        if m:
            t = m.group(1).strip()
            t = re.sub(r'\s*[-—|]\s*.*$', '', t)
            title = t.strip()

    # Date
    pub_date = ""
    m = re.search(r"style='font-size:12px; color:#999;'[^>]*>([^<]+)</td>", html)
    if m:
        pub_date = m.group(1).strip()
    # Better date pattern
    if not pub_date:
        m = re.search(r"font-size:12px; color:#999;[^>]*>(\d{4}-\d{2}-\d{2})", html)
        if m:
            pub_date = m.group(1).strip()

    # Content from bt_content div > zoom div
    content = ""
    attachments = []
    content_html = ""
    
    m = re.search(r"""<td class="bt_content"><div id="zoom"[^>]*>(.*?)</div>\s*</td>""", html, re.DOTALL)
    if m:
        content_html = m.group(1)
    else:
        # Try alternative
        m = re.search(r"""class="bt_content">(.*?)</td>""", html, re.DOTALL)
        if m:
            content_html = m.group(1)

    if content_html:
        # Extract attachments (PDF/doc links)
        att_pattern = r"""<a[^>]*href=["']([^"']*\.(?:pdf|doc|docx|xls|xlsx|rar|zip))["'][^>]*>(.*?)</a>"""
        for a_tag in re.finditer(att_pattern, content_html, re.I | re.DOTALL):
            link_url = a_tag.group(1)
            link_text = re.sub(r"<[^>]+>", "", a_tag.group(2)).strip()
            if not link_url.startswith("http"):
                link_url = "http://zsds.zjzwfw.gov.cn" + link_url if link_url.startswith("/") else link_url
            attachments.append({"name": link_text or link_url.split("/")[-1], "url": link_url})

        # Extract image references
        for img_url in re.findall(r'<img[^>]*src="([^"]+)"[^>]*>', content_html):
            if not img_url.startswith("http"):
                img_url = "http://zsds.zjzwfw.gov.cn" + img_url if img_url.startswith("/") else img_url
            attachments.append({"name": img_url.split("/")[-1], "url": img_url})

        # Parse paragraphs and tables
        parts = []
        # Find all p and table elements
        for elem in re.finditer(r'<p[^>]*>(.*?)</p>|<table[^>]*>(.*?)</table>', content_html, re.DOTALL | re.I):
            tag = elem.group(0)
            if tag.startswith("<table"):
                parts.append(tag.group(0))
            else:
                # Paragraph
                p_text = elem.group(1)
                p_text = re.sub(r'<[^>]+>', '', p_text)
                p_text = re.sub(r'&[nN][bB][sS][pP];', ' ', p_text)
                p_text = re.sub(r'&nbsp;', ' ', p_text)
                p_text = p_text.strip()
                if p_text:
                    parts.append(p_text)
        content = "\n\n".join(parts)

    # Fallback for empty content
    if not content or len(content.strip()) < 20:
        text = re.sub(r'<[^>]+>', ' ', content_html)
        text = re.sub(r'\s+', ' ', text).strip()
        if text and len(text) > 20:
            content = text

    return {
        "title": title,
        "content": content,
        "date": pub_date,
        "url": source_url,
        "attachments": attachments,
        "site_name": SITE_NAME,
        "group": GROUP,
    }


def crawl_test(url=None):
    """Test a single article."""
    if not url:
        url = "http://zsds.zjzwfw.gov.cn/art/2023/11/30/art_1460404_22707.html"
    print("=== Testing: %s ===" % url)
    html = fetch_page(url)
    result = extract_content(html, url)
    for k, v in result.items():
        if k == "attachments":
            print("%s: %s" % (k, v))
        elif k == "content":
            print("%s (%d chars):" % (k, len(v)))
            print(v[:800])
            print("...")
        else:
            print("%s: %s" % (k, v))
    return result


def crawl(test_mode=False, max_pages=MAX_PAGES):
    """Crawl list pages and detail pages."""
    import sqlite3

    # Step 1: get list items
    print("[%s] Fetching list..." % SITE_NAME)
    html = fetch_page(LIST_URL)
    items = extract_list_items(html)
    print("  Found %d items" % len(items))

    # Limit to requested pages
    if max_pages <= 1 or len(items) <= 9:
        pass  # Single page only
    else:
        max_items = max_pages * len(items)
        items = items[:max_items]

    if test_mode:
        for item in items[:3]:
            print("\n=== Detail: %s ===" % item["title"][:40])
            html = fetch_page(item["url"])
            result = extract_content(html, item["url"])
            print("  Title: %s" % result["title"])
            print("  Date: %s" % result["date"])
            print("  Content (%d chars): %s" % (len(result["content"]), result["content"][:200]))
            print("  Attachments: %d" % len(result["attachments"]))
        return

    # Step 2: save to database
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()

    inserted = 0
    existing = 0
    for item in items:
        c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
        if c.fetchone():
            existing += 1
            continue

        html = fetch_page(item["url"])
        result = extract_content(html, item["url"])
        attachments_json = json.dumps(result["attachments"], ensure_ascii=False) if result["attachments"] else ""

        c.execute(
            """INSERT INTO gov_raw (title, content, summary, publish_date, page_url, site_name, attachments, group_name, source_url)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (
                result["title"],
                result["content"],
                (result["title"] + " " + SITE_NAME + " " + (result["content"][:200] if result["content"] else "")),
                result["date"] or item["date"],
                result["url"],
                result["site_name"],
                attachments_json,
                result["group"],
                result["url"],
            ),
        )
        inserted += 1
        if inserted % 5 == 0:
            conn.commit()
            print("  Inserted %d/%d..." % (inserted, len(items)))

    conn.commit()
    conn.close()
    print("\nDone. Inserted: %d, Existing: %d" % (inserted, existing))


if __name__ == "__main__":
    if "--test" in sys.argv:
        url = None
        for i, arg in enumerate(sys.argv):
            if arg == "--test" and i + 1 < len(sys.argv) and sys.argv[i+1].startswith("http"):
                url = sys.argv[i+1]
                break
        if url:
            crawl_test(url)
        else:
            crawl(test_mode=True)
    elif "--list" in sys.argv:
        html = fetch_page(LIST_URL)
        items = extract_list_items(html)
        print("Page 1: %d items" % len(items))
        for item in items[:5]:
            print("  %s | %s | %s" % (item["title"][:40], item["date"], item["url"]))
    else:
        crawl(test_mode=False, max_pages=MAX_PAGES)