#!/usr/bin/env python3
"""Crawler for 太仓市人民政府 - 公示公告
CMS: Taicang CMS, UTF-8, 15条/页, 179页
Site: www.taicang.gov.cn/taicang/gsgg/
"""
import requests, re, json, sys, os, sqlite3
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "http://www.taicang.gov.cn"
LIST_PATH = "/taicang/gsgg"
LIST_URL = BASE_URL + LIST_PATH + "/xwzx_list.shtml"
SITE_NAME = "太仓市人民政府-公示公告"
GROUP = "江苏苏州"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 5
INCREMENTAL_DAYS = 7

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TIMEOUT = 20
session = requests.Session()
session.headers.update(HEADERS)


def fetch(url):
    resp = session.get(url, timeout=TIMEOUT)
    resp.encoding = "utf-8"
    return resp.text


def parse_list(html):
    """Parse list — div.page_list > ul > li > a(title+href) + span.date"""
    soup = BeautifulSoup(html, "html.parser")
    items = []
    div = soup.find("div", class_="page_list")
    if not div:
        return items
    ul = div.find("ul")
    if not ul:
        return items
    for li in ul.find_all("li", recursive=False):
        a = li.find("a", href=True)
        if not a:
            continue
        title = a.get("title") or a.get_text(strip=True)
        href = a["href"].strip()
        if not title or not href:
            continue
        span = li.find("span")
        date = span.get_text(strip=True) if span else ""
        if not href.startswith("http"):
            href = urljoin(BASE_URL, href)
        items.append((title.strip(), href, date[:10]))
    return items


def parse_detail(html, url):
    """Parse detail — div.hcontent > div.article"""
    soup = BeautifulSoup(html, "html.parser")

    # Title from ucaptitle or h1
    title = ""
    ucap_title = soup.find("ucaptitle")
    if ucap_title:
        title = ucap_title.get_text(strip=True)
    if not title:
        h1 = soup.find("h1", class_="article_title")
        if h1:
            title = h1.get_text(strip=True)
    if not title:
        title_tag = soup.find("title")
        if title_tag:
            title = title_tag.get_text(strip=True).replace("_ 太仓市人民政府", "").strip()

    # Date from meta PubDate or publishtime
    pub_date = ""
    pub_time = soup.find("publishtime")
    if pub_time:
        pub_date = pub_time.get_text(strip=True)[:10]
    if not pub_date:
        for m in soup.find_all("meta"):
            name = (m.get("name") or "").lower()
            if "pubdate" in name or "publishdate" in name:
                pub_date = (m.get("content") or "")[:10]
                break
    if not pub_date:
        m = re.search(r"时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})", html)
        if m:
            pub_date = m.group(1)

    # Content from ucapcontent
    content = ""
    hc = soup.find("div", class_="hcontent")
    if hc:
        ucap = hc.find("ucapcontent")
        if ucap:
            parts = []
            for child in ucap.children:
                if not hasattr(child, "name") or child.name is None:
                    continue
                tag = child.name.lower()
                if tag == "p":
                    text = child.get_text(" ", strip=True)
                    imgs = child.find_all("img")
                    for img in imgs:
                        src = img.get("src", "")
                        if src:
                            alt = img.get("alt", "")
                            full_src = urljoin(url, src)
                            text += "\n![%s](%s)" % (alt, full_src)
                    if text:
                        parts.append(text)
                elif tag == "table":
                    rows = []
                    for tr in child.find_all("tr"):
                        cells = [td.get_text(" ", strip=True).replace("|", "\\|") for td in tr.find_all(["td", "th"])]
                        if cells:
                            rows.append("| " + " | ".join(cells) + " |")
                    if rows:
                        parts.append("\n".join(rows))
                elif tag in ("div", "section", "blockquote"):
                    text = child.get_text(" ", strip=True)
                    if text:
                        parts.append(text)
                elif tag == "br":
                    parts.append("")
                elif tag == "img":
                    src = child.get("src", "")
                    if src:
                        alt = child.get("alt", "")
                        full_src = urljoin(url, src)
                        parts.append("![%s](%s)" % (alt, full_src))
                elif tag == "a":
                    a_href = child.get("href", "")
                    link_text = child.get_text(strip=True)
                    if a_href and link_text and not a_href.startswith("javascript"):
                        parts.append("[%s](%s)" % (link_text, urljoin(url, a_href)))

            content = "\n\n".join(p for p in parts if p)
            content = re.sub(r"\n{3,}", "\n\n", content)

    # Attachments (file links anywhere in page)
    attachments = []
    for a in soup.find_all("a", href=True):
        ahref = a["href"].strip().lower()
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", ahref):
            full_url = urljoin(url, a["href"].strip())
            title_text = a.get_text(strip=True) or full_url.split("/")[-1]
            attachments.append({"title": title_text, "url": full_url})

    # Fallback for short content
    if len(content.strip()) < 20 and attachments:
        content = '<p><a href="%s">%s</a></p>\n\n附件列表：\n' % (url, title)
        for att in attachments:
            content += '  - <p><a href="%s">%s</a></p>\n' % (att['url'], att['title'])

    return title, pub_date, content, attachments


def incremental_filter(items):
    cutoff = datetime.now() - timedelta(days=INCREMENTAL_DAYS)
    filtered = []
    for title, url, date_str in items:
        try:
            item_date = datetime.strptime(date_str, "%Y-%m-%d")
            if item_date >= cutoff:
                filtered.append((title, url, date_str))
        except (ValueError, IndexError):
            filtered.append((title, url, date_str))
    return filtered


def main():
    is_incremental = any(arg in sys.argv for arg in ["--incremental", "incremental", "inc"])

    # Page 1: xwzx_list.shtml, Page N>=2: xwzx_list_N.shtml
    page_urls = [LIST_URL]
    for i in range(2, MAX_PAGES + 1):
        page_urls.append(BASE_URL + LIST_PATH + "/xwzx_list_%d.shtml" % i)

    all_items = []
    for idx, url in enumerate(page_urls):
        try:
            html = fetch(url)
            items = parse_list(html)
            print("Page %d: %d items" % (idx + 1, len(items)), file=sys.stderr)
            all_items.extend(items)
        except Exception as e:
            print("Page %d error (%s): %s" % (idx + 1, url, e), file=sys.stderr)

    print("Total: %d" % len(all_items), file=sys.stderr)

    if is_incremental:
        all_items = incremental_filter(all_items)
        print("Incremental: %d" % len(all_items), file=sys.stderr)

    # Dedup
    seen = set()
    unique_items = []
    for item in all_items:
        if item[1] not in seen:
            seen.add(item[1])
            unique_items.append(item)

    results = []
    for title, url, list_date in unique_items:
        try:
            html = fetch(url)
            det_title, det_date, content, attachments = parse_detail(html, url)
            final_title = det_title or title
            final_date = det_date or list_date
            results.append({
                "title": final_title,
                "page_url": url,
                "publish_date": final_date,
                "content": content,
                "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
                "site_name": SITE_NAME,
            })
            print("  OK: %s" % final_title[:50], file=sys.stderr)
        except Exception as e:
            print("  ERR %s: %s" % (url, e), file=sys.stderr)

    saved = 0
    skipped = 0
    errors = 0
    for r in results:
        content_text = r.get("content", "")[:50000]
        summary = re.sub(r"\s+", " ", content_text[:200]).strip() or r["title"]
        attachments = r.get("attachments", "")
        if isinstance(attachments, str):
            pass
        else:
            attachments = json.dumps(attachments, ensure_ascii=False)
        try:
            conn = sqlite3.connect(DB_PATH, timeout=60)
            c = conn.cursor()
            c.execute("""INSERT OR IGNORE INTO gov_raw
                (page_url, title, publish_date, site_name, group_name, summary, content, attachments, source_url)
                VALUES (?,?,?,?,?,?,?,?,?)""",
                (r["page_url"], r["title"], r["publish_date"],
                 SITE_NAME, GROUP, summary, content_text,
                 attachments, r["page_url"]))
            if c.rowcount > 0:
                saved += 1
            else:
                skipped += 1
            conn.commit()
            conn.close()
        except Exception as e:
            errors += 1

    print("[%s] Done! Saved: %d, Skipped: %d, Errors: %d" % (SITE_NAME, saved, skipped, errors), file=sys.stderr)

    for r in results:
        print(json.dumps(r, ensure_ascii=False))


if __name__ == "__main__":
    main()