#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
四川新闻网-资阳频道-原创文艺
https://zq.newssc.org/2017ycxw/
"""
import re, sys, os, json, time, requests

DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
SITE_NAME = "资阳原创文艺"
GROUP = "资阳"
MAX_PAGES = 5

# Page URLs: page1 is index, pages 2-5 are /2017ycxw/system/more/0017004000000/0017004000000_N.shtml
# N starts at 158 and decrements
PAGE_URLS = [
    "https://zq.newssc.org/2017ycxw/?spm=zm5056-001.0.0.2.w9S7zK",
]
for n in range(158, 158 - (MAX_PAGES - 1), -1):
    PAGE_URLS.append(f"https://zq.newssc.org/2017ycxw/system/more/0017004000000/0017004000000_{n}.shtml")


def fetch_page(url):
    r = requests.get(url, headers=HEADERS, timeout=30)
    r.encoding = "gb2312"
    return r.text


def extract_list_items(html):
    """Parse list page: <ul class="ul-listy"> <li><a href="URL"><p>TITLE</p><span>DATE</span></a></li>"""
    items = []
    pattern = r'<li><a href="(https?://[^"]+)"[^>]*>\s*<p>(.*?)</p>\s*<span>(.*?)</span>'
    for m in re.finditer(pattern, html, re.DOTALL):
        url = m.group(1).strip()
        title = m.group(2).strip()
        date_str = m.group(3).strip()
        items.append({"url": url, "title": title, "date": date_str})
    return items


def extract_content(html, source_url):
    """Extract title, content, date from detail page."""
    # Title from h1
    title = ""
    m = re.search(r"<h1>(.*?)</h1>", html, re.DOTALL)
    if m:
        title = m.group(1).strip()
    else:
        m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if m:
            t = m.group(1).strip()
            # Remove site name suffix
            t = re.sub(r'\s*[-—|]\s*.*$', '', t)
            title = t.strip()

    # Date from note div
    pub_date = ""
    m = re.search(r'<div class="note">.*?<span>(\d{4}年\d{1,2}月\d{1,2}日\s*\d{1,2}:\d{2}:\d{2})</span>', html, re.DOTALL)
    if m:
        pub_date = m.group(1).strip()
    else:
        # Try other date patterns
        m = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2}\s*\d{1,2}:\d{2})', html)
        if m:
            pub_date = m.group(1).strip()

    # Content from content-main div
    content = ""
    attachments = []
    m = re.search(r'<div class="content-main">(.*?)(?:</div>\s*<div class="note1"|</main>)', html, re.DOTALL)
    if m:
        content_html = m.group(1)
    else:
        # Try alternative
        m = re.search(r'<div class="content-main">(.*)', html, re.DOTALL)
        if m:
            content_html = m.group(1)
        else:
            content_html = ""

    if content_html:
        # Extract attachments first
        for a_tag in re.finditer(r'<a\s+[^>]*href="([^"]*\.(?:doc|docx|pdf|xls|xlsx|rar|zip))"[^>]*>(.*?)</a>', content_html, re.IGNORECASE | re.DOTALL):
            link_url = a_tag.group(1)
            link_text = re.sub(r'<[^>]+>', '', a_tag.group(2)).strip()
            if not link_url.startswith("http"):
                link_url = "https://zq.newssc.org" + link_url
                link_url = link_url.replace("https://zq.newssc.orghttps://", "https://")
            attachments.append({"name": link_text or link_url.split("/")[-1], "url": link_url})

        # Extract image references
        img_pattern = re.findall(r'<img[^>]*src="([^"]+)"', content_html)
        for img_url in img_pattern:
            if not img_url.startswith("http"):
                img_url = "https://zq.newssc.org" + img_url if img_url.startswith("/") else img_url
            attachments.append({"name": img_url.split("/")[-1], "url": img_url})

        # Parse paragraphs and tables
        parts = []
        for elem in re.finditer(r'<p[^>]*>(.*?)</p>|<table[^>]*>(.*?)</table>', content_html, re.DOTALL | re.IGNORECASE):
            tag = elem.group(0)
            if tag.startswith("<table"):
                parts.append(tag.group(0))
            else:
                # Paragraph
                p_text = elem.group(1)
                # Remove inner tags but keep text
                p_text = re.sub(r'<[^>]+>', '', p_text)
                p_text = p_text.strip()
                if p_text:
                    parts.append(p_text)
        content = "\n\n".join(parts)

    # If content is empty or just images, use PDF/attachment fallback
    if not content or len(content.strip()) < 20:
        if attachments:
            links = "\n".join([f"- [{a['name']}]({a['url']})" for a in attachments])
            content = f"{title}\n\n{links}" if title else links
        else:
            # Fallback: get all text from content-main
            text = re.sub(r'<[^>]+>', ' ', content_html)
            text = re.sub(r'\s+', ' ', text).strip()
            if text and len(text) > 20:
                content = text

    return {
        "title": title,
        "content": content,
        "date": pub_date,
        "url": source_url,
        "attachments": attachments,
        "site_name": SITE_NAME,
        "group": GROUP,
    }


def crawl_test(url=None):
    """Test a single article."""
    if not url:
        url = "https://zq.newssc.org/system/20250114/001501646.html"
    print(f"=== Testing: {url} ===")
    html = fetch_page(url)
    result = extract_content(html, url)
    for k, v in result.items():
        if k == "attachments":
            print(f"{k}: {v}")
        elif k == "content":
            print(f"{k} ({len(v)} chars):")
            print(v[:1000])
            print("...")
        else:
            print(f"{k}: {v}")
    return result


def crawl_list(test_mode=False):
    """Crawl list pages and detail pages."""
    import sqlite3

    all_items = []
    seen_urls = set()

    # Step 1: gather list items from all pages
    for pi, page_url in enumerate(PAGE_URLS):
        print(f"[Page {pi+1}/{len(PAGE_URLS)}] {page_url}")
        html = fetch_page(page_url)
        items = extract_list_items(html)
        print(f"  Found {len(items)} items")
        for item in items:
            if item["url"] not in seen_urls:
                seen_urls.add(item["url"])
                all_items.append(item)
                if test_mode and len(all_items) >= 3:
                    break
        if test_mode and len(all_items) >= 3:
            break

    print(f"\nTotal unique items: {len(all_items)}")

    if test_mode:
        # Test first 3 detail pages
        for item in all_items[:3]:
            print(f"\n=== Detail: {item['title'][:40]}... ===")
            html = fetch_page(item["url"])
            result = extract_content(html, item["url"])
            print(f"  Title: {result['title']}")
            print(f"  Date: {result['date']}")
            print(f"  Content ({len(result['content'])} chars): {result['content'][:200]}")
            print(f"  Attachments: {len(result['attachments'])}")
        return

    # Step 2: save to database
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=15000")
    c = conn.cursor()

    inserted = 0
    existing = 0
    for item in all_items:
        # Check existing
        c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
        row = c.fetchone()
        if row:
            existing += 1
            continue

        html = fetch_page(item["url"])
        result = extract_content(html, item["url"])
        attachments_json = json.dumps(result["attachments"], ensure_ascii=False) if result["attachments"] else ""

        c.execute(
            """INSERT INTO gov_raw (title, content, summary, publish_date, page_url, site_name, attachments, group_name, source_url)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (
                result["title"],
                result["content"],
                (result["title"] + " " + SITE_NAME + " " + (result["content"][:200] if result["content"] else "")),
                result["date"],
                result["url"],
                result["site_name"],
                attachments_json,
                result["group"],
                result["url"],  # source_url
            ),
        )
        inserted += 1
        if inserted % 20 == 0:
            conn.commit()
            print(f"  Inserted {inserted}/{len(all_items)}...")

    conn.commit()
    conn.close()
    print(f"\nDone. Inserted: {inserted}, Existing: {existing}")


if __name__ == "__main__":
    if "--test" in sys.argv:
        url = None
        for i, arg in enumerate(sys.argv):
            if arg == "--test" and i + 1 < len(sys.argv) and sys.argv[i+1].startswith("http"):
                url = sys.argv[i+1]
                break
        if url:
            crawl_test(url)
        else:
            crawl_list(test_mode=True)
    elif "--list" in sys.argv:
        # Just show list items
        html = fetch_page(PAGE_URLS[0])
        items = extract_list_items(html)
        print(f"Page 1: {len(items)} items")
        for item in items[:5]:
            print(f"  {item['title'][:40]} | {item['date']} | {item['url']}")
    else:
        crawl_list(test_mode=False)