#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
辽宁（营口）沿海产业基地-通知公告
https://ykcyjd.yingkou.gov.cn/003/003003/about.html
"""
import re, sys, os, json, time, requests

DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
SITE_NAME = "辽宁（营口）沿海产业基地-通知公告"
GROUP = "营口"
PER_PAGE = 20
MAX_PAGES = 5

BASE_URL = "https://ykcyjd.yingkou.gov.cn/003/003003"


def get_list_url(page):
    if page == 1:
        return BASE_URL + "/about.html"
    else:
        return BASE_URL + "/%d.html" % page


def fetch_page(url):
    r = requests.get(url, headers=HEADERS, timeout=30, allow_redirects=True)
    r.encoding = "utf-8"
    return r.text


def extract_list_items(html, page=1):
    items = []
    m = re.search(r'<ul[^>]*id="infolist"[^>]*>(.*?)</ul>', html, re.DOTALL)
    if not m:
        return items
    
    ul_html = m.group(1)
    li_pattern = r'<li[^>]*>.*?<a[^>]*href="([^"]*)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>.*?<span[^>]*class="ewb-list-date"[^>]*>(.*?)</span>'
    for m2 in re.finditer(li_pattern, ul_html, re.DOTALL):
        url = m2.group(1)
        title = m2.group(2).strip()
        date_str = m2.group(4).strip()
        if url.startswith("/"):
            url = "https://ykcyjd.yingkou.gov.cn" + url
        elif not url.startswith("http"):
            url = "https://ykcyjd.yingkou.gov.cn/" + url
        if not url.startswith("https://ykcyjd.yingkou.gov.cn"):
            continue
        items.append({"url": url, "title": title, "date": date_str})
    return items


def extract_content(html, source_url):
    title = ""
    m = re.search(r'<h3[^>]*id="ivs_title"[^>]*>(.*?)</h3>', html, re.DOTALL)
    if m:
        title = m.group(1).strip()
    else:
        m = re.search(r"<title>(.*?)</title>", html, re.DOTALL)
        if m:
            t = m.group(1).strip()
            t = re.sub(r'\s*[-—|]\s*.*$', '', t)
            title = t.strip()

    pub_date = ""
    m = re.search(r'发布时间：(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        pub_date = m.group(1).strip()

    content = ""
    attachments = []
    content_html = ""
    
    m = re.search(r'<div[^>]*class="ewb-article-content"[^>]*>(.*?)</div>\s*<!--', html, re.DOTALL)
    if m:
        content_html = m.group(1)

    if content_html:
        for a_tag in re.finditer(r'<a[^>]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|rar|zip))"[^>]*>(.*?)</a>', content_html, re.I | re.DOTALL):
            link_url = a_tag.group(1)
            link_text = re.sub(r"<[^>]+>", "", a_tag.group(2)).strip()
            if not link_url.startswith("http"):
                link_url = "https://ykcyjd.yingkou.gov.cn" + link_url if link_url.startswith("/") else link_url
            attachments.append({"name": link_text or link_url.split("/")[-1], "url": link_url})

        for img_url in re.findall(r'<img[^>]*src="([^"]+)"', content_html):
            if not img_url.startswith("http"):
                img_url = "https://ykcyjd.yingkou.gov.cn" + img_url if img_url.startswith("/") else img_url
            attachments.append({"name": img_url.split("/")[-1], "url": img_url})

        parts = []
        for elem in re.finditer(r'<p[^>]*>(.*?)</p>|<table[^>]*>(.*?)</table>', content_html, re.DOTALL | re.I):
            tag = elem.group(0)
            if tag.startswith("<table"):
                parts.append(tag.group(0))
            else:
                p_text = elem.group(1)
                p_text = re.sub(r'<[^>]+>', '', p_text)
                p_text = re.sub(r'&[nN][bB][sS][pP];', ' ', p_text)
                p_text = p_text.strip()
                if p_text:
                    parts.append(p_text)
        content = "\n\n".join(parts)

    if not content or len(content.strip()) < 20:
        text = re.sub(r'<[^>]+>', ' ', content_html)
        text = re.sub(r'\s+', ' ', text).strip()
        if text and len(text) > 20:
            content = text

    return {
        "title": title,
        "content": content,
        "date": pub_date,
        "url": source_url,
        "attachments": attachments,
        "site_name": SITE_NAME,
        "group": GROUP,
    }


def crawl(test_mode=False, max_pages=MAX_PAGES):
    import sqlite3
    all_items = []
    seen_urls = set()

    for page in range(1, max_pages + 1):
        url = get_list_url(page)
        print("[Page %d/%d] %s" % (page, max_pages, url[:60]))
        html = fetch_page(url)
        items = extract_list_items(html, page)
        print("  Found %d items" % len(items))
        for item in items:
            if item["url"] not in seen_urls:
                seen_urls.add(item["url"])
                all_items.append(item)
        if test_mode and len(all_items) >= 3:
            break

    print("\nTotal unique items: %d" % len(all_items))

    if test_mode:
        for item in all_items[:3]:
            print("\n=== Detail: %s ===" % item["title"][:40])
            html = fetch_page(item["url"])
            result = extract_content(html, item["url"])
            print("  Title: %s" % result["title"])
            print("  Date: %s" % result["date"])
            print("  Content (%d chars): %s" % (len(result["content"]), result["content"][:200]))
            print("  Attachments: %d" % len(result["attachments"]))
        return

    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()

    inserted = 0
    existing = 0
    total = len(all_items)

    for idx, item in enumerate(all_items):
        c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
        if c.fetchone():
            existing += 1
            continue

        html = fetch_page(item["url"])
        result = extract_content(html, item["url"])
        attachments_json = json.dumps(result["attachments"], ensure_ascii=False) if result["attachments"] else ""

        c.execute(
            """INSERT INTO gov_raw (title, content, summary, publish_date, page_url, site_name, attachments, group_name, source_url)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (
                result["title"],
                result["content"],
                (result["title"] + " " + SITE_NAME + " " + (result["content"][:200] if result["content"] else "")),
                result["date"] or item["date"],
                result["url"],
                result["site_name"],
                attachments_json,
                result["group"],
                result["url"],
            ),
        )
        inserted += 1
        if inserted % 20 == 0:
            conn.commit()
            print("  Progress: %d/%d inserted..." % (inserted, total))

    conn.commit()
    conn.close()
    print("\nDone. Inserted: %d, Existing: %d (total: %d)" % (inserted, existing, total))


if __name__ == "__main__":
    if "--test" in sys.argv:
        crawl(test_mode=True)
    elif "--list" in sys.argv:
        html = fetch_page(get_list_url(1))
        items = extract_list_items(html)
        print("Page 1: %d items" % len(items))
        for item in items[:5]:
            print("  %s | %s | %s" % (item["title"][:40], item["date"], item["url"][:50]))
    else:
        crawl(test_mode=False, max_pages=MAX_PAGES)