#!/usr/bin/env python3
"""名山区（本溪市明山区）通知公告爬虫 - PowerCMS"""
import requests
import re
import sqlite3
import sys
import time
from bs4 import BeautifulSoup, Tag
from urllib.parse import urljoin
import urllib.parse

BASE_URL = "http://www.mingshan.gov.cn"
LIST_URL = BASE_URL + "/publicity/qzfxx/tzgg"
LIST_PAGES = 5
DB_PATH = "/root/search.db"
SITE_NAME = "本溪市明山区通知公告"
SLEEP = 1.5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}

def get_soup(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        return BeautifulSoup(r.text, "html.parser")
    except Exception as e:
        print(f"[WARN] 请求失败: {url} - {e}")
        return None

def parse_list_items(soup):
    items = []
    for a in soup.find_all("a", class_="tit"):
        href = a.get("href", "").strip()
        title = a.get("title", "").strip() or a.get_text(strip=True)
        if not href or not title:
            continue
        full_url = urljoin(BASE_URL, href)
        date_span = a.find_previous("span", class_="date")
        date_str = date_span.get_text(strip=True) if date_span else ""
        items.append((full_url, title, date_str))
    return items

def clean_title(t):
    return t.replace("\u007f", "").strip()

def parse_detail(soup, url):
    pub_date = ""
    meta_tds = soup.find_all("td", class_="tit")
    for td in meta_tds:
        if "发布日期" in td.get_text(strip=True):
            con_td = td.find_next("td", class_="con")
            if con_td:
                pub_date = con_td.get_text(strip=True)
            break
    if not pub_date:
        time_span = soup.find("span", string=re.compile(r"发布时间"))
        if time_span:
            m = re.search(r"\d{4}-\d{2}-\d{2}", time_span.get_text())
            if m:
                pub_date = m.group()
    content_parts = []
    con_txt = soup.find("div", class_="conTxt")
    if not con_txt:
        return "", pub_date, "[]"
    for child in con_txt.find_all(["p", "table", "img"], recursive=True):
        if child.name == "p":
            if child.find_parent("table"):
                continue
            text = child.get_text(strip=True)
            if text:
                content_parts.append(text)
        elif child.name == "table":
            md = html_table_to_html(child)
            if md:
                content_parts.append(md)
        elif child.name == "img":
            src = child.get("src", "").strip()
            alt = child.get("alt", "").strip()
            if src and not src.endswith(".gif") and "fileTypeImages" not in src:
                full_src = urljoin(BASE_URL, src)
                if alt:
                    content_parts.append(f"![{alt}]({full_src})")
                else:
                    content_parts.append(f"![]({full_src})")
    content = "\n\n".join(content_parts)
    attachments = []
    for a_tag in con_txt.find_all("a", href=re.compile(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|txt)$", re.I)):
        href = a_tag.get("href", "").strip()
        title = a_tag.get_text(strip=True) or a_tag.get("title", "") or href.split("/")[-1]
        if href:
            full_url = urljoin(BASE_URL, href)
            attachments.append({"url": full_url, "title": title})
    return content, pub_date, str(attachments)

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def get_page_url(page):
    anti_cache = "ttttttttttttttttttttttttt=12790"
    if page == 1:
        return f"{LIST_URL}?{anti_cache}"
    else:
        return f"{LIST_URL}_{page}?{anti_cache}"

def main():
    pages = LIST_PAGES
    if len(sys.argv) > 2 and sys.argv[1] == "--pages":
        pages = int(sys.argv[2])
    for page in range(1, pages + 1):
        url = get_page_url(page)
        print(f"[列表页] 第{page}页: {url}")
        soup = get_soup(url)
        if not soup:
            continue
        items = parse_list_items(soup)
        print(f"  找到 {len(items)} 条")
        for item_url, title, list_date in items:
            title = clean_title(title)
            print(f"  → {title}")
            conn = sqlite3.connect(DB_PATH)
            c = conn.cursor()
            c.execute("SELECT id FROM gov_raw WHERE page_url=? AND site_name=?", (item_url, SITE_NAME))
            if c.fetchone():
                conn.close()
                print(f"    已存在，跳过")
                continue
            conn.close()
            time.sleep(SLEEP)
            detail_soup = get_soup(item_url)
            if not detail_soup:
                continue
            content, pub_date, attachments = parse_detail(detail_soup, item_url)
            date = pub_date or list_date
            if not content:
                print(f"    正文为空，跳过")
                continue
            conn = sqlite3.connect(DB_PATH)
            c = conn.cursor()
            try:
                summary = content[:200] if content else ""
                has_table = 1 if "| ---" in content else 0
                c.execute("""
                    INSERT OR REPLACE INTO gov_raw
                    (page_url, title, publish_date, site_name, content, attachments, summary, has_table)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                """, (item_url, title, date, SITE_NAME, content, attachments, summary, has_table))
                conn.commit()
                paras = len(content.split("\n\n"))
                has_table = 1 if "| ---" in content else 0
                has_att = 1 if len(attachments) > 5 else 0
                print(f"    入库成功 ({paras}段, 表格={has_table}, 附件={has_att})")
            except Exception as e:
                print(f"    入库失败: {e}")
            finally:
                conn.close()
    print(f"\n✅ 名山区爬取完成！")

if __name__ == "__main__":
    main()