#!/usr/bin/env python3
"""茂名滨海新区（mgs.gov.cn）环境保护信息公开爬虫"""
import requests
import re
import sqlite3
import sys
import time
from bs4 import BeautifulSoup, Tag
from urllib.parse import urljoin
import urllib.parse

BASE_URL = "http://www.mgs.gov.cn"
LIST_URL = BASE_URL + "/bhzdlyxxgkzl/hjbh/index.html"
LIST_PAGES = 5
DB_PATH = "/root/search.db"
SITE_NAME = "茂名滨海新区环境保护"
SLEEP = 1.5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}

def get_soup(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        return BeautifulSoup(r.text, "html.parser")
    except Exception as e:
        print(f"[WARN] 请求失败: {url} - {e}")
        return None

def parse_list_items(soup):
    items = []
    for li in soup.select("ul.list_con > li"):
        a = li.find("a")
        p = li.find("p", class_="title")
        span = li.find("span", class_="time")
        if not a or not p:
            continue
        href = a.get("href", "").strip()
        title = p.get_text(strip=True)
        date_str = span.get_text(strip=True).strip("[]") if span else ""
        if href and title:
            full_url = urljoin(BASE_URL, href)
            items.append((full_url, title, date_str))
    return items

def parse_detail(soup, url):
    # Title from h1.title
    title_tag = soup.find("h1", class_="title")
    title = title_tag.get_text(strip=True) if title_tag else ""
    
    # Date from meta
    pub_date = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date:
        content = meta_date.get("content", "")
        m = re.search(r"\d{4}-\d{2}-\d{2}", content)
        if m:
            pub_date = m.group()
    
    # Content from div.artile_con
    content_parts = []
    artile_con = soup.find("div", class_="artile_con")
    if not artile_con:
        return title, pub_date, "", "[]"
    
    for child in artile_con.find_all(["p", "table", "img"], recursive=True):
        if child.name == "p":
            if child.find_parent("table"):
                continue
            text = child.get_text(strip=True)
            if text:
                content_parts.append(text)
        elif child.name == "table":
            md = html_table_to_html(child)
            if md:
                content_parts.append(md)
        elif child.name == "img":
            src = child.get("src", "").strip()
            alt = child.get("alt", "").strip()
            if src and not src.endswith(".gif") and "fileTypeImages" not in src:
                full_src = urljoin(BASE_URL, src)
                content_parts.append(f"![{alt}]({full_src})" if alt else f"![]({full_src})")
    
    content = "\n\n".join(content_parts)
    
    # Attachments
    attachments = []
    for a_tag in artile_con.find_all("a", class_="nfw-cms-attachment"):
        href = a_tag.get("href", "").strip()
        title = a_tag.get_text(strip=True) or href.split("/")[-1]
        if href:
            full_url = urljoin(BASE_URL, href)
            attachments.append({"url": full_url, "title": title})
    # Also catch regular PDF links
    for a_tag in artile_con.find_all("a", href=re.compile(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|txt)$", re.I)):
        href = a_tag.get("href", "").strip()
        if any(a.get("href","") == href for a in artile_con.find_all("a", class_="nfw-cms-attachment")):
            continue
        title = a_tag.get_text(strip=True) or href.split("/")[-1]
        if href:
            full_url = urljoin(BASE_URL, href)
            attachments.append({"url": full_url, "title": title})
    
    return title, pub_date, content, str(attachments)

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def get_page_url(page):
    if page == 1:
        return LIST_URL
    else:
        return f"{BASE_URL}/bhzdlyxxgkzl/hjbh/index_{page}.html"

def main():
    pages = LIST_PAGES
    if len(sys.argv) > 2 and sys.argv[1] == "--pages":
        pages = int(sys.argv[2])
    
    for page in range(1, pages + 1):
        url = get_page_url(page)
        print(f"[列表页] 第{page}页: {url}")
        soup = get_soup(url)
        if not soup:
            continue
        items = parse_list_items(soup)
        if not items:
            print(f"  无数据，可能已到末页")
            break
        print(f"  找到 {len(items)} 条")
        
        for item_url, list_title, list_date in items:
            title = list_title
            print(f"  → {title}")
            
            conn = sqlite3.connect(DB_PATH, timeout=60)
            c = conn.cursor()
            c.execute("SELECT id FROM gov_raw WHERE page_url=? AND site_name=?", (item_url, SITE_NAME))
            if c.fetchone():
                conn.close()
                print(f"    已存在，跳过")
                continue
            conn.close()
            
            time.sleep(SLEEP)
            detail_soup = get_soup(item_url)
            if not detail_soup:
                continue
            
            detail_title, pub_date, content, attachments = parse_detail(detail_soup, item_url)
            date = pub_date or list_date
            use_title = detail_title or title
            
            if not content:
                print(f"    正文为空，跳过")
                continue
            
            conn = sqlite3.connect(DB_PATH, timeout=60)
            c = conn.cursor()
            try:
                summary = content[:200] if content else ""
                has_table = 1 if "| ---" in content else 0
                c.execute("""
                    INSERT OR REPLACE INTO gov_raw (page_url, title, publish_date, site_name, content, attachments, summary, has_table, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'crawl_mgs_hjbh.py')
                """, (item_url, use_title, date, SITE_NAME, content, attachments, summary, has_table))
                conn.commit()
                paras = len(content.split("\n\n"))
                has_att = 1 if len(attachments) > 5 else 0
                print(f"    入库成功 ({paras}段, 表格={has_table}, 附件={has_att})")
            except Exception as e:
                print(f"    入库失败: {e}")
            finally:
                conn.close()
    
    print(f"\n✅ 茂名滨海新区爬取完成！")

if __name__ == "__main__":
    main()