#!/usr/bin/env python3
"""
揭阳大南海石化工业区 - 公示公告 (jieyang.gov.cn)
==============================================
列表: /szfjg/jysdnhshgyq/zwgk/gsgg/
分页: index.html → index_2.html → ... → index_38.html
详情: .../content/post_XXXX.html
正文: div.article + div.article-content (兜底)

用法:
  python3 crawl_jieyang_gsgg.py          # 全量爬取（38页）
  python3 crawl_jieyang_gsgg.py --test   # 测试模式（仅第1页，5条）
  python3 crawl_jieyang_gsgg.py --pages=5  # 指定爬取页数
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "揭阳大南海-公示公告"
CATEGORY = "政府公告"
BASE_URL = "http://www.jieyang.gov.cn"
LIST_URL = "http://www.jieyang.gov.cn/szfjg/jysdnhshgyq/zwgk/gsgg/"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
MAX_PAGES = 5           # 从最后一页链接得知共38页
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365 * 3)).strftime("%Y-%m-%d")


def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"    ⚠️ 请求失败: {e}")
        return None


def parse_list(html):
    """
    提取列表页中的条目。
    <ul class="ul_newsList active">
      <li><a href="...content/post_XXXX.html">title</a><span class="span_time">YYYY-MM-DD HH:MM</span></li>
    </ul>
    """
    items = []
    # 先找列表 ul
    m = re.search(r'<ul[^>]*class="ul_newsList[^"]*"[^>]*>(.*?)</ul>', html, re.DOTALL)
    if not m:
        print("    ❌ 未找到 ul_newsList")
        return items
    ul = m.group(1)
    # 提取每个 li
    for m2 in re.finditer(
        r'<li[^>]*>.*?<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>.*?<span[^>]*class="[^"]*span_time[^"]*"[^>]*>(\d{4}-\d{2}-\d{2})',
        ul, re.DOTALL
    ):
        href = m2.group(1)
        title = re.sub(r'<[^>]+>', '', m2.group(2)).strip()
        date = m2.group(3).strip()
        if title and href:
            # 补全相对URL
            if href.startswith('http'):
                full_url = href
            elif href.startswith('/'):
                full_url = BASE_URL + href
            else:
                full_url = BASE_URL + '/' + href
            items.append((title, full_url, date))
    return items


def extract_detail(html, url):
    """提取详情页的标题、正文HTML、发布日期"""
    title = ""
    # 标题: <title> 或 h1
    m = re.search(r'<title>([^<]+)', html)
    if m:
        title = re.sub(r'[-_—].*', '', m.group(1)).strip()
    if not title:
        m = re.search(r'<h1[^>]*>([^<]+)', html)
        if m:
            title = m.group(1).strip()

    # 正文: 尝试多种容器
    content = ""
    # 1) div.article-content (最常用)
    m = re.search(
        r'<div[^>]*class="[^"]*article-content[^"]*"[^>]*>(.*?)</div>\s*</div>',
        html, re.DOTALL
    )
    if m:
        content = m.group(1)
    # 2) div.article
    if not content or len(content) < 50:
        m = re.search(
            r'<div[^>]*class="[^"]*article[^"]*"[^>]*>(.*?)</div>\s*</div>',
            html, re.DOTALL
        )
        if m:
            content = m.group(1)
    # 3) div.TRS_Editor (政府站常见)
    if not content or len(content) < 50:
        m = re.search(
            r'<div[^>]*class="[^"]*TRS_Editor[^"]*"[^>]*>(.*?)</div>\s*</div>',
            html, re.DOTALL
        )
        if m:
            content = m.group(1)
    # 4) div.Custom_UnionStyle
    if not content or len(content) < 50:
        m = re.search(
            r'<div[^>]*class="[^"]*Custom_UnionStyle[^"]*"[^>]*>(.*?)</div>\s*</div>',
            html, re.DOTALL
        )
        if m:
            content = m.group(1)
    # 5) div.bt_content
    if not content or len(content) < 50:
        m = re.search(
            r'<div[^>]*class="[^"]*bt_content[^"]*"[^>]*>(.*?)</div>\s*</div>',
            html, re.DOTALL
        )
        if m:
            content = m.group(1)

    # 清洗
    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r' style="[^"]*"', '', content)
        content = content.strip()

    # 日期: 从各种位置提取
    date = ""
    # meta 标签
    m = re.search(
        r'<meta[^>]*name=["\']?(?:publish_date|pubdate|date)["\']?[^>]*content=["\']([^"\']+)["\']',
        html, re.I
    )
    if m:
        date = m.group(1)[:10]
    if not date:
        m = re.search(r'(\d{4}-\d{2}-\d{2})\s+\d{2}:\d{2}', html)
        if m:
            date = m.group(1)
    if not date:
        m = re.search(r'发布时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', html)
        if m:
            date = m.group(1).replace('/', '-')

    return title, content, date


def get_max_pages(html):
    """从列表页提取总页数"""
    m = re.search(r'index_(\d+)\.html[^>]*>最后一页', html)
    if m:
        return int(m.group(1))
    # fallback: 找所有分页数字
    pages = re.findall(r'index_(\d+)\.html', html)
    if pages:
        return max(int(p) for p in pages)
    return 1


def generate_page_urls(base_list_url, max_pages):
    """生成所有分页URL"""
    urls = [base_list_url]
    for p in range(2, max_pages + 1):
        urls.append(base_list_url.replace('index.html', f'index_{p}.html'))
    return urls


def run(args=None):
    # 解析参数
    test_mode = '--test' in sys.argv
    pages_limit = MAX_PAGES
    for a in sys.argv:
        if a.startswith('--pages='):
            try:
                pages_limit = int(a.split('=')[1])
            except:
                pass

    print(f"🌐 {SITE_NAME}")
    print(f"   分类: {CATEGORY}")
    print(f"   模式: {'测试(仅第1页,5条)' if test_mode else f'全量(最多{pages_limit}页)'}")

    # 先获取第1页，确定总页数
    html = fetch(LIST_URL)
    if not html:
        print("  ❌ 无法获取列表页")
        return

    total_pages = get_max_pages(html)
    actual_pages = min(total_pages, pages_limit)
    print(f"   总页数: {total_pages}, 将爬: {actual_pages}")

    # 收集所有条目
    all_items = []
    page_urls = generate_page_urls(LIST_URL, actual_pages)

    for page_idx, page_url in enumerate(page_urls, 1):
        if page_idx > 1:
            html = fetch(page_url)
            if not html:
                print(f"  ⚠️ 第{page_idx}页获取失败，跳过")
                continue

        items = parse_list(html)
        print(f"  📋 第{page_idx}页: {len(items)} 条")
        all_items.extend(items)

        if test_mode and len(all_items) >= 15:
            all_items = all_items[:15]
            break

        time.sleep(0.5)

    # 按日期排序（最新的在前）
    def sort_key(item):
        try:
            return item[2]  # date string YYYY-MM-DD
        except:
            return "0000-00-00"
    all_items.sort(key=sort_key, reverse=True)

    print(f"\n📥 共 {len(all_items)} 条（去重后）")

    # 爬取详情
    results = []
    seen_urls = set()
    for i, (title, url, date) in enumerate(all_items):
        if url in seen_urls:
            continue
        seen_urls.add(url)

        print(f"  [{i + 1}/{len(all_items)}] {title[:40]}...", end=" ", flush=True)

        detail_html = fetch(url)
        if not detail_html:
            print("⚠️ 无法获取详情")
            continue

        dt, content, d2 = extract_detail(detail_html, url)
        final_title = dt or title
        final_date = d2 or date

        # 跳过无内容或内容过短
        if not content or len(content) < 30:
            print("⏭️ 无有效正文")
            continue

        content_len = len(content)
        results.append({
            "site_name": SITE_NAME,
            "title": final_title,
            "url": url,
            "content": content,
            "pub_date": final_date,
            "category": CATEGORY,
            "tags": "揭阳,大南海,公示公告",
        })
        print(f"✅ ({content_len}字) [{final_date}]")

        if test_mode and len(results) >= 5:
            print("  ⏹️ 测试模式，5条停止")
            break

        time.sleep(0.3)

    # 入库
    if results:
        print(f"\n📤 入库 {len(results)} 条...")
        push_to_searchdb(results, "jieyang_gsgg")
    else:
        print("\n⏭️ 无数据入库")

    print(f"✅ 完成: {len(results)} 条")
    return results


if __name__ == "__main__":
    run(sys.argv[1] if len(sys.argv) > 1 else None)