#!/usr/bin/env python3
"""永修县人民政府 - 自然资源局-通知公告 爬虫
静态HTML分页: ./ (page1), index_1.html (page2) ... index_63.html (page64), 15条/页, 64页
列表: <td class="info"><a href="url" title="title">title</a></td> + <td class="fbrq">date</td>
详情: <div id="content" class="xl-text"><p id="wznr"><div class="trs_editor_view">HTML正文</div></p></div>
"""

import os
import re
import sys
import urllib.request
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "yongxiu.gov.cn-自然资源局-通知公告"
BASE_URL = "https://www.yongxiu.gov.cn"
LIST_DIR = "/bmxzxxgk/bmgk/zrzyj/qtfdgkxx/gggs_191304"
PAGE_SIZE = 15
TOTAL_PAGES = 64

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
def _make_img_abs(html_content, page_url):
    """将正文中<img>相对图片URL转为绝对URL"""
    base_url = page_url.rsplit("/", 1)[0] + "/"
    def _fix(m):
        src = m.group(1)
        if src.startswith("http"):
            return m.group(0)
        if src.startswith("./"):
            new_src = base_url + src[2:]
        elif src.startswith("/"):
            new_src = "https://www.yongxiu.gov.cn" + src
        else:
            new_src = base_url + src
        return m.group(0).replace('src="' + src + '"', 'src="' + new_src + '"')
    return re.sub(r'\bsrc="([^"]+)"', _fix, html_content)

CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
print(f"[info] 日期过滤: >= {CUTOFF_DATE}")


def fetch_list_page(page_no):
    """获取列表页HTML"""
    if page_no == 1:
        url = f"{BASE_URL}{LIST_DIR}/"
    else:
        url = f"{BASE_URL}{LIST_DIR}/index_{page_no - 1}.html"
    try:
        req = urllib.request.Request(url, headers=HEADERS)
        resp = urllib.request.urlopen(req, timeout=30)
        return resp.read().decode("utf-8")
    except Exception as e:
        print(f"[error] 第{page_no}页请求失败: {e}")
        return None


def parse_list_items(html):
    """解析列表HTML"""
    # Pattern: <td class="info"><a href="url" title="title">title</a></td>
    # with date in <td class="fbrq">date</td>
    items = re.findall(
        r'<td class="info">\s*<a[^>]*href="([^"]+)"[^>]*title="([^"]*)">[^<]*</a>\s*</td>\s*<td class="fbrq">\s*([^<]+)\s*</td>',
        html
    )
    results = []
    for url, title, date in items:
        if url.startswith("/"):
            url = BASE_URL + url
        elif url.startswith("http"):
            pass  # already absolute
        else:
            url = BASE_URL + LIST_DIR + "/" + url
        results.append({"title": title.strip(), "url": url, "date": date.strip()})
    return results


def fetch_detail_content(url):
    """从详情页提取正文HTML和完整标题"""
    try:
        req = urllib.request.Request(url, headers=HEADERS)
        resp = urllib.request.urlopen(req, timeout=30)
        html = resp.read().decode("utf-8")
    except Exception as e:
        print(f"  [warn] 详情页失败: {url[-60:]}, {e}")
        return "", ""

    # Full title from meta
    full_title = ""
    meta_t = re.search(r'<meta name="ArticleTitle" content="([^"]+)"', html)
    if meta_t:
        full_title = meta_t.group(1).strip()

    # Content: <div id="content" class="xl-text"><p id="wznr"><div class="trs_editor_view ...">...</div></p></div>
    m = re.search(
        r'<div class="trs_editor_view[^"]*"[\s\S]*?</div>',
        html
    )
    if m:
        content = m.group(0).strip()
        content = _make_img_abs(content, url)
        return content, full_title

    # Fallback: try div#content.xl-text
    m2 = re.search(
        r'<div id="content" class="xl-text">([\s\S]*?)</div>\s*<div class="xl-tab3"',
        html
    )
    if m2:
        content = m2.group(1).strip()
        content = _make_img_abs(content, url)
        return content, full_title

    return "", full_title


def main():
    import sqlite3

    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    c.execute("PRAGMA journal_mode=WAL")

    print(f"[info] 总页数: {TOTAL_PAGES}, 每页: {PAGE_SIZE}")

    new_count = 0
    old_count = 0
    dup_count = 0
    err_count = 0

    for page in range(1, TOTAL_PAGES + 1):
        html = fetch_list_page(page)
        if not html:
            continue

        items = parse_list_items(html)
        if not items:
            print(f"[warn] 第{page}页未解析到条目")
            continue

        page_new = 0
        for item in items:
            title = item["title"]
            url = item["url"]
            date = item["date"]

            if date and date < CUTOFF_DATE:
                old_count += 1
                continue

            c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=?", (url,))
            if c.fetchone()[0] > 0:
                dup_count += 1
                continue

            content, full_title = fetch_detail_content(url)
            if not content:
                content = "[无正文] 附件型条目，请查看原文链接"

            final_title = full_title if full_title else title

            try:
                c.execute("""INSERT OR IGNORE INTO gov_raw
                    (page_url, title, content, site_name, publish_date, summary, category)
                    VALUES (?, ?, ?, ?, ?, ?, ?)""",
                    (url, final_title, content, SITE_NAME, date, final_title, "通知公告"))
                if c.rowcount > 0:
                    page_new += 1
            except Exception as e:
                print(f"  [error] 入库失败: {e}")
                err_count += 1

        conn.commit()
        new_count += page_new
        print(f"[page {page}/{TOTAL_PAGES}] +{page_new} new, cumulative: {new_count}")

    conn.close()
    print(f"\n[完成] {SITE_NAME}: 新增 {new_count} 条, 旧跳过 {old_count}, 重复 {dup_count}, 错误 {err_count}")


if __name__ == "__main__":
    main()