"""
scrapy_monitor/spiders/list_spider.py

从本地 sites_cache.json 读取站点列表（秒级启动）
用 sync_sites.py 每周同步一次 Notion 数据

站点类型：
  普通 HTML 站点 → sites_cache.json 里有 selector 字段
  API  JSON 站点 → sites_cache.json 里有 api 字段
"""

import re
import os
import sys
import json
import random
import scrapy

sys.path.insert(0, os.path.dirname(os.path.dirname(
    os.path.dirname(os.path.abspath(__file__)))))
import config

from scrapy_monitor.items import ArticleItem

CACHE_FILE = os.path.join(
    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
    "sites_cache.json"
)

GENERIC_TITLE_SELECTORS = [
    "h1 a", "h2 a", "h3 a", "h4 a",
    "li a", "td a",
    ".title a", ".list a",
    ".news-title a", ".article-title a",
    "article a", ".item a",
]

# 条目级“语义日期”选择器：明确带日期含义的元素，优先用这些
SEMANTIC_DATE_SELECTORS = [
    "time::attr(datetime)",
    "time::text",
    "[class*=date]::text",
    "[class*=Date]::text",     # 驼峰 class，如 timeFormat 用 e_timeFormat
    "[class*=time]::text",
    "[class*=Time]::text",     # 驼峰，如 e_timeFormat-4
    "[class*=pub]::text",
    "[class*=riqi]::text",     # 拼音 riqi=日期，部分政府站用
    "[class*=sj]::text",       # 拼音 shijian=时间 缩写
    "span.date::text",
    "em.date::text",
    "p.date::text",
]

# 弱兜底：只在语义选择器全失败后才用，容易误命中标题数字，仅取短文本
WEAK_DATE_SELECTORS = [
    "span::text",
    "em::text",
    "i::text",
]

# 日期正则：年-月-日各种分隔；纯 8 位数字加左右边界防误吞文号/电话/编号
DATE_RE = re.compile(
    r'(\d{4})[年\-/\.](\d{1,2})[月\-/\.](\d{1,2})'   # 2024-01-15 / 2024年1月15日
    r'|(?<!\d)(\d{4})(\d{2})(\d{2})(?!\d)'            # 20240115（前后不能再接数字）
)


def parse_date_str(raw: str):
    """从字符串提取 YYYY-MM-DD，失败返回 None。带月/日合法性校验。"""
    if not raw:
        return None
    m = DATE_RE.search(raw.strip())
    if not m:
        return None
    if m.group(1):
        y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
    else:
        y, mo, d = int(m.group(4)), int(m.group(5)), int(m.group(6))
    # 合法性校验，过滤把编号/电话误判成日期的情况
    if not (1990 <= y <= 2099 and 1 <= mo <= 12 and 1 <= d <= 31):
        return None
    return f"{y:04d}-{mo:02d}-{d:02d}"


def _nearest_container(a_selector):
    """
    找到条目 a 标签最近的“行容器”，优先 li/tr/dd/dt/article。
    取不到就返回 a 的父元素。
    """
    for xp in ('ancestor::li[1]', 'ancestor::tr[1]',
               'ancestor::dd[1]', 'ancestor::dt[1]',
               'ancestor::article[1]'):
        node = a_selector.xpath(xp)
        if node:
            return node
    parent = a_selector.xpath('..')
    return parent if parent else a_selector


def extract_item_date(a_selector) -> str:
    """
    从条目 a 标签找发布日期，策略（比旧版只往上两层更稳）：
      1. 在最近行容器内用“语义日期选择器”找
      2. 容器内语义全失败 → 用弱兜底选择器找（仅取 <=40 字的短文本，避开长标题）
      3. 仍失败 → 看 a 的兄弟节点（很多列表日期是 a 的兄弟 span）
    """
    container = _nearest_container(a_selector)

    # 1. 容器内语义日期
    for sel in SEMANTIC_DATE_SELECTORS:
        for txt in container.css(sel).getall():
            d = parse_date_str(txt)
            if d:
                return d

    # 2. 容器内弱兜底（短文本优先，避免命中长标题里的数字）
    for sel in WEAK_DATE_SELECTORS:
        for txt in container.css(sel).getall():
            t = (txt or "").strip()
            if 0 < len(t) <= 40:
                d = parse_date_str(t)
                if d:
                    return d

    # 3. 兄弟节点兜底（日期常是 a 的兄弟 span/em）
    for xp in ('following-sibling::*', 'preceding-sibling::*'):
        for sib in a_selector.xpath(xp):
            for sel in SEMANTIC_DATE_SELECTORS:
                for txt in sib.css(sel).getall():
                    d = parse_date_str(txt)
                    if d:
                        return d

    # 4. 文字关键词兜底：容器内任意元素，文本含“发布日期/发布时间/日期/时间”
    #    覆盖 class 不含 date/time 但文字标注了日期的情况（如 <div>发布日期：2026-04-03</div>）
    for el in container.css("p, div, span, td, li, em"):
        txt = " ".join(el.css("::text").getall())
        if ("日期" in txt or "时间" in txt or "发布" in txt) and len(txt) <= 60:
            d = parse_date_str(txt)
            if d:
                return d

    # 5. 上一层容器兜底：标题与日期不直接相邻（分属并列块）时，
    #    向上多走一层再用语义选择器找一次；范围仍受控，避免抓到页头日期。
    outer = container.xpath('..')
    if outer:
        # 上层元素若文本过长（像整页），跳过以防误判
        outer_txt = " ".join(outer.css("::text").getall())
        if len(outer_txt) <= 400:
            for sel in SEMANTIC_DATE_SELECTORS:
                for txt in outer.css(sel).getall():
                    d = parse_date_str(txt)
                    if d:
                        return d

    return ""


def load_sites_from_cache() -> list[dict]:
    """从本地缓存文件读取站点列表"""
    if not os.path.exists(CACHE_FILE):
        raise FileNotFoundError(
            f"找不到 {CACHE_FILE}，请先运行 python sync_sites.py 同步站点列表"
        )
    with open(CACHE_FILE, encoding="utf-8") as f:
        cache = json.load(f)
    synced_at = cache.get("synced_at", "未知")
    sites     = cache.get("sites", [])
    print(f"  缓存时间: {synced_at}，共 {len(sites)} 个站点")
    return sites


def auto_selector(url: str, custom_selector: str) -> str:
    """
    选择器推断优先级：
    1. 有自定义选择器 → 直接用
    2. URL 模式匹配 → 自动推断
    3. 都没有 → 返回空，走启发式逻辑
    """
    if custom_selector:
        return custom_selector

    # 模式1: list-{ID}-{page}.html → content-{ID}
    m = re.search(r'list-(\d+)-\d+', url)
    if m:
        return f'a[href*="content-{m.group(1)}"]'

    # 模式2: /art/ 路径（常见政府站文章路径）
    if '/art/' in url:
        return 'a[href*="/art/"]'

    # 模式3: .shtml 静态化站点（文章链接通常也是 .shtml）
    if url.endswith('.shtml') or '/shtml/' in url:
        return 'a[href$=".shtml"]'

    # 模式4: .jhtml Java 模板站点
    if url.endswith('.jhtml') or 'jhtml' in url:
        return 'a[href$=".jhtml"]'

    # 模式5: content_XXXX.html 模式（科创/TRS WCM 等 CMS）
    if 'content_' in url or re.search(r'/\d+/index\.htm', url):
        return 'a[href*="content_"]'

    # 模式6: /info/XXXXX/XXXXX.htm 模式（常见政务信息公开）
    if '/info/' in url:
        return 'a[href*="/info/"]'

    # 模式7: /xxgk/ 或 /zwgk/ 政务公开类
    m2 = re.search(r'/(xxgk|zwgk|news|gk)/[^/]*', url)
    if m2:
        prefix = m2.group(1)
        return f'a[href*="/{prefix}/"]'

    # 模式8: index.html 结尾的列表页，取倒数第二层路径中的数字段
    m3 = re.search(r'/(\d+)/index\.s?html?$', url)
    if m3:
        return f'a[href*="/{m3.group(1)}/"]'

    # 模式9: default.htm 结尾（如 hngy.gov.cn 这类）
    m4 = re.search(r'/(\d+)/default\.s?html?$', url)
    if m4:
        return f'a[href*="/info/"], a[href*="content_"], a[href*="/{m4.group(1)}/"]'

    return ""


def extract_items(response, site_name: str, custom_selector: str):
    items = []
    selectors = [custom_selector] if custom_selector else GENERIC_TITLE_SELECTORS

    for sel in selectors:
        links = response.css(sel)
        if not links:
            continue

        candidates = []
        for a in links:
            text = " ".join(a.css("::text").getall()).strip()
            href = a.attrib.get("href", "")
            if 5 < len(text) < 150 and href:
                pub_date = extract_item_date(a)
                candidates.append((text, response.urljoin(href), pub_date))

        if len(candidates) >= 3:
            for text, href, pub_date in candidates:
                item = ArticleItem()
                item["title"]    = text
                item["url"]      = href
                item["pub_date"] = pub_date
                item["source"]   = site_name
                items.append(item)
            break

    # 兜底：onclick="window.location.href='...'" 模式
    if not items:
        items = extract_onclick_items(response, site_name)

    return items


# onclick 正则：匹配 window.location.href='...' 或 location.href='...'
ONCLICK_RE = re.compile(r"""location\.href\s*=\s*['"]([^'"]+)['"]""")


def extract_onclick_items(response, site_name: str):
    """从带 onclick 的元素中提取链接和标题"""
    items = []
    candidates = []

    for el in response.css("[onclick]"):
        onclick = el.attrib.get("onclick", "")
        m = ONCLICK_RE.search(onclick)
        if not m:
            continue
        href = m.group(1).strip()
        if not href or href.startswith("javascript"):
            continue

        text = " ".join(el.css("::text").getall()).strip()
        text = re.sub(r'\s+', ' ', text).strip()

        if len(text) < 5 or len(text) > 300:
            continue

        pub_date = extract_item_date(el)
        candidates.append((text, response.urljoin(href), pub_date))

    if len(candidates) >= 3:
        for text, href, pub_date in candidates:
            item = ArticleItem()
            item["title"]    = text
            item["url"]      = href
            item["pub_date"] = pub_date
            item["source"]   = site_name
            items.append(item)

    return items


def extract_page_date(response) -> str:
    """页面级日期兜底，当条目级日期取不到时使用（只用语义选择器，避免误判）"""
    for sel in SEMANTIC_DATE_SELECTORS:
        val = response.css(sel).get("")
        d = parse_date_str(val.strip() if val else "")
        if d:
            return d
    return ""


def _dig(obj, path: str):
    """
    按点分路径从嵌套 dict/list 中取值。
    例: _dig(data, "data.list") → data["data"]["list"]
    """
    for key in path.split("."):
        if isinstance(obj, dict):
            obj = obj.get(key)
        else:
            return None
        if obj is None:
            return None
    return obj


def _build_api_payload(api_cfg: dict, page_index: int) -> dict:
    """构造 API 请求 body，自动注入 page_index 和随机数"""
    payload = dict(api_cfg.get("payload", {}))
    # 支持自定义分页字段名（默认 page_index）
    page_field = api_cfg.get("page_field", "page_index")
    payload[page_field] = page_index
    # 如果原始 payload 里有 randNumber 则刷新
    if "randNumber" in payload:
        payload["randNumber"] = random.random()
    return payload


class ListMonitorSpider(scrapy.Spider):
    name = "list_monitor"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.sites = load_sites_from_cache()
        self.total      = len(self.sites)
        self.done       = 0
        self.ok_count   = 0
        self.fail_count = 0
        self.item_count = 0
        self._progress_file = open("progress.log", "a", encoding="utf-8")
        from datetime import datetime
        self._progress_file.write(
            f"\n=== 运行时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ===\n"
        )
        self._progress_file.flush()
        self.logger.info(f"从本地缓存加载了 {self.total} 个站点")

    # ------------------------------------------------------------------
    # 启动：按站点类型分发请求
    # ------------------------------------------------------------------
    async def start(self):
        for site in self.sites:
            if site.get("api"):
                # API 类型站点：POST 到 JSON 接口
                yield self._make_api_request(site, page_index=1)
            else:
                # 普通 HTML 站点
                yield scrapy.Request(
                    url=site["url"],
                    callback=self.parse_list,
                    errback=self.on_error,
                    meta={
                        "site_name": site["name"],
                        "selector":  site.get("selector", ""),
                        "handle_httpstatus_list": [403, 404, 500],
                    },
                )

    # ------------------------------------------------------------------
    # API 站点：构造请求
    # ------------------------------------------------------------------
    def _make_api_request(self, site: dict, page_index: int) -> scrapy.Request:
        api = site["api"]
        payload = _build_api_payload(api, page_index)
        method  = api.get("method", "POST").upper()

        return scrapy.Request(
            url=api["url"],
            method=method,
            body=json.dumps(payload, ensure_ascii=False),
            headers={
                "Content-Type": "application/json",
                "Referer": site["url"],
                "User-Agent": (
                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                    "AppleWebKit/537.36 (KHTML, like Gecko) "
                    "Chrome/126.0.0.0 Safari/537.36"
                ),
            },
            callback=self.parse_api,
            errback=self.on_error,
            meta={
                "site":       site,
                "page_index": page_index,
                "handle_httpstatus_list": [403, 404, 500],
            },
        )

    # ------------------------------------------------------------------
    # API 站点：解析响应
    # ------------------------------------------------------------------
    def parse_api(self, response):
        site      = response.meta["site"]
        site_name = site["name"]
        api       = site["api"]
        fields    = api.get("fields", {})

        if response.status in (403, 404, 500):
            self.done += 1
            self.fail_count += 1
            self._log_progress(f"HTTP {response.status}", site_name)
            return

        try:
            data = response.json()
        except Exception as e:
            self.done += 1
            self.fail_count += 1
            self._log_progress(f"JSON解析失败", site_name)
            self.logger.warning(f"  ↳ {site['api']['url']} - {e}")
            return

        # 取列表
        items_path = fields.get("items", "data.list")
        raw_list   = _dig(data, items_path)

        if not raw_list or not isinstance(raw_list, list):
            self.done += 1
            self.fail_count += 1
            self._log_progress("API无条目", site_name)
            self.logger.debug(f"  ↳ items_path={items_path} 取值为空，响应: {str(data)[:200]}")
            return

        # 字段映射（支持自定义，默认值覆盖荆州模式）
        title_field = fields.get("title", "title")
        url_field   = fields.get("url",   "pub_url")
        date_field  = fields.get("date",  "rel_time")

        scrapy_items = []
        for row in raw_list:
            title = str(row.get(title_field, "")).strip()
            url   = str(row.get(url_field,   "")).strip()
            date  = parse_date_str(str(row.get(date_field, ""))) or ""

            if not title or not url or len(title) < 5:
                continue

            item = ArticleItem()
            item["title"]    = title
            item["url"]      = url
            item["pub_date"] = date
            item["source"]   = site_name
            scrapy_items.append(item)

        self.done += 1
        if not scrapy_items:
            self.fail_count += 1
            self._log_progress("API无有效条目", site_name)
            return

        self.ok_count   += 1
        self.item_count += len(scrapy_items)
        self._log_progress(f"+{len(scrapy_items)}条(API)", site_name)

        for item in scrapy_items:
            yield item

    # ------------------------------------------------------------------
    # 普通 HTML 站点：解析响应
    # ------------------------------------------------------------------
    def _log_progress(self, tag: str, site_name: str):
        pct = self.done * 100 // self.total if self.total else 0
        line = (
            f"[{self.done}/{self.total} {pct}%] {tag} [{site_name}]  "
            f"(ok={self.ok_count} fail={self.fail_count} items={self.item_count})"
        )
        print(line, flush=True)
        self._progress_file.write(line + "\n")
        self._progress_file.flush()

    def parse_list(self, response):
        site_name = response.meta["site_name"]

        if response.status in (403, 404, 500):
            self.done += 1
            self.fail_count += 1
            self._log_progress(f"HTTP {response.status}", site_name)
            return

        selector = auto_selector(response.url, response.meta["selector"])
        items    = extract_items(response, site_name, selector)

        self.done += 1
        if not items:
            self.fail_count += 1
            self._log_progress("无条目", site_name)
            return

        self.ok_count   += 1
        self.item_count += len(items)
        self._log_progress(f"+{len(items)}条", site_name)

        page_date = extract_page_date(response)

        for item in items:
            if not item["pub_date"]:
                item["pub_date"] = page_date
            yield item

    def on_error(self, failure):
        site_name = failure.request.meta.get(
            "site_name",
            failure.request.meta.get("site", {}).get("name", "?")
        )
        self.done += 1
        self.fail_count += 1
        self._log_progress("请求失败", site_name)
        self.logger.debug(f"  ↳ {failure.request.url} - {failure.value}")