"""通用页面解析器 - 处理各种政府网站的不同HTML结构"""

import re
import html
from urllib.parse import urljoin, urlparse
from scrapy.selector import Selector, SelectorList
from .ljobx_rules import LJOBX_RULES
from .ljobx_list_rules import LJOBX_LIST_RULES


class PageParser:
    """解析政府网站的列表页和详情页"""

    # ljobx字段映射 - 从LocoySpider导出规则提取
    # 完整规则表在 ljobx_rules.py (1051条)
    # (title_start, title_end, content_start, content_end)
    LJOBX_RULES = LJOBX_RULES

    # ljobx列表页规则 - 从LocoySpider导出
    # (area_start, area_end, url_must, url_forbid, pagination)
    LJOBX_LIST_RULES = LJOBX_LIST_RULES

    # 排除导航区关键词 — 匹配class/id
    EXCLUDE_REGIONS = [
        'nav', 'menu', 'header', 'footer', 'sidebar', 'topbar',
        'toolbar', 'crumb', 'breadcrumb', 'pagination', 'pagebar',
        'head', 'foot', 'slide', 'carousel', 'banner', 'ad',
        'search', 'login', 'register', 'friendlink', 'link',
        'copy', 'copyright', 'bottom', 'subnav', 'sub-nav',
    ]

    # 政府网站常见页脚/导航噪声文本（匹配即过滤）
    GOV_NOISE_TEXTS = [
        'ICP', '备案号', '版权所有', 'All Rights Reserved', '技术支持',
        '主办单位', '承办单位', '协办单位', '建设单位', '维护单位',
        '网站标识码', '站点地图', '关于我们', '联系我们', '法律声明',
        '隐私政策', '网站声明', '网站纠错', '无障碍浏览', '长者模式',
        '政府网站找错', '我为网站找错',
        '国务院办公厅', '各省、自治区', '各市、县',
        '电话：', '传真：', '邮箱：', '地址：', '邮编：',
        '今日访问', '总访问量', '访问统计', '在线人数',
        '分享到', '微信', '微博', '扫码', '扫一扫',
        '推荐使用', '最佳分辨率', '浏览器',
        '公示日期', '公示时间', '发布机构', '发文日期',
        '信息分类', '索引号', '主题分类', '体裁分类', '服务对象',
        '信息来源', '生成日期', '有效期', '公开方式', '公开时限',
        '关键词：', '相关附件：', '附件下载', '附件：',
    ]

    # 政府公告正文特征词（用于评分判断是否为真实正文）
    GOV_CONTENT_KEYWORDS = [
        '根据', '按照', '为加强', '为进一步', '为规范', '为保障',
        '我局', '我县', '我单位', '本公司', '我公司',
        '公示', '公告', '通知', '批复', '核准', '备案',
        '环境影响', '环境保护', '污染防治', '污染治理',
        '建设项目', '工程', '招标', '投标', '中标',
        '验收', '竣工', '投产', '试运行',
        '安全评价', '风险评估', '应急预案',
        '公众参与', '征求意见', '听证',
        '现予以', '特此', '经研究', '决定',
        '符合', '要求', '标准', '规定', '条件',
        '有关单位', '社会各界', '广大群众',
        '如有异议', '如有意见', '如有问题', '如有疑问',
        '请在公示期内', '公示期',
        '联系人', '联系电话', '监督电话',
    ]

    # 导航特征链接文本 — 文章列表里不会出现
    NAV_TEXTS = [
        '首页', '返回', '关于', '联系', '搜索', '登录', '注册',
        '帮助', '网站地图', '设为首页', '收藏本站', '加入收藏',
        '法律声明', '隐私政策', '网站导航', '友情链接',
    ]

    # 导航特征URL片段 — 文章详情URL不会包含
    NAV_URL_PATTERNS = [
        r'/about', r'/contact', r'/login', r'/register',
        r'/search', r'/sitemap', r'/help', r'/tag',
    ]

    # 文章链接特征模式 — 得分权重
    ARTICLE_URL_PATTERNS = [
        r'/content/', r'/article/', r'/news/', r'/detail/',
        r'/info/', r'/show/', r'/view/', r'/read/',
        r'20\d{2}/',           # 年份路径 e.g. /2026/
        r'id=\d+',             # 数字ID参数
        r'\d{5,}',             # 5位以上数字
        r'\.html?\b',          # .html / .htm 结尾
    ]

    LIST_ITEM_SELECTORS = [
        # 高精度：带xxgk/信息公开类
        '//ul[contains(@class, "xxgk")]//li[.//a[not(starts-with(@href, "#"))]]',
        '//div[contains(@class, "xxgk")]//li[.//a[not(starts-with(@href, "#"))]]',
        # 带newslist/news-list等明确列表类
        '//*[contains(@class, "news-list") or contains(@class, "news_list")]//li[.//a]',
        '//*[contains(@class, "newslist")]//li[.//a]',
        # 内容区article类容器
        '//*[contains(@class, "article") or contains(@class, "content")]//li[.//a]',
        '//*[contains(@class, "main-con") or contains(@class, "maincon")]//li[.//a]',
        '//*[contains(@class, "right-con") or contains(@class, "rightcon")]//li[.//a]',
        # 带list类但排除导航层
        '//*[contains(@class, "list")][not(ancestor::*[@class and (contains(@class, "nav") or contains(@class, "menu") or contains(@class, "head") or contains(@class, "foot"))])]//li[.//a]',
        # tr模式但排除菜单/导航table
        '//table[not(contains(@class, "nav") or contains(@class, "menu") or contains(@class, "head") or contains(@class, "top"))]//tr[.//a[not(starts-with(@href, "#"))]]',
        # li中有date类span（高精度日期标记）
        '//li[.//span[contains(@class, "date")] or .//span[contains(@class, "time")] or .//em[contains(@class, "date")]][.//a]',
    ]

    # 列表容器选择器 — 按容器分组判定的备选
    CONTAINER_SELECTORS = [
        '//ul[not(ancestor::nav) and not(ancestor::*[contains(@class,"nav") or contains(@class,"menu") or contains(@class,"header") or contains(@class,"footer")])]',
        '//ol[not(ancestor::nav) and not(ancestor::*[contains(@class,"nav") or contains(@class,"menu") or contains(@class,"header") or contains(@class,"footer")])]',
        '//div[not(ancestor::nav) and not(ancestor::*[contains(@class,"nav") or contains(@class,"menu") or contains(@class,"header") or contains(@class,"footer")])][descendant::li[.//a]]',
        '//table[not(ancestor::nav) and not(ancestor::*[contains(@class,"nav") or contains(@class,"menu") or contains(@class,"header") or contains(@class,"footer")])]',
    ]

    TITLE_IN_ITEM = [
        './/a/text()',
        './/a/@title',
        './/h2/a/text()',
        './/h3/a/text()',
        './/span[contains(@class, "title")]/a/text()',
        './/span[contains(@class, "title")]/a/@title',
    ]

    DATE_IN_ITEM = [
        './/span[contains(@class, "date")]/text()',
        './/span[contains(@class, "time")]/text()',
        './/em[contains(@class, "date")]/text()',
        './/em[contains(@class, "time")]/text()',
        './/td[last()]/text()',
        './/td[2]/text()',
        './/text()',
    ]

    DATE_PATTERNS = [
        r'(20\d{2})[-/](\d{1,2})[-/](\d{1,2})',
        r'(\d{4})年(\d{1,2})月(\d{1,2})日',
    ]

    MIN_TITLE_LEN = 6

    def __init__(self, base_url=''):
        self.base_url = base_url

    def _filter_text_noise(self, text):
        """过滤中文政府网站噪声文本（页脚、导航、版权等）"""
        if not text:
            return ''
        lines = text.split('\n')
        clean_lines = []
        for line in lines:
            line = line.strip()
            if not line:
                continue
            # 按行匹配噪声
            is_noise = False
            for noise in self.GOV_NOISE_TEXTS:
                if noise in line:
                    is_noise = True
                    break
            if not is_noise:
                clean_lines.append(line)
        merged = ' '.join(clean_lines)
        # 去除多余空白
        merged = re.sub(r'\s+', ' ', merged).strip()
        return merged

    def _score_text_quality(self, text):
        """评分：文本越像政府公告正文，分数越高"""
        if not text or len(text) < 30:
            return -100

        score = 0
        # 长度加分
        if len(text) > 500:
            score += 20
        elif len(text) > 200:
            score += 10
        elif len(text) > 100:
            score += 5

        # 政府正文特征词加分
        keyword_hits = 0
        for kw in self.GOV_CONTENT_KEYWORDS:
            if kw in text:
                keyword_hits += 1
        score += keyword_hits * 5

        # 中文占比（真实正文中文占比高）
        chinese_chars = len(re.findall(r'[\u4e00-\u9fff]', text))
        total_chars = len(text.strip())
        chinese_ratio = chinese_chars / max(total_chars, 1)
        if chinese_ratio > 0.7:
            score += 30
        elif chinese_ratio > 0.4:
            score += 15

        # 噪声文本扣分
        noise_hits = 0
        for noise in self.GOV_NOISE_TEXTS:
            if noise in text:
                noise_hits += 1
        score -= noise_hits * 8

        # 短行占比（大量短行 = 导航/菜单）
        lines = text.split('\n')
        short_lines = sum(1 for l in lines if len(l.strip()) < 10 and l.strip())
        if lines:
            short_ratio = short_lines / len(lines)
            if short_ratio > 0.5:
                score -= 20
            elif short_ratio > 0.3:
                score -= 10

        return score

    def _xtext(self, node, selector):
        """安全提取文本 - 兼容Scrapy Selector和lxml"""
        result = node.xpath(selector)
        if isinstance(result, (SelectorList, list)) and len(result) > 0:
            # Scrapy Selector
            if hasattr(result[0], 'get'):
                return [r.get() for r in result]
            # lxml element or string
            return [str(r) if hasattr(r, 'tag') else r for r in result]
        return []

    def _xfirst(self, node, selector):
        """取第一个匹配的文本值"""
        texts = self._xtext(node, selector)
        return texts[0] if texts else ''

    def _score_items(self, items, response=None):
        """多维度评分 — 判断一组item是否来自真实文章列表"""
        if not items:
            return 0

        total = len(items)
        if total == 0:
            return 0

        with_date = sum(1 for i in items if i['date'])
        long_title = sum(1 for i in items if len(i['title']) > 10)
        nav_texts = sum(1 for i in items if any(
            nt in i['title'] for nt in self.NAV_TEXTS))
        article_urls = 0
        nav_urls = 0
        if response:
            for i in items:
                url = i.get('url', '')
                if any(re.search(p, url) for p in self.ARTICLE_URL_PATTERNS):
                    article_urls += 1
                if any(re.search(p, url) for p in self.NAV_URL_PATTERNS):
                    nav_urls += 1

        date_density = with_date / total

        # 核心评分规则
        score = 0

        # 1. 日期密度 — 文章列表通常>=30%有日期
        if date_density >= 0.8:
            score += 50
        elif date_density >= 0.5:
            score += 30
        elif date_density >= 0.2:
            score += 10
        else:
            score -= 20  # 低于20%有日期的不太可能是文章列表

        # 2. 标题质量
        score += long_title * 3  # 每个长标题+3
        score += max(0, total - 3) * 2  # 超过3条额外加分

        # 3. 导航文本扣分 — 列表中导航文本越多越不像文章
        score -= nav_texts * 10

        # 4. URL模式分析
        if response:
            # 文章链接占比越高越好
            article_ratio = article_urls / total if total > 0 else 0
            if article_ratio >= 0.5:
                score += 40
            elif article_ratio >= 0.2:
                score += 15

            # 导航链接占比高 = 扣分
            nav_ratio = nav_urls / total if total > 0 else 0
            if nav_ratio >= 0.3:
                score -= 30

        # 5. 数量太少 (<3条) 且没有日期 → 大概率不是文章列表
        if total < 3 and not with_date:
            score -= 50

        return score

    def _extract_from_nodes(self, nodes, response):
        items = []
        for node in nodes:
            try:
                title = self._extract_title(node)
                url = self._extract_url(node, response)
                date = self._extract_date(node)

                if title and url and len(title.strip()) > self.MIN_TITLE_LEN:
                    items.append({
                        'title': title.strip(),
                        'url': url,
                        'date': date or '',
                    })
            except Exception:
                continue
        return items

    def parse_list_page(self, response):
        """解析列表页 — 两阶段：先精确选择器，再容器分组兜底"""
        from urllib.parse import urlparse
        
        # ljobx列表规则：提取区域 + URL过滤 + 分页限制
        domain = urlparse(response.url).netloc
        area_start, area_end, url_must, url_forbid, pagination = self.get_list_rules(domain)
        
        # 如果定义了列表区域，缩小解析范围
        if area_start:
            sub_html = self.extract_list_region(response.text, area_start, area_end)
            if sub_html != response.text:
                from scrapy.selector import Selector
                response._cached_selector = Selector(text=sub_html)
        
        best_items = []
        best_score = -999

        # 阶段1：精确选择器匹配
        for selector in self.LIST_ITEM_SELECTORS:
            nodes = response.xpath(selector)
            if not nodes:
                continue

            items = self._extract_from_nodes(nodes, response)
            if not items:
                continue

            score = self._score_items(items, response)
            if score > best_score:
                best_items = items
                best_score = score

        # 阶段2：如果阶段1没有分数>0的结果，用容器分组法
        if best_score <= 0:
            container_items = self.parse_list_page_by_container(response)
            c_score = self._score_items(container_items, response)
            if c_score > best_score:
                best_items = container_items
                best_score = c_score

        # ljobx URL过滤
        if url_must or url_forbid:
            best_items = self.filter_items_by_rules(best_items, url_must, url_forbid)

        # ljobx分页限制
        if pagination and pagination != (0, 0):
            self.ljobx_pagination = pagination  # 供spider使用

        # 最终过滤：分数必须>0才算有效，否则返回空
        if best_score <= 0:
            return []

        return best_items[:50]

    def parse_list_page_by_container(self, response):
        """容器分组法 — 按<ul>/<ol>/<div>/<table>分组，每组独立提取"""
        all_groups = []

        for selector in self.CONTAINER_SELECTORS:
            containers = response.xpath(selector)
            for container in containers:
                # 跳过导航区容器
                cont_class = ''.join(container.xpath('@class | @id').extract() if hasattr(
                    container.xpath('@class | @id'), 'extract') else [
                    str(x) for x in container.xpath('@class | @id')]).lower()
                if any(kw in cont_class for kw in self.EXCLUDE_REGIONS):
                    continue
                # 跳过祖先已是导航区的
                ancestors = container.xpath('ancestor::*[contains(@class, "nav") or contains(@class, "menu") or contains(@class, "header") or contains(@class, "footer")]')
                if ancestors:
                    continue

                # 提取此容器下所有li中的链接项
                if container.root.tag in ('ul', 'ol'):
                    # 直接子li
                    items = self._extract_from_nodes(container.xpath('./li'), response)
                elif container.root.tag == 'table':
                    items = self._extract_from_nodes(
                        container.xpath('.//tr[.//a[not(starts-with(@href, "#")) and not(contains(@href, "javascript"))]]'),
                        response)
                else:
                    # div: 提取所有子li
                    items = self._extract_from_nodes(
                        container.xpath('.//li[.//a[not(starts-with(@href, "#")) and not(contains(@href, "javascript"))]]'),
                        response)

                if items:
                    score = self._score_items(items, response)
                    all_groups.append((items, score))

        if not all_groups:
            return []

        # 按分数降序取最优组
        all_groups.sort(key=lambda x: x[1], reverse=True)
        return all_groups[0][0]

    def _extract_title(self, node):
        for sel in self.TITLE_IN_ITEM:
            texts = self._xtext(node, sel)
            for t in texts:
                t = t.strip()
                if len(t) > self.MIN_TITLE_LEN:
                    return t

        all_texts = self._xtext(node, './/text()')
        texts = [t.strip() for t in all_texts if t.strip() and len(t.strip()) > self.MIN_TITLE_LEN]
        if texts:
            return max(texts, key=len)
        return ''

    def _extract_url(self, node, response):
        href = self._xfirst(node, './/a/@href')
        if not href:
            return ''
        href = href.strip()
        if not href or href.startswith('#') or href.startswith('javascript'):
            return ''
        if href.startswith('http'):
            return href
        return urljoin(response.url, href)

    def _extract_date(self, node):
        for sel in self.DATE_IN_ITEM:
            texts = self._xtext(node, sel)
            for t in texts:
                t = t.strip()
                for pattern in self.DATE_PATTERNS:
                    match = re.search(pattern, t)
                    if match:
                        y, m, d = match.groups()
                        return f'{y}-{m.zfill(2)}-{d.zfill(2)}'

        full_text = ''.join(self._xtext(node, './/text()'))
        for pattern in self.DATE_PATTERNS:
            match = re.search(pattern, full_text)
            if match:
                y, m, d = match.groups()
                return f'{y}-{m.zfill(2)}-{d.zfill(2)}'
        return ''

    def _is_generic_rule(self, content_start, content_end):
        """Check if rule is generic full-page extraction."""
        generic = ['<(*)', '<html', '</html', '<body', '</body']
        for g in generic:
            if g in content_start or g in content_end:
                return True
        return False

    def _extract_full_body(self, html_text):
        """Extract all text content from body, stripping HTML/JS/CSS and filtering noise."""
        # Remove scripts and styles
        clean = re.sub(r'<script[^>]*>.*?</script>', '', html_text, flags=re.DOTALL)
        clean = re.sub(r'<style[^>]*>.*?</style>', '', clean, flags=re.DOTALL)
        # Remove HTML tags, convert to line-broken text
        clean = re.sub(r'<[^>]+>', '\n', clean)
        # Unescape HTML entities
        clean = html.unescape(clean)
        # Remove non-content leading chars
        clean = re.sub(r'^[^a-zA-Z\u4e00-\u9fff0-9]+', '', clean)
        # Filter noise lines
        clean = self._filter_text_noise(clean)
        # Remove excessive whitespace
        clean = re.sub(r'\s+', ' ', clean).strip()
        return clean

    def get_list_rules(self, domain):
        """Get ljobx list page rules for a domain: (area_start, area_end, url_must, url_forbid, pagination)."""
        return self.LJOBX_LIST_RULES.get(domain, ('', '', '', '', (1, 3)))

    def extract_list_region(self, html_text, area_start, area_end):
        """Extract HTML between area_start and area_end markers."""
        if not area_start or not area_end:
            return html_text
        pattern = re.escape(area_start) + '(.*?)' + re.escape(area_end)
        m = re.search(pattern, html_text, re.DOTALL | re.IGNORECASE)
        if m:
            return '<html><body>' + m.group(1) + '</body></html>'
        return html_text

    def filter_items_by_rules(self, items, url_must='', url_forbid=''):
        """Filter list items by URL must/forbid patterns."""
        if not url_must and not url_forbid:
            return items
        filtered = []
        for item in items:
            url = item.get('url', '')
            if url_must and url_must not in url:
                continue
            if url_forbid and url_forbid in url:
                continue
            filtered.append(item)
        return filtered

    def _extract_between(self, html_text, start_str, end_str):
        """Extract text between two HTML strings using regex."""
        pattern = re.escape(start_str) + '(.*?)' + re.escape(end_str)
        m = re.search(pattern, html_text, re.DOTALL | re.IGNORECASE)
        if m:
            text = m.group(1)
            # Strip HTML tags
            clean = re.sub(r'<[^>]+>', '', text)
            # Remove leading/trailing non-text (>, whitespace, etc.)
            clean = re.sub(r'^[^a-zA-Z\u4e00-\u9fff0-9]+', '', clean)
            clean = re.sub(r'\s+', ' ', clean).strip()
            return clean
        return ''

    def extract_by_ljobx_rules(self, response):
        """Try ljobx field mapping rules first. Returns (title, content) or (None, None)."""
        from urllib.parse import urlparse
        domain = urlparse(response.url).netloc

        rules = self.LJOBX_RULES.get(domain)
        if not rules:
            return None, None

        title_start, title_end, content_start, content_end = rules

        # Handle non-text responses (PDFs, etc.)
        try:
            html_text = response.text
        except AttributeError:
            return None, None  # Non-text response, fall back to other parsers

        # Extract content
        if self._is_generic_rule(content_start, content_end):
            # Generic <(*)~</html> → full body extraction
            content = self._extract_full_body(html_text)
        else:
            # Precise StartStr/EndStr
            content = self._extract_between(html_text, content_start, content_end)

        # Extract title
        title = ''
        if title_start and title_end and not self._is_generic_rule(title_start, title_end):
            title = self._extract_between(html_text, title_start, title_end)

        return title, content

    def parse_detail_page(self, response):
        # Try ljobx field mapping rules first
        ljobx_title, ljobx_content = self.extract_by_ljobx_rules(response)
        content_selectors = [
            '//div[contains(@class, "content")]',
            '//div[contains(@class, "article")]',
            '//div[contains(@class, "text")]',
            '//div[@id="content"]',
            '//div[@class="main"]',
            '//div[contains(@class, "TRS")]',
            '//div[contains(@class, "news-content")]',
            '//div[contains(@class, "article-content")]',
            '//div[contains(@class, "detail")]',
            '//div[contains(@class, "info-content")]',
            '//div[@class="ewb-article"]',
            '//div[@class="bt_content"]',
            '//div[@class="con_text"]',
            '//div[@class="conTxt"]',
            '//div[contains(@class, "maintext")]',
            '//div[@id="UCAP-CONTENT"]',
            '//div[contains(@id, "zoom")]',
            '//article',
            '//article[contains(@class, "articleCon")]',
            # Additional selectors for ASP/simple CMS sites
            '//td[@class="content"]',
            '//td[@id="content"]',
            '//div[@id="article"]',
            '//div[@class="article"]',
            '//div[@class="arc_body"]',
            '//div[@class="show_content"]',
            '//div[contains(@class, "showtext")]',
            '//div[contains(@class, "newstext")]',
            '//div[contains(@class, "bodytext")]',
            '//div[contains(@class, "page-content")]',
            '//div[contains(@class, "page_content")]',
            '//div[contains(@class, "article-body")]',
            '//div[@class="entry-content"]',
            '//div[contains(@class, "post-content")]',
            # ASP.NET sites
            '//span[contains(@id, "lblcontent")]',
            '//span[contains(@id, "Content")]',
            # Chinese government sites
            '//div[contains(@class, "txtcontent-div")]',
            '//div[contains(@class, "nry-info")]',
            '//div[contains(@class, "TRS_Editor")]',
            # iframe content pages
            '//div[@class="ewb-article"]//td',
        ]

        best_text = ''
        best_score = -1000
        for selector in content_selectors:
            nodes = response.xpath(selector)
            if nodes:
                texts = self._xtext(nodes[0], './/text()') if isinstance(nodes, (SelectorList, list)) else []
                if not texts and hasattr(nodes[0], 'xpath'):
                    texts = self._xtext(nodes[0], './/text()')
                clean = ' '.join(t.strip() for t in texts if t.strip())
                if clean:
                    clean = html.unescape(clean)
                    clean = self._filter_text_noise(clean)
                    score = self._score_text_quality(clean)
                    if score > best_score:
                        best_text = clean
                        best_score = score

        # ljobx rule — score it too
        ljobx_clean = ''
        ljobx_score = -1000
        if ljobx_content:
            ljobx_clean = html.unescape(ljobx_content)
            ljobx_clean = self._filter_text_noise(ljobx_clean)
            ljobx_score = self._score_text_quality(ljobx_clean)

        # Pick the best quality content
        if ljobx_score > best_score and ljobx_score > 0:
            return ljobx_clean[:5000]
        if best_score > 0:
            return best_text[:5000]
        
        # Phase 2: Try to find content by excluding navigation elements
        candidates = response.xpath('//div | //section | //td | //article | //main')
        best_text = ''
        best_score = -1000
        for node in candidates:
            class_text = ''.join(node.xpath('@class | @id').extract() if hasattr(node.xpath('@class | @id'), 'extract') else [str(x) for x in node.xpath('@class | @id')]).lower()
            if any(kw in class_text for kw in ['nav', 'menu', 'header', 'footer', 'sidebar',
                                                 'top', 'bottom', 'banner', 'head', 'foot',
                                                 'toolbar', 'search', 'pagination', 'crumb',
                                                 'breadcrumb', 'slide', 'carousel', 'ad']):
                continue
            texts = self._xtext(node, './/text()')
            clean = ' '.join(t.strip() for t in texts if t.strip())
            if clean:
                clean = html.unescape(clean)
                clean = self._filter_text_noise(clean)
                score = self._score_text_quality(clean)
                if score > best_score:
                    best_text = clean
                    best_score = score

        if best_score > 0:
            return best_text[:5000]

        # Phase 3: Fallback — body text with aggressive noise removal
        texts = self._xtext(response, '//body//text()')
        meaningful = [t.strip() for t in texts if t.strip() and len(t.strip()) > 4]
        clean = '\n'.join(meaningful)
        clean = html.unescape(clean)
        clean = self._filter_text_noise(clean)
        score = self._score_text_quality(clean)
        if score > 0:
            return clean[:5000]

        # Last resort: if ljobx found something (even low quality)
        if ljobx_clean and len(ljobx_clean) > 50:
            return ljobx_clean[:5000]
        
        return ''

    def find_next_page(self, response):
        patterns = [
            '//a[contains(text(), "下一页")]/@href',
            '//a[contains(text(), "下页")]/@href',
            '//a[contains(text(), "next")]/@href',
            '//a[contains(@class, "next")]/@href',
            '//a[contains(@class, "page_next")]/@href',
            '//div[contains(@class, "page")]//a[last()]/@href',
            '//a[contains(@href, "page=2")]/@href',
            '//a[contains(@href, "index_2")]/@href',
        ]

        for pattern in patterns:
            href = self._xfirst(response, pattern)
            if href and href.strip() and 'javascript' not in href:
                next_url = urljoin(response.url, href.strip())
                if next_url != response.url:
                    return next_url

        url = response.url
        match = re.search(r'page[_=](\d+)', url)
        if match:
            current = int(match.group(1))
            if current < 5:
                s = match.group(0)
                prefix = url[:match.start()] + s[:4]
                next_url = url[:match.start()] + s[:4] + str(current + 1) + url[match.end():]
                if next_url != url:
                    return next_url
        return None

    def extract_attachments(self, response):
        links = []
        for ext in ['.pdf', '.doc', '.docx']:
            for link in response.css(f'a[href$="{ext}"]'):
                href = link.attrib.get('href', '')
                text = link.css('::text').get('')
                if href:
                    links.append({
                        'name': text.strip(),
                        'url': urljoin(response.url, href),
                    })
        return links