#!/usr/bin/env python3
"""crawl_ldnews_hjbh.py - 娄底新闻网-环境保护 (湖南娄底)

站点: https://m.ldnews.cn/pccategory/100.shtml (移动端栏目页)
CMS: cmstop (AJAX JSON API 分页)
列表: ⚠️ 非静态 HTML — POST https://m.ldnews.cn/pccategory/100.shtml
      data: {catid:100, page:N, pagesize:20} → json{total:780, data:[{title,url,published,...}]}
      首页静态渲染 P1 (20条), AJAX 从 page=2 起; total=780 → 39页
      CUTOFF 边界: P6 末条 2023-08-09 (3年窗口=前6页≈120条)
      注意: url 有两种格式 — m.ldnews.cn/pcarticle/ID.shtml 与 news.ldnews.cn/news/YYYYMM/ID.shtml
详情: PC端 http 301→https; 标题 h1.ui-article-title / <title> 分割; 日期 <time> 文本
      正文双容器: PC端 div.right_body 平衡div (尾部登录区清洗) | 移动端 div#content-show
"""
import sys, os, requests, re, argparse, json
from datetime import datetime, timedelta
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365 * 3)).strftime('%Y-%m-%d')
SITE_NAME = '娄底新闻网-环境保护'
GROUP_NAME = '湖南'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Referer': 'https://m.ldnews.cn/pccategory/100.shtml',
}
BASE = 'https://m.ldnews.cn'
LIST_URL = BASE + '/pccategory/100.shtml'
CATID = 100


def fetch(url, post_data=None):
    try:
        if post_data:
            r = requests.post(url, data=post_data, timeout=(5, 20), headers=HEADERS, verify=False)
        else:
            r = requests.get(url, timeout=(5, 20), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr)
        return ''


def extract_items(html, page_no):
    """列表条目: AJAX JSON API 解析 (首页静态 HTML 用正则, 分页用 JSON)"""
    items = []
    if page_no == 1:
        # 首页静态渲染的列表 (div.ui-mod-picsummary > a[href] > h3.ui-title)
        for m in re.finditer(r'<a href="([^"]+)"[^>]*>\s*(?:<img[^>]*>)?\s*<h3 class="ui-title">(.*?)</h3>', html, re.DOTALL):
            href, title = m.group(1), re.sub(r'<[^>]+>', '', m.group(2)).strip()
            if len(title) < 4:
                continue
            url = href if href.startswith('http') else urljoin(BASE, href)
            url = url.replace('http://', 'https://')
            items.append({'url': url, 'title': title})
        return items
    # 分页: POST API
    try:
        r = requests.post(LIST_URL, data={'catid': CATID, 'page': page_no, 'pagesize': 20},
                          timeout=(5, 20), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
        d = json.loads(r.text)
        for it in d.get('data', []):
            title = (it.get('title') or '').replace('\u200b', '').strip()
            url = (it.get('url') or '').replace('http://', 'https://')
            if not title or not url or len(title) < 4:
                continue
            items.append({'url': url, 'title': title})
    except Exception as e:
        print(f"[WARN] API page {page_no} failed: {e}", file=sys.stderr)
    return items


def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配 (大小写不敏感)"""
    m = re.search(open_tag_re, html, re.I)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:], re.I):
        if mm.group(0).startswith('<div') or mm.group(0).startswith('<DIV'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]


def fetch_detail(url):
    html = fetch(url)
    if not html:
        return '', '', ''
    # 标题
    title = ''
    m = re.search(r'<h1[^>]*class="[^"]*ui-article-title[^"]*"[^>]*>(.*?)</h1>', html, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    if not title:
        m = re.search(r'<title>\s*([^<]+?)\s*</title>', html, re.DOTALL)
        if m:
            title = re.sub(r'\s+', '', m.group(1)).strip()
            title = re.split(r'_娄底新闻网|_|[-–—]', title)[0].strip()
    title = title.replace('\u200b', '').strip()
    title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
    # 日期: <time>2025-09-17 16:54</time> 或文本
    date_text = ''
    m = re.search(r'<time[^>]*>(\d{4}-\d{2}-\d{2})', html)
    if m:
        date_text = m.group(1)
    if not date_text:
        m = re.search(r'(\d{4}-\d{2}-\d{2})\s+\d{2}:\d{2}', html)
        if m:
            date_text = m.group(1)
    if not date_text:
        m = re.search(r'<meta[^>]*name="[Pp]ub[Dd]ate"[^>]*content="([^"]*)"', html)
        if m:
            md = re.search(r'(\d{4}-\d{2}-\d{2})', m.group(1))
            if md:
                date_text = md.group(1)
    # 正文: 双容器 — PC端 right_body | 移动端 content-show
    content = ''
    c = extract_balanced_div(html, r'<div[^>]*class="[^"]*right_body[^"]*"[^>]*>')
    if c.strip():
        content = c.strip()
    else:
        m = re.search(r'<div[^>]*id="[^"]*content-show[^"]*"[^>]*>(.*?)</div>\s*(?:</section>|<section)', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    # 清洗: 移除尾部登录区/评论相关 (PC端)
    for kw in ['用户登录', '用户名', '密码', '下次自动登录', '没有账号', '马上注册',
               '使用第三方账号登录', 'QQ账户', '新浪微博', '腾讯微博', '验证码']:
        idx = content.find(kw)
        if idx > 0 and idx > len(content) * 0.5:
            content = content[:idx]
            break
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    return title, date_text, content


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=6)
    args = ap.parse_args()

    items = []
    seen = set()
    for page_no in range(1, args.pages + 1):
        post = {'catid': CATID, 'page': page_no, 'pagesize': 20} if page_no > 1 else None
        html = fetch(LIST_URL, post)
        if not html:
            continue
        page_items = extract_items(html, page_no)
        before = len(items)
        for it in page_items:
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            items.append(it)
        print(f"  page {page_no}: +{len(items) - before} items (total {len(items)})")
        if not page_items:
            break  # 空页自动停

    records = []
    for it in items:
        title, date_text, content = fetch_detail(it['url'])
        if not title:
            title = it['title']
        if not content:
            print(f"    SKIP empty content: {it['url']}")
            continue
        pd = date_text
        if pd and pd < CUTOFF:
            continue
        # 附件提取
        atts = []
        for am in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', content, re.DOTALL):
            h, t = am.group(1), re.sub(r'<[^>]+>', '', am.group(2)).strip()
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps)$', h, re.I) or 'download' in h.lower():
                atts.append({'title': t or '附件', 'url': h})
        records.append({
            'title': title, 'url': it['url'], 'source_url': it['url'],
            'pub_date': pd, 'site_name': SITE_NAME, 'content': content,
            'summary': '', 'attachments': json.dumps(atts, ensure_ascii=False) if atts else '[]',
            'group_name': GROUP_NAME,
        })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "ldnews_hjbh")
    return len(valid)


if __name__ == '__main__':
    main()