#!/usr/bin/env python3
"""crawl_shifang_gggq.py - 什邡市人民政府-公告公示

站点: https://www.shifang.gov.cn/info/iList.jsp?node_id=GKsfs&cat_id=16389
CMS: 德阳站群 (与 deyang.gov.cn 同源)
列表: div#list_content > ul > li > i + a[title=完整标题] + span 日期
      链接 /gk/dtxx/gggq/ID.htm
分页: iList.jsp?node_id=GKsfs&cur_page=N&cat_id=16389, 共11页
详情: div.ewb-article-tt 标题 + meta PubDate 日期
      正文 div#_file_content_fck.ewb-article-detail:
        - HTML 文本型: 直接取内容
        - 纯 PDF 型: <pdfView src="...pdf"> → 提取 PDF 链接作为附件
附件: /wcm.files/upload/CMSshifang/...pdf 相对根路径 → BASE + href
标题: strip &middot;&nbsp; 实体前缀
"""
import sys, os, re, json, argparse
import requests
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
from datetime import datetime, timedelta

CUTOFF = (datetime.now() - timedelta(days=365 * 3)).strftime('%Y-%m-%d')
SITE_NAME = '什邡市-公告公示'
GROUP_NAME = '四川'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
}
BASE = 'https://www.shifang.gov.cn'
LIST_URL = BASE + '/info/iList.jsp?node_id=GKsfs&cat_id=16389'


def fetch(url):
    try:
        r = requests.get(url, timeout=(5, 20), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr)
        return ''


def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配"""
    m = re.search(open_tag_re, html)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:]):
        if mm.group(0).startswith('<div'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]


def fetch_list(page_idx):
    """列表页 (0=首页, N=cur_page=N)"""
    if page_idx == 0:
        url = LIST_URL
    else:
        url = LIST_URL + f'&cur_page={page_idx}'
    html = fetch(url)
    if not html:
        return []
    items = []
    for m in re.finditer(r'<li><i></i>\s*<a href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>\s*<span[^>]*>([^<]*)</span>', html, re.DOTALL):
        href, title_attr, title_html, date_text = m.group(1).strip(), m.group(2).strip(), m.group(3), m.group(4).strip()
        title = title_attr or re.sub(r'<[^>]+>', '', title_html).strip()
        title = re.sub(r'\s+', ' ', title).strip()
        # strip &middot;&nbsp; 实体前缀
        title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
        if len(title) < 4:
            continue
        url = urljoin(BASE, href)
        md = re.search(r'(\d{4}-\d{2}-\d{2})', date_text)
        pd = md.group(1) if md else ''
        items.append({'url': url, 'pub_date': pd, 'title': title})
    return items


def fetch_detail(url):
    """详情页: ewb-article-tt 标题 + _file_content_fck 正文 (HTML 或 pdfView)"""
    html = fetch(url)
    if not html:
        return '', '', '', []
    # 标题
    title = ''
    m = re.search(r'<div[^>]*class="[^"]*ewb-article-tt[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        title = re.sub(r'\s+', ' ', title).strip()
    if not title:
        m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]+)"', html)
        if m:
            title = m.group(1).strip()
    title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
    # 日期: meta PubDate
    date_text = ''
    m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]+)"', html)
    if m:
        md = re.search(r'(\d{4}-\d{2}-\d{2})', m.group(1))
        if md:
            date_text = md.group(1)
    # 正文: div#_file_content_fck.ewb-article-detail
    content = ''
    c = extract_balanced_div(html, r'<div[^>]*id="_file_content_fck"[^>]*class="[^"]*ewb-article-detail[^"]*"[^>]*>')
    if c.strip():
        content = c.strip()
    if not content.strip():
        c = extract_balanced_div(html, r'<div[^>]*class="[^"]*ewb-article-detail[^"]*"[^>]*>')
        if c.strip():
            content = c.strip()
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    content = re.sub(r'<style.*?</style>', '', content, flags=re.DOTALL | re.I)
    # 移除二维码区 (正文容器内部自带 <div id="qr_code">扫一扫...</div>)
    content = re.sub(r'<!--\s*二维码开始\s*-->.*?<!--\s*二维码结束\s*-->', '', content, flags=re.DOTALL)
    content = re.sub(r'<div[^>]*id="qr_code"[^>]*>.*?</div>\s*</div>', '', content, flags=re.DOTALL)
    content = re.sub(r'<div[^>]*id="qr_code"[^>]*>.*?</div>', '', content, flags=re.DOTALL)
    # 附件: pdfView src 内嵌 PDF
    atts = []
    pdf_view = ''
    m = re.search(r'<pdfView[^>]*src="([^"]+)"', content)
    if m:
        pdf_url = m.group(1).strip()
        if not pdf_url.startswith('http'):
            pdf_url = BASE + pdf_url if pdf_url.startswith('/') else urljoin(url, pdf_url)
        pdf_view = pdf_url
        atts.append({'title': title + ' (PDF原文)', 'url': pdf_url})
        # 纯 PDF 型: 正文保留 PDF 链接段落
        content = f'<p>附件：<a href="{pdf_url}">{title}（PDF原文）</a></p>'
    else:
        # HTML 型: 提取正文内附件
        for am in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', content, re.DOTALL):
            h, t = am.group(1), re.sub(r'<[^>]+>', '', am.group(2)).strip()
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps)$', h, re.I) or 'attach' in h.lower() or 'wcm.files' in h:
                if not h.startswith('http'):
                    h = BASE + h if h.startswith('/') else urljoin(url, h)
                if h not in [a['url'] for a in atts]:
                    atts.append({'title': t or '附件', 'url': h})
    # href/src 相对路径转绝对
    content = re.sub(r'href="(?!https?://|//|#|javascript:)([^"]+)"',
                     lambda mm: f'href="{BASE}{mm.group(1)}"' if mm.group(1).startswith('/') else f'href="{urljoin(url, mm.group(1))}"',
                     content)
    content = re.sub(r'src="(?!https?://|//)([^"]+)"',
                     lambda mm: f'src="{BASE}{mm.group(1)}"' if mm.group(1).startswith('/') else f'src="{urljoin(url, mm.group(1))}"',
                     content)
    # 清首尾空 p
    content = re.sub(r'^(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+', '', content)
    content = re.sub(r'(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+$', '', content)
    return title, date_text, content.strip(), atts


def run(max_pages=5):
    records = []
    seen = set()
    for pg in range(max_pages):
        items = fetch_list(pg)
        if not items:
            print(f"  Page {pg + 1}: 0 items (stop)")
            break
        print(f"  Page {pg + 1}: {len(items)} items")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, date_text, content, atts = fetch_detail(it['url'])
            if not content:
                print(f"    SKIP empty content: {it['url']}")
                continue
            if not title:
                title = it['title']
            pd = date_text or it['pub_date']
            records.append({
                'title': title, 'url': it['url'], 'source_url': it['url'],
                'pub_date': pd, 'site_name': SITE_NAME, 'content': content,
                'summary': '', 'attachments': json.dumps(atts, ensure_ascii=False) if atts else '[]',
                'group_name': GROUP_NAME,
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "shifang_gggq")
    return len(valid)


if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    n = run(args.pages)
    print(f"[RESULT] {SITE_NAME}: {n} items")