#!/usr/bin/env python3
"""crawl_rizhao_sthjj.py - 日照市生态环境局 (2栏目: 行政许可/环境管理)

站点: http://sthjj.rizhao.gov.cn/col/col179934/index.html (行政许可)
      http://sthjj.rizhao.gov.cn/col/col353302/index.html (环境管理)
CMS: 大汉版通 jcms1 + jpage 动态分页
列表: table > tr > td > a.bt_link[title=标题](href=/art/Y/M/D/art_{col}_ID.html) + b.dateqi(YYYY-MM-DD)
分页API: POST /module/web/jpage/dataproxy.jsp
         startrecord/endrecord/perpage/unitid/webid/path/webname/col/columnid/sourceContentType/permissiontype
         (返回XML: <record><![CDATA[<tr>...]]></record>)
栏目(--col):
  179934 = 行政许可 (totalRecord=466)
  353302 = 环境管理 (totalRecord=626)
详情: <title> 标题 + div#zoom 正文
修复(2026-08-08): ①zoom 容器非贪婪(.*?)</div> 在表格嵌套div处截断 → 清单/名录类正文表格全丢
②改用平衡div匹配 ③href/src 相对路径转绝对(含 /module/download/downfile.jsp 附件)
"""
import sys, os, requests, re, argparse
from datetime import datetime, timedelta
from urllib.parse import quote

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'}
BASE = 'http://sthjj.rizhao.gov.cn'
WEBID = '157'
UNITID = '758780'
PERPAGE = 20

COLUMNS = {
    '179934': {'name': '日照市生态环境局-行政许可', 'columnid': '179934', 'total': 466},
    '353302': {'name': '日照市生态环境局-环境管理', 'columnid': '353302', 'total': 626},
}

def fetch_list(col, page_no):
    colid = COLUMNS[col]['columnid']
    start = (page_no - 1) * PERPAGE + 1
    end = page_no * PERPAGE
    url = BASE + '/module/web/jpage/dataproxy.jsp'
    data = {
        'startrecord': start, 'endrecord': end, 'perpage': PERPAGE,
        'unitid': UNITID, 'webid': WEBID,
        'path': BASE + '/', 'webname': quote('日照市生态环境局'),
        'col': 1, 'columnid': colid, 'sourceContentType': 1, 'permissiontype': 0,
    }
    try:
        r = requests.post(url, data=data, timeout=(5, 20), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
    except:
        return []
    records = re.findall(r'<record><!\[CDATA\[(.*?)\]\]></record>', r.text, re.DOTALL)
    results = []
    for rec in records:
        m = re.search(r"<a href='([^']+)'[^>]*title='([^']*)'[^>]*>(.*?)</a><b[^>]*>(\d{4}-\d{2}-\d{2})</b>", rec, re.DOTALL)
        if not m:
            m = re.search(r"<a href='([^']+)'[^>]*>(.*?)</a><b[^>]*>(\d{4}-\d{2}-\d{2})</b>", rec, re.DOTALL)
            if not m:
                continue
            href, title_text, pd = m.group(1), m.group(2), m.group(3)
        else:
            href, title_text, pd = m.group(1), m.group(2) or m.group(3), m.group(4)
        title = re.sub(r'<[^>]+>', '', title_text).strip()
        title = re.sub(r'\s+', '', title).strip()
        if len(title) < 4:
            continue
        url = href if href.startswith('http') else BASE + href
        results.append({'url': url, 'pub_date': pd, 'title': title})
    return results

def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配: 从匹配到的 <div...> 的 '>' 之后, 到对应 </div>, 返回内部 HTML。
    修复非贪婪 (.*?)</div> 在正文含嵌套 div(表格/附件区)时截断的问题"""
    m = re.search(open_tag_re, html)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:]):
        if mm.group(0).startswith('<div'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]

def fetch_detail(url):
    for _attempt in range(2):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return '', ''
    title = ''
    m = re.search(r'<title>\s*([^<]+?)\s*</title>', r.text, re.DOTALL)
    if m:
        title = re.sub(r'\s+', '', m.group(1)).strip()
        title = re.split(r'[-–—|_]', title)[0].strip()
    content = ''
    # 正文容器: 优先平衡匹配 id="zoom" (修复清单/名录类表格在嵌套div中被截断)
    c = extract_balanced_div(r.text, r'id="zoom"[^>]*>')
    if c.strip():
        content = c.strip()
    if not content.strip():
        c = extract_balanced_div(r.text, r'<div[^>]*class="[^"]*wz_zoom[^"]*"[^>]*>')
        if c.strip():
            content = c.strip()
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    content = re.sub(r'href="(?!https?://|//|#)(/[^"]+)"', lambda mm: f'href="{BASE}{mm.group(1)}"', content)
    content = re.sub(r'src="(?!https?://|//)(/[^"]+)"', lambda mm: f'src="{BASE}{mm.group(1)}"', content)
    content = re.sub(r'(<p>\s*(<br\s*/?>)?\s*</p>)+$', '', content)
    return title, content.strip()

def run(col, max_pages=5):
    records = []
    seen = set()
    for pg in range(max_pages):
        items = fetch_list(col, pg + 1)
        if not items:
            print(f"  Page {pg+1}: 0 items (stop)")
            break
        print(f"  Page {pg+1}: {len(items)} items")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, content = fetch_detail(it['url'])
            if not title:
                title = it['title']
            if not content.strip():
                continue
            records.append({
                'title': title, 'url': it['url'], 'source_url': it['url'],
                'pub_date': it['pub_date'],
                'site_name': COLUMNS[col]['name'], 'content': content, 'summary': '',
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, f"rizhao_sthjj_{col}")
    return len(valid)

if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--col', default='179934', choices=list(COLUMNS.keys()))
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    cnt = run(args.col, max_pages=args.pages)
    print(f"Done: {cnt} records")