#!/usr/bin/env python3
import os
"""
仪征化工园区 - 通知公告
Hanweb CMS, API返回第1页(15条)，详情页抓取HTML正文
"""
import os, re, logging, sqlite3, requests
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
from datetime import date

logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
log = logging.getLogger(__name__)

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "仪征化工园区-通知公告"
BASE = "https://www.yizheng.gov.cn"
LIST_PAGE = BASE + "/zfxxgk/yqxzbmxxgkml/yqxz/hxgyyq/fdzdgknr/tzgg/index.html"
API_URL = BASE + "/api-gateway/jpaas-publish-server/front/page/build/unit"
CUTOFF = "2023-06-18"

API_PARAMS = {
    'parseType': 'bulidstatic',
    'webId': 'y6CgeNe9yhPDWezyQsTp8',
    'tplSetId': 'I1p2W2qXfR0Sxtl3xljQj',
    'pageType': 'column',
    'tagId': '列表一list',
    'pageId': '8sF38ay0nwtQcANIwKF8z',
    'pageSize': '15',
    'pageNo': '1',
}

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
    'Referer': LIST_PAGE,
    'Accept': 'application/json, text/plain, */*',
}

def fetch_json():
    """Fetch list page from API using requests."""
    try:
        resp = requests.get(API_URL, params=API_PARAMS, headers=HEADERS, verify=False, timeout=30)
        resp.raise_for_status()
        return resp.json()
    except Exception as e:
        log.warning(f"API error: {e}")
        return None

def parse_list(html):
    """Extract title, date, detail URL from list HTML."""
    items = []
    for row in re.findall(r'<li class="clearfix">(.*?)</li>', html, re.DOTALL):
        m = re.search(r'<a href="([^"]+)"[^>]*>([^<]+)</a>', row)
        d = re.search(r'<span>([^<]+)</span>', row)
        if m:
            url = BASE + m.group(1) if m.group(1).startswith('/') else m.group(1)
            title = m.group(2).strip()
            pub_date = d.group(1).strip() if d else ''
            items.append({'title': title, 'url': url, 'pub_date': pub_date})
    return items

def fetch_detail(url):
    """Fetch detail page, extract HTML content from bt-content div."""
    try:
        resp = requests.get(url, headers={'User-Agent': HEADERS['User-Agent']}, verify=False, timeout=30)
        resp.raise_for_status()
        html = resp.text
        # Extract content from bt-content div
        m = re.search(r'<div class="bt-content[^"]* zoom[^"]* clearfix[^>]*>(.*?)</div>\s*<!--/正文-->', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
        else:
            # Fallback: try other known patterns
            m = re.search(r'class="bt-content[^>]*>(.*?)</div>', html, re.DOTALL)
            content = m.group(1).strip() if m else ''
        return content
    except Exception as e:
        log.warning(f"Detail error {url}: {e}")
        return ''

def main():
    log.info(f"仪征化工园区 - 通知公告")
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    
    data = fetch_json()
    if not data:
        log.error("API failed!")
        return
    
    html = data['data']['html']
    items = parse_list(html)
    log.info(f"API返回 {len(items)} 条")
    
    new = 0
    skip = 0
    
    for item in items:
        if item['pub_date'] and item['pub_date'] < CUTOFF:
            continue
        
        # Check duplicate
        if db.execute("SELECT 1 FROM gov_raw WHERE page_url = ?", (item['url'],)).fetchone():
            skip += 1
            continue
        
        log.info(f"  抓取: {item['title'][:30]}...")
        content = fetch_detail(item['url'])
        
        summary = f"项目名称: {item['title']}\n发布时间: {item['pub_date']}"
        
        db.execute("""INSERT OR IGNORE INTO gov_raw 
            (site_name, source_url, page_url, title, publish_date, summary, content, status, category)
            VALUES (?,?,?,?,?,?,?,'normal','园区公告')""",
            (SITE_NAME, LIST_PAGE, item['url'], item['title'],
             item['pub_date'] or date.today().isoformat(), summary, content or summary))
        rowid = db.execute('SELECT last_insert_rowid()').fetchone()[0]
        if rowid:
            new += 1
            # Also write to FTS search index
            db.execute("""INSERT OR IGNORE INTO gov_search
                (rowid, title, site_name, summary) VALUES (?,?,?,?)""",
                (rowid, item['title'], SITE_NAME, (content or summary)[:300]))
        db.commit()
    
    db.close()
    log.info(f"Done! New: {new}, Skip: {skip}")

if __name__ == '__main__':
    main()