#!/usr/bin/env python3
"""Crawler for dykfq.qiannan.gov.cn - 通知公告
List: li > a.pic_list + span.date, Pagination: index_N.html
Detail: div.trs_editor_view
"""

import requests, re, sqlite3, sys, time
from datetime import datetime

SITE_URL = "https://dykfq.qiannan.gov.cn"
LIST_PATH = "/xwzx_103/tzgg"
DOMAIN = "dykfq.qiannan.gov.cn"
SITE_NAME = "都匀经济开发区-通知公告"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
DB_PATH = "/root/search.db"
INCREMENTAL = "--incremental" in sys.argv

def get_text(html):
    return re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', '', html)).strip()

def crawl_page(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        return r.text if r.status_code == 200 else None
    except:
        return None

def parse_list(html):
    items = []
    pattern = re.compile(
        r'<a[^>]*class="pic_list"[^>]*title="([^"]*)"[^>]*href="([^"]+)"[^>]*>.*?</a>\s*<span>([^<]+)</span>',
        re.DOTALL
    )
    for m in pattern.finditer(html):
        title = m.group(1).strip()
        href = m.group(2).strip()
        date = m.group(3).strip()
        if not href.startswith('http'):
            href = SITE_URL + href if href.startswith('/') else SITE_URL + '/' + href
        items.append((title, href, date))
    return items

def parse_detail(html):
    content = ""
    m = re.search(r'<div[^>]*class="[^"]*trs_editor_view[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    if not content:
        m = re.search(r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    pub_date = ""
    m2 = re.search(r'发布时间：(\d{4}[-/]\d{2}[-/]\d{2})', html)
    if m2: pub_date = m2.group(1).strip()
    atts = []
    for am in re.finditer(r'<a[^>]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar))"[^>]*>(.*?)</a>', html, re.DOTALL | re.IGNORECASE):
        au = am.group(1).strip()
        att_url = au if au.startswith('http') else (SITE_URL + au if au.startswith('/') else SITE_URL + '/' + au)
        atts.append((get_text(am.group(2)), att_url))
    return content, pub_date, atts

def save(items):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    ins = 0
    for title, url, ds, content, pd, atts in items:
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
        if c.fetchone(): continue
        ah = '<div class="attachments">' + '<br>'.join(f'<a href="{u}" target="_blank">{t}</a>' for t, u in atts) + '</div>' if atts else ''
        fc = content + '\n' + ah if ah else content
        summary = get_text(content)[:200] if content else title
        c.execute("INSERT INTO gov_raw(title,page_url,content,summary,publish_date,site_name) VALUES(?,?,?,?,?,?)",
                  (title, url, fc, summary, ds or pd, SITE_NAME))
        ins += 1
    conn.commit()
    conn.close()
    return ins

def main():
    mode = "INCREMENTAL" if INCREMENTAL else "FULL"
    print(f"=== {SITE_NAME} [{mode}] ===")
    
    if INCREMENTAL:
        pages = [""]  # index.html (first page)
        max_pages = 5
    else:
        pages = [""] + [f"index_{i}.html" for i in range(2, 35)]
    
    total_ins = 0
    page_count = 0
    for ps in pages:
        page_count += 1
        url = f"{SITE_URL}{LIST_PATH}/{'index.html' if not ps else ps}"
        html = crawl_page(url)
        if not html: continue
        items = parse_list(html)
        if not items: break
        
        page_items = []
        for title, detail_url, list_date in items:
            if INCREMENTAL:
                conn = sqlite3.connect(DB_PATH, timeout=60)
                cu = conn.cursor()
                if cu.execute("SELECT id FROM gov_raw WHERE page_url=?", (detail_url,)).fetchone():
                    conn.close()
                    continue
                conn.close()
            print(f"  [{len(page_items)+1}/{len(items)}] {title[:45]}...")
            h2 = crawl_page(detail_url)
            if not h2: continue
            content, pd, atts = parse_detail(h2)
            if not content: content = title
            page_items.append((title, detail_url, list_date, content, pd, atts))
            time.sleep(0.3)
        
        if page_items:
            n = save(page_items)
            total_ins += n
            print(f"  Page {page_count}: +{n}")
        else:
            print(f"  Page {page_count}: 0 items (all skipped)")
        
        if INCREMENTAL and page_count >= max_pages:
            break
    
    print(f"=== Done: {total_ins} new ===")

if __name__ == "__main__":
    main()