#!/usr/bin/env python3
"""Crawler for naimanqi.gov.cn - 通知公告 /xwzx/tzgg_7894/"""
import requests, re, sqlite3, sys, time
from datetime import datetime

BASE_URL = "http://221.199.189.183/xwzx/tzgg_7894"
DOMAIN = "www.naimanqi.gov.cn"
SITE_NAME = "奈曼旗人民政府-通知公告"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Host": DOMAIN}
DB_PATH = "/root/search.db"
INCREMENTAL = "--incremental" in sys.argv

def get_text(html):
    return re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', '', html)).strip()

def crawl(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        return r.text if r.status_code == 200 else None
    except Exception as e:
        return None

def parse_list(html):
    items = []
    for m in re.finditer(r'<li[^>]*>.*?<a[^>]*href="\.([^"]+)"[^>]*>(.*?)</a>\s*<span[^>]*>(.*?)</span>', html, re.DOTALL):
        p = m.group(1).strip()
        url = BASE_URL + '/' + p[2:] if p.startswith('./') else BASE_URL + '/' + p.lstrip('/')
        items.append((get_text(m.group(2)), url, m.group(3).strip()))
    return items

def parse_detail(html):
    title = get_text(re.search(r'<title>(.*?)</title>', html, re.DOTALL).group(1)) if re.search(r'<title>(.*?)</title>', html, re.DOTALL) else ''
    for suf in ['_奈曼旗人民政府', ' - 奈曼旗人民政府', '通知公告']:
        title = title.replace(suf, '').strip()
    m = re.search(r'<div[^>]*class="[^"]*lis_list_part2_content[^"]*"[^>]*>(.*?)</div>\s*</div>\s*</div>', html, re.DOTALL)
    if not m:
        m = re.search(r'<div[^>]*class="[^"]*trs_editor_view[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    content = m.group(1).strip() if m else title
    pub_date = ''
    m2 = re.search(r'(20\d{2}[-/]\d{2}[-/]\d{2})', html)
    if m2: pub_date = m2.group(1)
    atts = []
    for am in re.finditer(r'<a[^>]*href="(\./[^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar|txt))"[^>]*>(.*?)</a>', html, re.DOTALL | re.IGNORECASE):
        au = am.group(1).strip()
        atts.append((get_text(am.group(2)), BASE_URL + '/' + au[2:] if au.startswith('./') else BASE_URL + '/' + au.lstrip('/')))
    return title, content, pub_date, atts

def save(items):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    ins = 0
    for title, url, ds, content, pd, atts in items:
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
        if c.fetchone(): continue
        ah = '<div class="attachments">' + '<br>'.join(f'<a href="{u}" target="_blank">{t}</a>' for t, u in atts) + '</div>' if atts else ''
        fc = content + '\n' + ah if ah else content
        c.execute("INSERT INTO gov_raw(title,page_url,content,summary,publish_date,site_name) VALUES(?,?,?,?,?,?)",
                  (title, url, fc, get_text(content)[:200] if content else title, ds or pd, SITE_NAME))
        ins += 1
    conn.commit()
    conn.close()
    return ins

def main():
    mode = 'INCREMENTAL' if INCREMENTAL else 'FULL'
    print(f"=== {SITE_NAME} [{mode}] ===")
    pages = [''] + [f'index_{i}.html' for i in range(1, 67)] if not INCREMENTAL else ['']
    ti = 0
    for pi, p in enumerate(pages, 1):
        html = crawl(f"{BASE_URL}/{'index.html' if not p else p}")
        if not html: continue
        items = parse_list(html)
        if not items: break
        pi_items = []
        for t, u, d in items:
            if INCREMENTAL:
                conn = sqlite3.connect(DB_PATH, timeout=60)
                cu = conn.cursor()
                ex = cu.execute("SELECT id FROM gov_raw WHERE page_url=?", (u,)).fetchone()
                conn.close()
                if ex: continue
            h2 = crawl(u)
            if not h2: continue
            dt, ct, pd, at = parse_detail(h2)
            pi_items.append((dt or t, u, d, ct, pd, at))
            time.sleep(0.2)
        if pi_items:
            n = save(pi_items)
            ti += n
            print(f"  Page {pi}: +{n} new")
        else:
            print(f"  Page {pi}: 0 items")
        if INCREMENTAL and pi >= 5: break
    print(f"=== Done: {ti} new ===")

if __name__ == '__main__':
    main()