#!/usr/bin/env python3
"""霍林郭勒市 - 通知公告"""
import sys, re, sqlite3, requests, warnings
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os
warnings.filterwarnings("ignore")

BASE_URL = "http://www.hlgls.gov.cn"
LIST_PATH = "/hsxw/tzgg/"
SITE_NAME = "霍林郭勒市-通知公告"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
CUT_DATE = (datetime.now() - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0"}
import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES


def fetch(url, label=""):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print("[ERROR] %s: %s" % (label, e))
        return ""

def parse_list(html):
    soup = BeautifulSoup(html, "lxml")
    lst = soup.find("div", class_="lis_list_part2_list")
    if not lst:
        return []
    items = []
    for li in lst.find_all("li"):
        a = li.find("a", href=True)
        span = li.find("span")
        if a and span:
            title = a.get("title", "") or a.get_text(strip=True)
            href = a["href"].strip()
            date = span.get_text(strip=True)
            if href.startswith("./"):
                href = LIST_PATH + href[2:]
            items.append((title, href, date))
    return items

def parse_detail(html):
    soup = BeautifulSoup(html, "lxml")
    title = ""
    t = soup.find("title")
    if t:
        title = t.get_text(strip=True)
    pub_date = ""
    m = re.search(r"发布时间[：:]\s*(\d{4}-\d{2}-\d{2})", html)
    if m:
        pub_date = m.group(1)
    content_div = soup.find(id="pare")
    if not content_div:
        content_div = soup.find(id="pareContent")
    if not content_div:
        content_div = soup.find("div", class_="lis_list_part2_content")
    if content_div:
        content_html = str(content_div)
    else:
        content_html = ""
    return title, pub_date, content_html

def get_total_pages(html):
    # Check index_33.html - last page with data (index_34.html=138 bytes empty)
    import requests as rq
    for i in range(50, 0, -1):
        try:
            r = rq.get(BASE_URL + LIST_PATH + "index_%d.html" % i, headers=HEADERS, timeout=5)
            if len(r.text) > 500:
                return i + 1  # 0-indexed pages
        except:
            pass
    return 1

def main(mode="full"):
    start = datetime.now()
    print("[霍林郭勒] mode=%s, cut_date=%s" % (mode, CUT_DATE))
    
    all_items = []
    for page in range(min(34, _MAX_PG or 34)):
        if page == 0:
            url = BASE_URL + LIST_PATH
        else:
            url = BASE_URL + LIST_PATH + "index_%d.html" % page
        html = fetch(url, "P%d" % (page+1))
        if not html or len(html) < 500:
            if page > 0:
                break
            continue
        items = parse_list(html)
        all_items.extend(items)
        if items:
            print("[霍林郭勒] P%d: %d条 (%s ~ %s)" % (page+1, len(items), items[0][2], items[-1][2]))
        if items and items[-1][2] < CUT_DATE:
            print("[霍林郭勒] %s 超过3年线，停止" % items[-1][2])
            break
    
    print("[霍林郭勒] 列表共 %d 条" % len(all_items))
    
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    new, exists, skipped = 0, 0, 0
    
    for idx, (title, href, date) in enumerate(all_items):
        if date and date < CUT_DATE:
            skipped += 1
            continue
        
        detail_url = href if href.startswith("http") else BASE_URL + href
        html = fetch(detail_url, "详情%d" % (idx+1))
        if not html:
            continue
        
        dt, pd, ch = parse_detail(html)
        title = dt or title
        pd = pd or date
        summary = ch[:200] if ch else title
        if ch:
            summary = BeautifulSoup(ch, "lxml").get_text(strip=True)[:200]
        
        c.execute("INSERT OR IGNORE INTO gov_raw (title, page_url, site_name, summary, content, publish_date) VALUES (?,?,?,?,?,?)",
                  (title, detail_url, SITE_NAME, summary, ch, pd))
        if c.rowcount:
            new += 1
        else:
            exists += 1
        
        if (idx+1) % 20 == 0:
            conn.commit()
            el = (datetime.now()-start).total_seconds()
            print("[霍林郭勒] %d/%d, 新增%d, 已存在%d, %ds" % (idx+1, len(all_items), new, exists, el))
    
    conn.commit()
    conn.close()
    el = (datetime.now()-start).total_seconds()
    print("[霍林郭勒] 完成! 新增%d, 已存在%d, 跳过%d, 耗时%ds" % (new, exists, skipped, el))

if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) > 1 else "full")