#!/usr/bin/env python3
"""和合承德网 - 公告通知 (www.hehechengde.cn/news/txy/)
CMS: CodeIgniter
列表: index.html + index_{N}.html (N=2~30, 共30页, 需要Cookie)
详情: title=<title>(-和合承德网), content=div.main-content
"""
import re, requests, sqlite3, os, sys, time
from datetime import datetime

DOMAIN = "www.hehechengde.cn"
BASE_LIST = "https://www.hehechengde.cn/news/txy"
BASE_DETAIL = "https://m.hehechengde.cn"
SITE_NAME = "和合承德网-公告通知"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TOTAL_PAGES = 30


def get_conn():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.row_factory = sqlite3.Row
    return conn


def make_session():
    """创建带Cookie的session"""
    s = requests.Session()
    s.headers.update(HEADERS)
    # 先访问首页获取cookie
    try:
        s.get(f"{BASE_LIST}/", timeout=30)
    except Exception:
        pass
    return s


def get_list_url(page):
    if page == 1:
        return f"{BASE_LIST}/"
    return f"{BASE_LIST}/index_{page}.html"


def fetch_list_page(page_num):
    """获取列表页，每页新session刷新cookie"""
    url = get_list_url(page_num)
    for attempt in range(3):
        try:
            s = requests.Session()
            s.headers.update(HEADERS)
            # 取首页刷新cookie
            s.get(f"{BASE_LIST}/", timeout=30)
            # 取目标页
            resp = s.get(url, timeout=30, allow_redirects=False)
            if resp.status_code in (301, 302):
                time.sleep(2)
                continue
            resp.encoding = "utf-8"
            if resp.status_code == 200 and len(resp.text) > 500:
                return s, resp.text
        except Exception:
            pass
        time.sleep(2)
    return None, None


def parse_list(html):
    """解析列表页，返回[{'title','url','date'}]"""
    items = []
    for dl in re.finditer(r'<dl[^>]*class="page-list"[^>]*>(.*?)</dl>', html, re.DOTALL):
        dl_html = dl.group(1)
        # 链接和标题
        a = re.search(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', dl_html, re.DOTALL)
        if not a:
            continue
        href = a.group(1).strip()
        # 优先用title属性（完整标题）
        title_attr = re.search(r'title="([^"]+)"', a.group(0))
        title = title_attr.group(1) if title_attr else re.sub(r'<[^>]+>', '', a.group(2)).strip()
        if not title:
            continue
        # 日期
        date = ""
        dm = re.search(r'<em>\s*(\d{4}-\d{2}-\d{2})', dl_html)
        if dm:
            date = dm.group(1)
        # 完整URL - 移动端域名转主站域名（相同cookie）
        if not href.startswith("http"):
            href = BASE_DETAIL + href if href.startswith("/") else f"{BASE_DETAIL}/{href}"
        href = href.replace("https://m.hehechengde.cn", "https://www.hehechengde.cn")
        items.append({"title": title.strip(), "url": href, "date": date})
    return items


def extract_detail(url, title_from_list):
    """提取详情页标题、日期、正文HTML、附件（每篇独立刷新session）"""
    # 每篇详情页独立session防cookie过期
    s = requests.Session()
    s.headers.update(HEADERS)
    try:
        s.get(f"{BASE_LIST}/", timeout=30)
        resp = s.get(url, timeout=30)
        resp.encoding = "utf-8"
        html = resp.text
    except Exception as e:
        return None

    # 标题 - 从<title>去站点后缀
    title = title_from_list
    tm = re.search(r'<title>(.*?)</title>', html)
    if tm:
        t = re.sub(r'-和合承德网.*$', '', tm.group(1)).strip()
        if t:
            title = t

    # 日期 - 正文中找
    date = ""
    dm = re.search(r'(\d{4})-(\d{2})-(\d{2})', html)
    if dm:
        date = f"{dm.group(1)}-{dm.group(2)}-{dm.group(3)}"

    # 正文 - div.main-content，备选section-content mt20
    content = ""
    cm = re.search(r'<div[^>]*class="main-content"[^>]*>(.*?)</div>', html, re.DOTALL)
    if not cm:
        cm = re.search(r'<div[^>]*class="section-content mt20"[^>]*>(.*?)</div>', html, re.DOTALL)
    if cm:
        raw = cm.group(1).strip()
        # 清洗干扰
        raw = re.sub(r'<script[^>]*>.*?</script>', '', raw, flags=re.DOTALL)
        raw = re.sub(r'<style[^>]*>.*?</style>', '', raw, flags=re.DOTALL)
        if raw:
            content = raw

    # 附件提取
    attachments = []
    for am in re.finditer(
        r'<a[^>]*href="([^"]*\.(pdf|doc|docx|xls|xlsx|zip|rar))"[^>]*>([^<]+)</a>',
        content if content else html, re.I
    ):
        furl = am.group(1)
        fname = am.group(3).strip()
        if furl.startswith("//"):
            furl = "https:" + furl
        elif furl.startswith("/"):
            furl = BASE_DETAIL + furl
        attachments.append({"name": fname, "url": furl})

    # PDF空内容处理
    summary = re.sub(r'<[^>]+>', ' ', content).strip()
    summary = re.sub(r'\s+', ' ', summary)[:500] if summary else title[:300]
    text_len = len(re.sub(r'<[^>]+>', '', content).strip())
    if text_len < 20 and attachments:
        content = f'<p><a href="{url}">{title}</a></p>' + content
        for att in attachments:
            content += f'\n<p><a href="{att["url"]}">{att["name"]}</a></p>'
    else:
        for att in attachments:
            if att["url"] not in (content or ""):
                content += f'\n<p><a href="{att["url"]}">{att["name"]}</a></p>'

    return {
        "title": title,
        "url": url,
        "date": date,
        "content": content,
        "summary": summary,
        "attachments": attachments,
    }


def main():
    incremental = False
    pages_override = None
    args = list(sys.argv[1:])
    i = 0
    while i < len(args):
        arg = args[i]
        if arg in ("1", "incremental", "--incremental"):
            incremental = True
        elif arg == "--pages" and i + 1 < len(args):
            pages_override = int(args[i+1])
            i += 1
        i += 1

    if pages_override:
        total_pages = pages_override
    else:
        total_pages = 1 if incremental else TOTAL_PAGES

    mode = "增量" if incremental else "全量"
    print(f"\n{'='*50}", flush=True)
    print(f"🏠 {SITE_NAME} ({mode})", flush=True)
    print(f"   DB: {DB_PATH}", flush=True)
    print(f"{'='*50}", flush=True)

    new_count = 0
    skip_count = 0
    error_count = 0

    for page in range(1, total_pages + 1):
        url = get_list_url(page)
        print(f"\n📄 列表页 {page}/{total_pages}: {url}", flush=True)
        session, html = fetch_list_page(page)
        if not html:
            print(f"  [WARN] 获取失败", flush=True)
            continue

        items = parse_list(html)
        if not items:
            print(f"  [WARN] 无文章链接", flush=True)
            break

        print(f"  发现 {len(items)} 篇文章", flush=True)

        for idx, item in enumerate(items):
            detail = extract_detail(item["url"], item["title"])
            if not detail:
                print(f"  [{idx+1}] ✗ {item['title'][:40]}...", flush=True)
                error_count += 1
                continue

            conn = get_conn()
            try:
                content = (detail["content"] or "")[:500000]
                summary = (detail["summary"] or "")[:300]
                conn.execute(
                    "INSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, summary, site_name, tags) "
                    "VALUES (?, ?, ?, ?, ?, ?, ?)",
                    (detail["title"], detail["url"], content, detail["date"] or "", summary, SITE_NAME, "环评阶段")
                )
                changed = conn.total_changes > 0
                conn.commit()
                if changed:
                    new_count += 1
                    if not incremental or True:
                        print(f"  [{idx+1}] ✅ {detail['title'][:50]} ({detail['date']})", flush=True)
                else:
                    skip_count += 1
            except Exception as e:
                print(f"  [ERROR] {e}", flush=True)
                error_count += 1
            finally:
                conn.close()

            time.sleep(0.3)

    print(f"\n{'─'*30}", flush=True)
    print(f"✅ 新增: {new_count} | 跳过: {skip_count} | 错误: {error_count}", flush=True)

    if new_count > 0:
        conn = get_conn()
        try:
            conn.execute(
                "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary, rowid_col) "
                "SELECT id, title, site_name, substr(content, 1, 500), id "
                "FROM gov_raw WHERE site_name=?", (SITE_NAME,)
            )
            conn.commit()

        except Exception as e:
            print(f"  [WARN] FTS同步:{e}", flush=True)
        finally:
            conn.close()

    print(f"📤 完成！共 {new_count} 条新数据", flush=True)


if __name__ == "__main__":
    main()