#!/usr/bin/env python3
import os
"""
大西北网 (www.dxbei.com) 爬虫
五个栏目：新疆/宁夏/青海/陕西/甘肃
需 cookie 会话绕过 WAF
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE = "大西北网"
BASE_URL = "http://www.dxbei.com"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"

SECTIONS = {
    "xinjiang": {"path": "/a/xinjiang/", "tid": 305, "name": "大西北-新疆"},
    "ningxia":  {"path": "/a/ningxia/",  "tid": 306, "name": "大西北-宁夏"},
    "qinghai":  {"path": "/a/qinghai/",  "tid": 308, "name": "大西北-青海"},
    "shanxi":   {"path": "/a/shanxi/",   "tid": 307, "name": "大西北-陕西"},
    "gansu":    {"path": "/news/gansuxinwen/", "tid": 300, "name": "大西北-甘肃"},
}

def make_session():
    s = requests.Session()
    s.headers.update({"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"})
    # 先拿 cookie
    try:
        s.get(BASE_URL, timeout=15)
    except:
        pass
    return s

def fetch_html(session, url):
    for _ in range(3):
        try:
            r = session.get(url, timeout=20)
            r.encoding = "utf-8"
            if r.text.strip() and "403 Forbidden" not in r.text[:100]:
                return r.text
        except:
            pass
        time.sleep(3)
    return None

def parse_list(html, tid):
    """提取 (title, url, date_str)"""
    items = []
    # 文章链接
    for m in re.finditer(r'<a[^>]*href="(/[^"]+)"[^>]*class="title"[^>]*>(.*?)</a>', html):
        href = m.group(1)
        title = re.sub(r"<[^>]+>", "", m.group(2)).strip()
        if not title or href.startswith("/w/"):  # 跳过无关栏目
            continue
        url = BASE_URL + href
        items.append([title, url, ""])
    # 匹配日期（按文章顺序）
    dates = re.findall(r"<small>日期[：:]</small>\s*(\d{4}-\d{2}-\d{2})", html)
    for i in range(min(len(items), len(dates))):
        items[i][2] = dates[i]
    return items

def fetch_detail(session, url):
    html = fetch_html(session, url)
    if not html:
        return None, None, None
    result = {}
    m = re.search(r"<title>([^<]+?)(?:_大西北网)?</title>", html)
    if m:
        result["title"] = m.group(1).strip()
    m = re.search(r'<div class="content"[^>]*>(.*?)<!-- /?content -->', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
        # 清理脚本/样式
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL | re.I)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL | re.I)
        # 补齐相对路径
        content = re.sub(r'src="(?!https?://)', f'src="{BASE_URL}', content)
        content = re.sub(r'href="(?!https?://)', f'href="{BASE_URL}', content)
        result["content"] = content
    m = re.search(r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})", html)
    if m:
        result["pub_date"] = m.group(1)[:10]
    return result.get("title"), result.get("content"), result.get("pub_date")

def main(incremental=False, limit=None):
    session = make_session()
    total_all = 0
    for key, sec in SECTIONS.items():
        print(f"\n{'='*50}\n🏠 {sec['name']}\n{'='*50}")
        html = fetch_html(session, f"{BASE_URL}{sec['path']}")
        if not html:
            print("❌ 首页获取失败，跳过")
            continue
        # 总页数
        pages = 1
        m = re.search(rf'list_{sec["tid"]}_(\d+)\.html[^>]*>末页', html)
        if m:
            pages = int(m.group(1))
        else:
            nums = [int(x) for x in re.findall(rf'list_{sec["tid"]}_(\d+)\.html', html)]
            if nums:
                pages = max(nums)
        print(f"📊 共 {pages} 页")
        if incremental:
            pages = min(pages, 1)  # 增量只爬前3页
            print(f"🔄 增量模式: 取前 {pages} 页")
        count = 0
        for pg in range(1, pages + 1):
            if limit and count >= limit:
                break
            if pg == 1:
                page_html = html
            else:
                url = f"{BASE_URL}{sec['path']}list_{sec['tid']}_{pg}.html"
                page_html = fetch_html(session, url)
                if not page_html:
                    continue
            items = parse_list(page_html, sec["tid"])
            if not items:
                continue
            # 日期过滤
            page_dates = [d for _, _, d in items if d]
            if page_dates:
                oldest = min(page_dates)
                if oldest < THREE_YEARS_AGO:
                    print(f"  ⏹ 第{pg}页最早 {oldest}，停止")
                    break
                print(f"  📄 第{pg}页 ({min(page_dates)}~{max(page_dates)})")
            # 单篇过滤 + 取详情
            batch = []
            for title, url, list_date in items:
                if list_date and list_date < THREE_YEARS_AGO:
                    continue
                if limit and count >= limit:
                    break
                print(f"  [{count+1}] {title[:50]}...", end=" ", flush=True)
                dt_title, content, pub_date = fetch_detail(session, url)
                final_title = dt_title or title
                final_date = pub_date or list_date or ""
                summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
                summary = re.sub(r"\s+", " ", summary)
                batch.append({
                    "site_name": sec["name"],
                    "title": final_title,
                    "url": url,
                    "content": content or "",
                    "pub_date": final_date,
                    "summary": summary,
                    "tags": sec["name"],
                })
                count += 1
                print("✅")
                time.sleep(0.5)
            if batch:
                push_to_searchdb(batch, "dxbei")
            if limit and count >= limit:
                break
            time.sleep(0.8)
        print(f"  ✅ {sec['name']}: {count} 条")
        total_all += count
    print(f"\n{'='*50}\n🎉 全部完成! 共 {total_all} 条\n{'='*50}")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--incremental", action="store_true")
    parser.add_argument("--limit", type=int)
    args = parser.parse_args()
    incremental = args.incremental or (len(sys.argv) > 1 and sys.argv[1] == "1")
    t0 = time.time()
    main(incremental=incremental, limit=args.limit)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")