#!/usr/bin/env python3
"""飞源集团-公司新闻 爬虫"""
import os, re, sqlite3, time, json
from datetime import datetime, timezone, timedelta
from urllib.parse import urljoin

BASE = 'https://www.feiyuanchem.com'
SITE_NAME = '飞源集团-公司新闻'
DB = os.environ.get('SEARCH_DB', '/root/search.db')
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'

tz = timezone(timedelta(hours=8))
cutoff = datetime.now(tz) - timedelta(days=365*3)
cutoff_str = cutoff.strftime('%Y-%m-%d')


def fetch(url):
    import urllib.request
    req = urllib.request.Request(url, headers={'User-Agent': UA})
    try:
        with urllib.request.urlopen(req, timeout=20) as r:
            data = r.read()
            enc = r.headers.get_content_charset() or 'utf-8'
            return data.decode(enc, errors='replace')
    except Exception as e:
        print(f'  [WARN] fetch fail: {e}')
        return ''


def extract_title(html):
    m = re.search(r'<div class="name">(.*?)</div>', html)
    return m.group(1).strip() if m else ''


def extract_date(html):
    m = re.search(r'发布时间[：:](\d{4}-\d{2}-\d{2})', html)
    if m:
        return m.group(1)
    return ''


def extract_content(html):
    m = re.search(r'<div class="txtContent">(.*?)</div>', html, re.DOTALL)
    if m:
        return m.group(1).strip()
    return ''


def list_articles(page=1):
    """返回该页所有文章 (url, title, date)"""
    if page == 1:
        url = '/?list_11/'
    else:
        url = f'/?list_11_{page}/'

    html = fetch(BASE + url)
    if not html:
        return []

    # 解析新闻列表
    articles = []
    # 每个 newsBox 包含一篇文章
    pattern = re.compile(
        r'<div class="newsBox">.*?'
        r'<p class="newsTime2">(\d{4}-\d{2})</p>.*?'
        r"<a href='(/\?\d+\.html)'(?:.*?)>(.*?)</a>",
        re.DOTALL,
    )
    for m in pattern.finditer(html):
        ym = m.group(1)  # 如 2026-06
        url_path = m.group(2)
        title = m.group(3).strip()

        # 取当月第一天作为近似日期
        date_str = f'{ym}-01'
        full_url = BASE + url_path
        articles.append((full_url, title, date_str))

    return articles


def get_total_pages(html):
    """从首页HTML中提取总页数"""
    m = re.search(r'当前1/(\d+)页', html)
    if m:
        return int(m.group(1))
    m = re.search(r'/?list_11_(\d+)/.*?尾页', html)
    if m:
        return int(m.group(1))
    return 1


def crawl(limit_pages=None):
    """主爬取逻辑"""
    # 首页获取总页数
    html = fetch(BASE + '/?list_11/')
    if not html:
        print('[ERROR] 首页打不开')
        return

    total_pages = get_total_pages(html)
    if limit_pages:
        total_pages = min(total_pages, limit_pages)

    print(f'总页数: {total_pages}')

    # 收集所有文章URL
    all_articles = []
    for page in range(1, total_pages + 1):
        articles = list_articles(page)
        if not articles:
            print(f'  第{page}页: 0 条')
            continue

        # 只保留近3年
        filtered = [(u, t, d) for u, t, d in articles if d >= cutoff_str]
        all_articles.extend(filtered)
        print(f'  第{page}页: {len(articles)} 条 (近3年: {len(filtered)})')

        if len(filtered) < len(articles):
            # 有文章超出日期范围，后面的页更老，可以停
            break

        time.sleep(0.3)

    print(f'\n共收集 {len(all_articles)} 条待处理')

    # 入库
    conn = sqlite3.connect(DB, timeout=60)
    c = conn.cursor()

    new_count = 0
    for url, title, list_date in all_articles:
        # 检查是否已存在
        c.execute('SELECT id FROM gov_raw WHERE page_url = ?', (url,))
        row = c.fetchone()

        if row:
            continue  # 已有内容，跳过

        html = fetch(url)
        if not html:
            print(f'  ✗ {title[:30]}... 下载失败')
            continue

        detail_title = extract_title(html) or title
        detail_date = extract_date(html) or list_date
        content = extract_content(html)

        if row and not row[1] and content:
            # 有记录但无正文，更新
            c.execute(
                'UPDATE gov_raw SET title=?, publish_date=?, content=? WHERE page_url=?',
                (detail_title, detail_date, content, url),
            )
            new_count += 1
            print(f'  ✓ {detail_title[:30]}... (正文补全)')
        elif not row:
            # 全新
            c.execute(
                'INSERT OR IGNORE INTO gov_raw (site_name, title, publish_date, page_url, source_url, content) VALUES (?,?,?,?,?,?)',
                (SITE_NAME, detail_title, detail_date, url, url, content),
            )
            new_count += 1
            print(f'  ✓ {detail_title[:30]}... (+1)')
        else:
            print(f'  - {detail_title[:30]}... (跳过)')

        time.sleep(0.2)

    conn.commit()

    # 统计
    c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    total = c.fetchone()[0]
    conn.close()

    print(f'\n新增: {new_count} | 总计: {total}')


if __name__ == '__main__':
    import sys
    full = '--full' in sys.argv
    limit_pages = None
    if full:
        limit_pages = 100
    elif len(sys.argv) > 1 and sys.argv[1].isdigit():
        limit_pages = int(sys.argv[1])
    # 增量模式只爬第一页
    if not full:
        print('增量模式: 只爬第1页')
        crawl(limit_pages=1)
    else:
        crawl(limit_pages=limit_pages)