#!/usr/bin/env python3
"""山东先达农化股份有限公司-环保公示爬虫

DedeCMS 自定义企业站
列表: /news_42.html (第1页, 8条/页)
      第N页: /newsp_42_{N}.html (N=1~27)
详情: /newsinfo_{id}.html
内容: div.newsinfo_3 > img (扫描图片)
日期: div.newsinfo_2 中 "发布时间：YYYY-MM-DD"
"""

import requests
import re
import sys
import json
import time
import argparse
from bs4 import BeautifulSoup
from urllib.parse import urljoin

sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import push_to_searchdb
import sqlite3

DB_PATH = '/root/search.db'


def get_db_connection():
    try:
        conn = sqlite3.connect(DB_PATH, timeout=30)
        conn.execute("PRAGMA busy_timeout=30000")
        conn.execute("PRAGMA journal_mode=WAL")
        return conn
    except Exception as e:
        print(f'  WARN DB connect: {e}')
        return None


BASE_URL = 'https://www.cynda.cn'

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}

SITE_NAME = '先达股份-环保公示'

sess = requests.Session()
sess.headers.update(HEADERS)


def fetch_page(url):
    resp = sess.get(url, timeout=15)
    resp.encoding = 'utf-8'
    return resp.text


def parse_list(html):
    """解析列表页"""
    soup = BeautifulSoup(html, 'html.parser')
    items = []

    for div in soup.select('.news_in1'):
        date_el = div.select_one('.news_in1a')
        link_el = div.select_one('.news_in1b a')
        if not link_el:
            continue

        href = link_el.get('href', '')
        if not href:
            continue

        title = link_el.get('title', '') or link_el.get_text(strip=True)
        detail_url = href if href.startswith('http') else urljoin(BASE_URL, href)
        pub_date = date_el.get_text(strip=True) if date_el else ''

        items.append({
            'title': title,
            'url': detail_url,
            'pub_date': pub_date,
        })

    return items


def fetch_detail(item):
    """抓取详情页"""
    url = item['url']

    try:
        html = fetch_page(url)
    except Exception as e:
        print(f'  WARN skip: {url} - {e}')
        return item['title'], item.get('pub_date', ''), '', []

    soup = BeautifulSoup(html, 'html.parser')

    # 标题
    title_el = soup.select_one('.newsinfo_1')
    title = title_el.get_text(strip=True) if title_el else item['title']

    # 日期
    pub_date = item.get('pub_date', '')
    info_el = soup.select_one('.newsinfo_2')
    if info_el:
        m = re.search(r'(\d{4}-\d{2}-\d{2})', info_el.get_text())
        if m:
            pub_date = m.group(1)

    # 正文 - 扫描图片
    content = ''
    attachments = []
    ns3 = soup.select_one('.newsinfo_3')
    if ns3:
        parts = []
        for img in ns3.find_all('img'):
            src = img.get('src', '')
            alt = img.get('alt', '') or '环保公示图片'
            if src:
                full_src = urljoin(url, src)
                parts.append(f'![{alt}]({full_src})')
                # 图片本身作为附件
                attachments.append({'name': alt, 'url': full_src})

        # 顺便检查是否有 PDF 附件链接
        for a in ns3.find_all('a', href=True):
            a_href = a.get('href', '')
            a_text = a.get_text(strip=True)
            if a_href and re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', a_href, re.I):
                full_href = urljoin(url, a_href)
                attachments.append({'name': a_text or '附件', 'url': full_href})
                parts.append(f'[{a_text or "附件"}]({full_href})')

        content = '\n\n'.join(parts)

    return title, pub_date, content, attachments


def main():
    parser = argparse.ArgumentParser(description='先达股份-环保公示')
    parser.add_argument('--pages', type=int, default=0, help='页数，0=全量27页')
    parser.add_argument('--column', type=int, default=42, help='栏目ID')
    args = parser.parse_args()

    max_pages = args.pages if args.pages > 0 else 27
    col_id = args.column
    all_items = []

    for page in range(1, max_pages + 1):
        if page == 1:
            url = f'{BASE_URL}/news_{col_id}.html'
        else:
            url = f'{BASE_URL}/newsp_{col_id}_{page}.html'

        print(f'Page {page}/{max_pages}...')
        try:
            html = fetch_page(url)
            items = parse_list(html)
            if not items:
                print(f'  empty, stop')
                break
            print(f'  found {len(items)} items')
            all_items.extend(items)
        except Exception as e:
            print(f'  WARN: {e}')
            break
        time.sleep(0.3)

    print(f'\nTotal list: {len(all_items)} items')

    if not all_items:
        print('No items found, abort')
        return

    # 详情
    records = []
    for i, item in enumerate(all_items):
        print(f'  [{i+1}/{len(all_items)}] {item["title"][:40]}...')
        title, pub_date, content, attachments = fetch_detail(item)

        if not title:
            title = item['title']
        title = re.sub(r'\s+', ' ', title).strip()
        summary = content[:500] if content else ''

        record = {
            'url': item['url'],
            'source_url': item['url'],
            'title': title,
            'content': content,
            'summary': summary,
            'pub_date': pub_date,
            'site_name': SITE_NAME,
            'attachments': json.dumps(attachments, ensure_ascii=False) if attachments else '',
        }
        records.append(record)
        time.sleep(0.2)

    # 入库
    print(f'\nInserting {len(records)} records...')
    if records:
        push_to_searchdb(records, SITE_NAME)

    # FTS验证
    conn = get_db_connection()
    if conn:
        try:
            cursor = conn.execute(
                "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)
            )
            raw_count = cursor.fetchone()[0]
            cursor = conn.execute(
                "SELECT COUNT(*) FROM gov_search WHERE site_name=?", (SITE_NAME,)
            )
            fts_count = cursor.fetchone()[0]
            print(f'DB: {raw_count} rows (FTS: {fts_count} rows)')
        except Exception as e:
            print(f'  WARN DB query: {e}')
        finally:
            conn.close()

    print('DONE')


if __name__ == '__main__':
    main()