#!/usr/bin/env python3
"""中海油天津化工研究设计院有限公司 - 公示信息 爬虫"""
import re, requests
from bs4 import BeautifulSoup
from datetime import datetime
import sqlite3, os

BASE_URL = "https://www.trici.cn"
LIST_PATH = "/xwzx/gsxx/"
SITE_NAME = "trici.cn-公示信息"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
CATEGORY = "eia"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
CUTOFF_DATE = "2023-06-16"
CUTOFF_INT = int(CUTOFF_DATE.replace("-", ""))


def get_conn():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.row_factory = sqlite3.Row
    return conn


def clean_text(text):
    if not text:
        return ""
    return re.sub(r'\s+', ' ', text.strip())


def get_list_url(page):
    """获取列表页URL, page从0开始"""
    if page == 0:
        return f"{BASE_URL}{LIST_PATH}"
    return f"{BASE_URL}{LIST_PATH}index_{page}.html"


def extract_articles(session, page):
    """从列表页提取文章链接和标题"""
    url = get_list_url(page)
    try:
        resp = session.get(url, headers=HEADERS, timeout=30)
        resp.raise_for_status()
        resp.encoding = 'utf-8'
    except Exception as e:
        print(f"  [WARN] 列表页{page}获取失败: {e}", flush=True)
        return []

    soup = BeautifulSoup(resp.text, 'html.parser')
    articles = []

    for a in soup.find_all('a', href=True):
        href = a['href']
        txt = a.get_text(strip=True)
        # 文章链接格式: ./YYYYMM/tYYYYMMDD_ID.html
        m = re.match(r'\./(\d{6})/t(\d{8})_\d+\.html$', href)
        if m and txt and len(txt) > 5:
            full_url = BASE_URL + LIST_PATH + href[2:]  # Remove ./
            articles.append({
                'url': full_url,
                'title': txt,
                'date_str': m.group(2),  # YYYYMMDD
            })

    return articles


def extract_detail(session, article):
    """获取详情页的标题、日期、正文"""
    url = article['url']
    try:
        resp = session.get(url, headers=HEADERS, timeout=30)
        resp.raise_for_status()
        resp.encoding = 'utf-8'
    except Exception as e:
        print(f"  [WARN] 详情页获取失败({article['title'][:30]}...): {e}", flush=True)
        return None, None, None

    html = resp.text
    soup = BeautifulSoup(html, 'html.parser')

    # 标题 - 从<title>标签取_前面的部分
    title_tag = soup.select_one('title')
    title = ''
    if title_tag:
        title = title_tag.get_text(strip=True).split('_')[0].strip()
    if not title:
        title = article['title']

    # 日期 - 优先从meta标签
    pub_date = ''
    for meta in soup.find_all('meta'):
        name = (meta.get('name') or '').lower()
        if name in ['publishdate', 'pubdate', 'date']:
            m = re.match(r'(\d{4}-\d{2}-\d{2})', meta.get('content', ''))
            if m:
                pub_date = m.group(1)
                break
    # 如果meta没取到，从页面文本提取
    if not pub_date:
        m = re.search(r'发布日期[：:]\s*(\d{4}-\d{2}-\d{2})', html)
        if m:
            pub_date = m.group(1)
    # 如果还没取到，从URL日期取
    if not pub_date:
        m = re.match(r'(\d{4})-(\d{2})-(\d{2})', article['date_str'][:4] + '-' +
                     article['date_str'][4:6] + '-' + article['date_str'][6:8])
        if m:
            pub_date = m.group(0)

    # 正文 - zw_con 或 trs_editor_view (TRS Editor)
    content = ''
    content_el = soup.select_one('.zw_con')
    if not content_el:
        content_el = soup.find(class_=re.compile(r'trs_?editor', re.I))
    if content_el:
        content_html = str(content_el)
        # 图片转为base64内嵌
        img_headers = {**HEADERS, 'Referer': BASE_URL + '/'}
        def embed_img(m):
            src = m.group(1)
            if src.startswith('./'):
                dir_part = article['url'].rsplit('/', 1)[0]
                src = dir_part + '/' + src[2:]
            elif src.startswith('/'):
                src = BASE_URL + src
            elif not src.startswith('http'):
                dir_part = article['url'].rsplit('/', 1)[0]
                src = dir_part + '/' + src
            return f'<img src=\"{src}\"' 
        content_html = re.sub(r'<img[^>]+?\bsrc=[\"\']([^\"\']+)[\"\']', embed_img, content_html)
        content = content_html

    # 处理附件链接(相对路径转绝对)
    content = re.sub(r'href=["\'](\./[^"\']+)["\']',
                     lambda m: f'href="{BASE_URL}{LIST_PATH}{m.group(1)[2:]}"',
                     content)

    summary = clean_text(content[:200]) if content else ''

    return title, pub_date, content, summary


def main():
    print(f"=== {SITE_NAME} 爬虫 ===", flush=True)
    print(f"数据库: {DB_PATH}", flush=True)

    session = requests.Session()
    conn = get_conn()
    cur = conn.cursor()

    total_new = 0
    total_skip = 0
    total_error = 0

    # 遍历所有列表页
    for page in range(8):
        articles = extract_articles(session, page)
        if not articles:
            print(f"  第{page+1}页: 无文章", flush=True)
            continue
        print(f"  第{page+1}页: {len(articles)}条文章", flush=True)

        for article in articles:
            # 跳过截止日期之前的
            date_int = int(article['date_str'])
            if date_int < CUTOFF_INT:
                total_skip += 1
                continue

            # 获取详情
            title, pub_date, content, summary = extract_detail(session, article)
            if not title or not content:
                total_skip += 1
                continue

            if not pub_date:
                total_error += 1
                continue

            date_rank = int(pub_date.replace('-', ''))
            source_url = f"{BASE_URL}{LIST_PATH}"

            try:
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw (page_url, title, content, site_name, publish_date, summary, category, date_rank, source_url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
                    (article['url'], title, content, SITE_NAME, pub_date, summary, CATEGORY, date_rank, source_url)
                )
                conn.commit()
                total_new += 1
                print(f"  [OK] {title[:40]}... | {pub_date}", flush=True)
            except Exception as e:
                print(f"  [ERROR] 入库失败: {title[:30]}... - {e}", flush=True)
                conn.rollback()
                total_error += 1

    conn.close()
    print(f"\n=== 完成: 新增 {total_new}, 跳过 {total_skip}, 错误 {total_error} ===", flush=True)


if __name__ == '__main__':
    main()