#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
广州天赐高新材料 - 子公司动态 (环评公示) 爬虫
CMS: 自定义PHP
WAF: 首请求403，带session cookie第二次放行
列表: /zgsdt/ (第1页), /zgsdt_{n}/ (第n页, n=2~7)
分页: ~8条/页, 共55条/7页
详情: /zgsdt/{id}.html
详情标题: <title>
详情正文: div.newsxqnyb > p
详情日期: Published Date: YYYY-MM-DD

用法:
  python3 crawl_tinci.py
  python3 crawl_tinci.py --pages 7
"""

import re
import sys
import time
import requests
import sqlite3
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://www.tinci.com"
LIST_PAGE1 = BASE_URL + "/zgsdt/"
DB_PATH = "/root/search.db"
SITE_NAME = "天赐材料-子公司动态"
CATEGORY = GROUP = "环评"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}
MAX_PAGES = 7
TIMEOUT = 30


def bypass_get(url):
    """WAF: 首请求403，第二请求带cookie放行"""
    sess = requests.Session()
    sess.headers.update(HEADERS)
    sess.get(url, timeout=TIMEOUT)
    r = sess.get(url, timeout=TIMEOUT)
    return r.text


def fetch(url):
    for attempt in range(3):
        try:
            return bypass_get(url)
        except Exception as e:
            if attempt == 2:
                print("  [WARN] 获取失败 (%d/3): %s - %s" % (attempt + 1, url, e), file=sys.stderr)
                return ""
            time.sleep(2)


def extract_list_items(html):
    items = []
    soup = BeautifulSoup(html, "html.parser")
    news_div = soup.find("div", class_="news_nyw")
    if not news_div:
        return items
    for nr in news_div.find_all("div", class_="nr"):
        a_tag = nr.find_parent("a")
        if not a_tag:
            a_tag = nr.find("a")
            if not a_tag:
                continue
        if not a_tag:
            continue
        href = a_tag.get("href", "")
        if not href or "/zgsdt/" not in href:
            continue
        full_url = urljoin(BASE_URL, href)
        # 标题：从 <h2> 提取；日期：从 <span> 提取
        title = ""
        date = ""
        span_tag = nr.find("span")
        if span_tag:
            date = span_tag.get_text(strip=True)
        h2_tag = nr.find("h2")
        if h2_tag:
            title = h2_tag.get_text(strip=True)
        items.append((full_url, title, date))
    return items


def extract_detail(html):
    soup = BeautifulSoup(html, "html.parser")

    # 标题：<title> 去掉 "_广州天赐..." 后缀
    title = ""
    title_tag = soup.find("title")
    if title_tag and title_tag.string:
        title = title_tag.string.strip()
        title = re.sub(r"_广州天赐高新材料股份有限公司$", "", title).strip()

    # 日期：newsxqnya 中的 Published Date
    date = ""
    meta_div = soup.find("div", class_="newsxqnya")
    if meta_div:
        m = re.search(r"Published Date[：:]\s*(\d{4}-\d{2}-\d{2})", meta_div.get_text())
        if m:
            date = m.group(1)

    # 正文：newsxqnyb 内的 <p> 标签
    content = ""
    content_div = soup.find("div", class_="newsxqnyb")
    if content_div:
        parts = []
        for p in content_div.find_all("p"):
            text = p.get_text(strip=True)
            if text and len(text) > 2 and text != title:
                parts.append(text)
        content = "\n\n".join(parts)

    # 附件：PDF链接
    attachments = []
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if ".pdf" in href.lower():
            name = a.get_text(strip=True) or href.split("/")[-1]
            attachments.append({"name": name, "url": urljoin(BASE_URL, href)})

    return title, date, content, attachments


def push_to_db(conn, items):
    saved = 0
    skipped = 0
    for url, title, date, content, attachments in items:
        import json
        attachments_json = json.dumps(attachments, ensure_ascii=False) if attachments else ""
        summary = content[:200].replace("\n", " ") if content else title
        if not content:
            if attachments:
                content = '\n'.join('<p><a href="%s">%s</a></p>' % (a['url'], a['name']) for a in attachments)
            else:
                content = title
        try:
            cur = conn.execute(
                "INSERT OR REPLACE INTO gov_raw (page_url, title, site_name, publish_date, content, date_rank, summary, attachments, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'crawl_tinci.py')",
                (url, title, SITE_NAME, date, content, date, summary, attachments_json)
            )
            if cur.rowcount > 0:
                saved += 1
            else:
                skipped += 1
        except Exception as e:
            print("  [ERR] DB: %s - %s" % (url, e), file=sys.stderr)
            skipped += 1
    return saved, skipped


def main():
    pages = MAX_PAGES
    for i, a in enumerate(sys.argv):
        if a == "--pages" and i + 1 < len(sys.argv):
            pages = int(sys.argv[i + 1])
            break

    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    all_items = []

    print("天赐材料-子公司动态: 爬取 %d 页" % pages)

    for idx in range(pages):
        if idx == 0:
            list_url = LIST_PAGE1
        else:
            list_url = BASE_URL + "/zgsdt_%d/" % (idx + 1)

        print("  [列表页 %d/%d] %s" % (idx + 1, pages, list_url))
        html = fetch(list_url)
        if not html:
            continue

        items = extract_list_items(html)
        if not items:
            print("    -> 无数据，停止")
            break

        print("    -> 找到 %d 条" % len(items))

        for item_url, item_title, item_date in items:
            time.sleep(0.5)
            print("    [%d] %s..." % (len(all_items) + 1, item_title[:40]), end=" ", flush=True)
            detail_html = fetch(item_url)
            if not detail_html:
                print("fail")
                continue

            title, date, content, attachments = extract_detail(detail_html)
            if not title:
                title = item_title
            if not date:
                date = item_date

            all_items.append((item_url, title, date, content, attachments))
            status = "ok" if content else "empty"
            attach_msg = ""
            if attachments:
                attach_msg = " +%dpdf" % len(attachments)
            print("(%s, %d字%s)" % (status, len(content), attach_msg))

    if all_items:
        saved, skipped = push_to_db(conn, all_items)
        conn.commit()
        print("\n完成! 新增: %d, 跳过: %d, 总共: %d" % (saved, skipped, len(all_items)))

    conn.close()
    return 0


if __name__ == "__main__":
    sys.exit(main())