#!/usr/bin/env python3
"""肇庆市环科所环境科技有限公司 公示公告爬虫
站点: http://www.zqeia.com/h-col-101.html
CMS: 凡科(Faisco)建站
列表: JS内嵌newsList数据
详情: /h-nd-{id}.html
"""
import os, re, sys, json, time, sqlite3, warnings
import requests
import urllib.parse
warnings.filterwarnings("ignore")

BASE_URL = "http://www.zqeia.com"
LIST_PATH = "/h-col-101.html"
SITE_NAME = "肇庆环科所-公示公告"
SITE_DISPLAY = "肇庆市环科所环境科技有限公司"
GROUP_NAME = "环评平台"
SEARCH_DB = os.environ.get("SEARCH_DB", "/root/search.db")

_MAX_PG = None
for i, a in enumerate(sys.argv):
    if a == "--pages" and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break
if _MAX_PG is None and len(sys.argv) > 1 and sys.argv[1].isdigit():
    _MAX_PG = int(sys.argv[1])

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
           "Referer": BASE_URL + LIST_PATH}

# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)

def fetch_html(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print("[WARN] fetch failed: %s - %s" % (url, e), file=sys.stderr)
        return ""

def extract_items(html):
    items = []
    for m in re.finditer(r'{"aid":\d+,"id":(\d+),"title":"((?:\\.|[^"\\])*)","date":(\d+),', html):
        item_id = m.group(1)
        title = m.group(2)
        ts = int(m.group(3))
        if ts > 1000000000000:
            ts = ts / 1000
        date = time.strftime("%Y-%m-%d", time.localtime(ts)) if ts else ""
        url = "%s/h-nd-%s.html" % (BASE_URL, item_id)
        items.append({"url": url, "title": title, "date": date, "id": item_id})
    seen = set()
    unique = []
    for it in items:
        if it["url"] not in seen:
            seen.add(it["url"])
            unique.append(it)
    print("[INFO] 列表共 %d 条" % len(unique), file=sys.stderr)
    return unique

def extract_content_html(html):
    m = re.search(r'<div\s+class=["\']richContent[^"\']*["\'][^>]*>(.*?)</div>\s*<div\s+class=["\']line2["\']', html, re.DOTALL)
    if m:
        c = m.group(1)
        c = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', c, flags=re.DOTALL|re.I)
        c = re.sub(r'<div\s+class="[^"]*share[^"]*"[^>]*>.*?</div>', '', c, flags=re.DOTALL|re.I)
        c = re.sub(r'<div\s+class="[^"]*(?:print|tool|btn|operate)[^"]*"[^>]*>.*?</div>', '', c, flags=re.DOTALL|re.I)
        return c.strip()
    return ""

def html_to_markdown(html):
    if not html:
        return "", []
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")
    parts = []
    attachments = []
    for el in soup.find_all(["p", "table", "img"], recursive=True):
        if el.name == "img":
            src = el.get("src", "").strip()
            alt = el.get("alt", "").strip()
            if src and not src.startswith("data:"):
                if src.startswith("//"):
                    src = "http:" + src
                elif src.startswith("/"):
                    src = BASE_URL + src
                elif not src.startswith("http"):
                    src = BASE_URL + "/" + src.lstrip("/")
                parts.append("![%s](%s)" % (alt, src))
            continue
        if el.find_parent("table") and el.name != "table":
            continue
        if el.name == 'table':
            parts.append(html_table_to_html(el, BASE_URL))
        text = body_text(el)
        if text:
            for a in el.find_all("a", href=True):
                h = a["href"].lower()
                if h.endswith((".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar")):
                    ah = a["href"]
                    if ah.startswith("//"):
                        ah = "http:" + ah
                    elif ah.startswith("/"):
                        ah = BASE_URL + ah
                    attachments.append({"name": a.get_text(strip=True) or ah.split("/")[-1], "url": ah})
            parts.append(text)
    content = "\n\n".join(p for p in parts if p.strip())
    return content, attachments

def run():
    print("\n%s" % ("=" * 50), file=sys.stderr)
    print(" %s" % SITE_NAME, file=sys.stderr)
    print("%s" % ("=" * 50), file=sys.stderr)

    html = fetch_html(BASE_URL + LIST_PATH)
    if not html:
        print("[FAIL] 首页为空", file=sys.stderr)
        return

    items = extract_items(html)

    if _MAX_PG:
        limit = _MAX_PG * 20
        if len(items) > limit:
            print("[INFO] 限制为前 %d 条 (max_pages=%d)" % (limit, _MAX_PG), file=sys.stderr)
            items = items[:limit]

    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    existing = set()
    try:
        cur = conn.execute("SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,))
        for row in cur.fetchall():
            existing.add(row[0])
    except:
        pass
    conn.close()

    items = [it for it in items if it["url"] not in existing]
    print("[INFO] DB已有: %d 条, 待抓: %d 条" % (len(existing), len(items)), file=sys.stderr)

    new = skip = empty = errors = 0
    for i, item in enumerate(items, 1):
        html = ""
        for retry in range(3):
            try:
                r = requests.get(item["url"], headers=HEADERS, timeout=30)
                if r.status_code == 200:
                    r.encoding = "utf-8"
                    html = r.text
                    break
            except:
                if retry < 2:
                    time.sleep(2)

        if not html:
            errors += 1
            continue

        title = item["title"]
        date_text = item["date"]
        tm = re.search(r'<title>(.*?)<', html)
        if tm:
            raw = tm.group(1).strip()
            suffix = " - 肇庆市环科所环境科技有限公司"
            if raw.endswith(suffix):
                raw = raw[:-len(suffix)]
            if raw:
                title = raw
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        if dm:
            date_text = dm.group(1)

        raw_html = extract_content_html(html)
        text_len = len(re.sub(r'<[^>]+>', '', raw_html).strip()) if raw_html else 0
        if text_len < 20:
            empty += 1
            continue

        markdown, attachments = html_to_markdown(raw_html)
        attachments_json = json.dumps(attachments, ensure_ascii=False)

        conn = sqlite3.connect(SEARCH_DB, timeout=60)
        conn.execute("PRAGMA journal_mode=WAL")
        try:
            summary = markdown[:200].replace("\n", " ").strip()
            conn.execute("INSERT OR IGNORE INTO gov_raw (page_url, title, site_name, group_name, publish_date, summary, content, attachments, date_rank) VALUES (?,?,?,?,?,?,?,?,?)",
                         (item["url"], title, SITE_NAME, GROUP_NAME, date_text, summary, markdown, attachments_json, date_text))
            conn.commit()
            if conn.total_changes:
                new += 1
            else:
                skip += 1
        except Exception as e:
            print("[WARN] DB error: %s - %s" % (title[:30], e), file=sys.stderr)
            errors += 1
        finally:
            conn.close()

        if i % 5 == 0:
            print("  ...%d/%d" % (i, len(items)), file=sys.stderr)
        time.sleep(0.3)

    print("\n %s: 新增%d, 跳过%d, 空正文%d, 错误%d" % (SITE_NAME, new, skip, empty, errors), file=sys.stderr)
    print("[DONE] %s: %d new items" % (SITE_NAME, new))

if __name__ == "__main__":
    run()