#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""高青县-环评信息 (淄博 gongkai 子系统 site_gqxsthjj)
列表: 服务端渲染 table.sk-table.zfxxgk-list, tr > td.zfxxgk-list-title > a[doc_xxx.html] + td.zfxxgk-list-time
分页: URL 参数 ?open=fdzdgknr&nr_page=N&nr_per_page=10 (服务端渲染, 非 AJAX)
详情: doc_xxx.html, h1.details-title + div.details-content (排除 sk-breadcrumb), 日期 发布日期：YYYY-MM-DD
"""
import sys, os, re, time, html as html_lib, urllib.parse, sqlite3, json

_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, _BASE_DIR)
try:
    from crawler_lib import push_to_searchdb
except Exception:
    push_to_searchdb = None

try:
    from bs4 import BeautifulSoup
except ImportError:
    BeautifulSoup = None

import requests, urllib3
urllib3.disable_warnings()

SITE_NAME = "高青县-环评信息"
CATEGORY = "环评"
GROUP_NAME = "山东"
SCRIPT_NAME = os.path.basename(__file__)
BASE_URL = "http://www.gaoqing.gov.cn"
LIST_URL = "http://www.gaoqing.gov.cn/gongkai/site_gqxsthjj/channel_c_5f9f6c84b9dfe9bfb0a8c873_n_1605684619.4558/"

MAX_PAGES = int(os.environ.get("MAX_PAGES", "0") or "0")
DAYS_CUTOFF = 365 * 3
CUTOFF_DATE = time.strftime("%Y-%m-%d", time.localtime(time.time() - DAYS_CUTOFF * 86400))
PAGES = MAX_PAGES if MAX_PAGES >= 1 else 5
JSONL_PATH = os.environ.get("JSONL_PATH", "")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
    "Referer": "http://www.gaoqing.gov.cn/gongkai/site_gqxsthjj/channel_c_5f9f6c84b9dfe9bfb0a8c873_n_1605684619.4558/",
}

ROW_PAT = re.compile(
    r'<td class="zfxxgk-list-title">\s*<a href="([^"]+\.html)">([^<]+)</a>\s*</td>\s*<td class="zfxxgk-list-time">([^<]+)</td>',
    re.S,
)


def clean_title(t):
    if not t:
        return ""
    t = html_lib.unescape(t)
    t = re.sub(r"\s+", " ", t).strip()
    for bad in ("\u200b", "\u200c", "\u200d", "\ufeff"):
        t = t.replace(bad, "")
    return t


def fetch(url, retries=3):
    for i in range(retries):
        try:
            r = requests.get(url, headers=HEADERS, timeout=25, verify=False)
            r.encoding = r.apparent_encoding or "utf-8"
            if r.status_code == 200:
                return r.text
        except Exception:
            pass
        time.sleep(1.2)
    return None


def build_list_url(page):
    if page <= 1:
        return LIST_URL
    return LIST_URL + "?nr_page=%d" % page


def parse_list(html_text, page_url):
    items = []
    for m in ROW_PAT.finditer(html_text):
        href, t, d = m.group(1), clean_title(m.group(2)), m.group(3).strip()
        if not t or len(t) < 6:
            continue
        if not href.startswith("http"):
            href = urllib.parse.urljoin(BASE_URL, href)
        items.append((href, t, d))
    return items


def parse_detail(html_text, page_url):
    title = ""
    date = ""
    content_html = ""
    if BeautifulSoup:
        soup = BeautifulSoup(html_text, "html.parser")
        h = soup.select_one("h1.details-title")
        if h:
            title = clean_title(h.get_text(strip=True))
        d = soup.select_one("div.details-content div#details-content") or soup.select_one("#details-content") or soup.select_one("div.details-content")
        if d:
            # 去掉面包屑
            for b in d.select(".sk-breadcrumb"):
                b.decompose()
            content_html = "".join(str(c) for c in d.contents)
        di = soup.select_one(".details-release-date")
        if di:
            m = re.search(r"20\d{2}-\d{2}-\d{2}", di.get_text())
            if m:
                date = m.group(0)
    return title, date, content_html


def html_to_text(html_text, page_url):
    """BS 容器提取, 附件仅扩展名判定, 普通链接保留 <a>"""
    content = ""
    atts = []
    if BeautifulSoup:
        soup = BeautifulSoup(html_text, "html.parser")
        for tag in soup(["script", "style"]):
            tag.decompose()
        for div in soup.find_all("div", class_="bdsharebuttonbox"):
            div.decompose()
        for p in soup.find_all(["p", "div", "tr", "li", "h1", "h2", "h3", "h4", "blockquote"]):
            _link_repl(p, page_url)
        content = "".join(str(c) for c in soup.contents)
    return content, False, atts


_ATTACH_EXTS = (".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".zip", ".rar", ".7z", ".wps", ".et", ".dps", ".txt", ".csv")


def _link_repl(tag, page_url):
    """仅附件扩展名链接进 attachments; 普通链接保留内联 <a href>"""
    for a in tag.find_all("a", href=True):
        href = a["href"].strip()
        if not href.startswith("http"):
            href = urllib.parse.urljoin(page_url, href)
        low = href.lower()
        if low.startswith("javascript:") or low.startswith("#"):
            a.decompose()
            continue
        is_attach = any(low.endswith(ext) for ext in _ATTACH_EXTS)
        if is_attach:
            a.replace_with(" " + href + " ")
        else:
            a["href"] = href


def main():
    _pages = PAGES
    args = sys.argv[1:]
    for i, a in enumerate(args):
        if a == "--pages" and i + 1 < len(args):
            try:
                _pages = int(args[i + 1])
            except ValueError:
                pass
        elif a.startswith("--pages="):
            try:
                _pages = int(a.split("=", 1)[1])
            except ValueError:
                pass
    print("=== %s | pages=%d | cutoff=%s ===" % (SITE_NAME, _pages, CUTOFF_DATE), flush=True)
    batch = []
    jsonl_rows = []
    seen = set()
    new_count = 0
    skip_count = 0
    for page in range(1, _pages + 1):
        try:
            url = build_list_url(page)
            html_text = fetch(url)
            if not html_text:
                print("Page %d fetch failed" % page, flush=True)
                break
            items = parse_list(html_text, url)
        except Exception as e:
            print("Page %d fetch error: %s" % (page, e), flush=True)
            break
        if not items:
            print("Page %d: empty, stop" % page, flush=True)
            break
        print("Page %d: found %d items" % (page, len(items)), flush=True)
        for abs_url, title, date in items:
            if abs_url in seen:
                continue
            seen.add(abs_url)
            if date and date < CUTOFF_DATE:
                continue
            dhtml = fetch(abs_url)
            if not dhtml:
                skip_count += 1
                continue
            dtitle, ddate, content_html = parse_detail(dhtml, abs_url)
            if not dtitle:
                dtitle = title
            if not ddate:
                ddate = date
            if ddate and ddate < CUTOFF_DATE:
                skip_count += 1
                continue
            content, has_table, atts = html_to_text(content_html, abs_url)
            plain_len = len(re.sub(r"<[^>]+>", "", content).strip())
            no_para = not re.search(r"<p[ >]", content_html, re.I) and not has_table
            if plain_len < 10 or no_para:
                print("  skip empty/placeholder: %s" % dtitle[:40], flush=True)
                skip_count += 1
                continue
            item = {
                "site_name": SITE_NAME, "source_url": abs_url, "url": abs_url,
                "title": dtitle, "pub_date": ddate, "summary": "",
                "content": content, "category": CATEGORY, "tags": "",
                "group_name": GROUP_NAME,
                "attachments": "; ".join("%s|%s" % (u, t) for u, t in atts) if atts else "",
            }
            if JSONL_PATH:
                jsonl_rows.append(item)
                print("  JSONL + %s (%s)" % (dtitle[:40], ddate), flush=True)
                new_count += 1
                continue
            batch.append(item)
            new_count += 1
            print("  + %s (%s)" % (dtitle[:40], ddate), flush=True)
            time.sleep(0.08)
        time.sleep(0.3)

    if JSONL_PATH:
        with open(JSONL_PATH, "w", encoding="utf-8") as f:
            for row in jsonl_rows:
                f.write(json.dumps(row, ensure_ascii=False) + "\n")
        print("JSONL written: %s (%d rows)" % (JSONL_PATH, len(jsonl_rows)), flush=True)
    else:
        push_to_searchdb(batch, batch_label=SCRIPT_NAME)

    print("新增: %d" % new_count, flush=True)
    print("跳过: %d" % skip_count, flush=True)
    print("=== %s done ===" % SITE_NAME, flush=True)


if __name__ == "__main__":
    main()