#!/usr/bin/env python3
"""铜陵市生态环境局 - 环评公众参与公示 爬虫
站点: sthjj.tl.gov.cn (政府站)
栏目: 专题专栏 > 环评公众参与公示 (hpgzcygs)
API: POST /queryList (current, pageSize, webSiteCode[], channelCode[])
详情: /tlssthjj/hpgzcygs/pc/content/content_{id}.html
"""

import sys, os, re, time, json, urllib.request, urllib.parse
sys.path.insert(0, "/root/gov_crawler")
from base_crawler import GovCrawler

SITE_NAME = "铜陵市生态环境局-环评公示"
DOMAIN = "sthjj.tl.gov.cn"
BASE = "https://sthjj.tl.gov.cn"
API_URL = f"{BASE}/queryList"
CHANNEL_CODE = "hpgzcygs"
SITE_CODE = "tlssthjj"
CTX = None


def http_post(url, data):
    """POST JSON/Form data and return parsed JSON"""
    req_data = urllib.parse.urlencode(data).encode()
    req = urllib.request.Request(url, data=req_data,
        headers={"Content-Type": "application/x-www-form-urlencoded",
                 "User-Agent": "Mozilla/5.0"})
    try:
        with urllib.request.urlopen(req, context=CTX, timeout=20) as r:
            return json.loads(r.read().decode('utf-8'))
    except Exception as e:
        print(f"  ⚠️ POST error: {e}")
        return None


class TlEnviroCrawler(GovCrawler):
    def do_crawl(self):
        total_pages = getattr(self, 'max_pages', 3)
        for page in range(1, total_pages + 1):
            resp = http_post(API_URL, {
                "current": page,
                "pageSize": 20,
                "webSiteCode[]": SITE_CODE,
                "channelCode[]": CHANNEL_CODE,
            })
            if not resp:
                print(f"  ⚠️ 第{page}页API失败")
                continue

            results = resp.get("data", {}).get("results", [])
            if not results:
                print(f"  第{page}页无数据")
                break

            items_found = 0
            for item in results:
                src = item.get("source", {})
                title = src.get("title", "").strip()
                pub_date = src.get("pubDate", "")
                if pub_date:
                    # "2026-05-25 15:15" -> "2026-05-25"
                    pub_date = pub_date.split(" ")[0]

                # detail URL
                urls_raw = src.get("urls", "{}")
                try:
                    urls = json.loads(urls_raw) if isinstance(urls_raw, str) else urls_raw
                except:
                    urls = {}
                detail_path = urls.get("pc", "")
                detail_url = BASE + detail_path if detail_path else ""

                if not title or not detail_url:
                    continue

                # 获取详情
                detail_html = self.http_get(detail_url)
                if not detail_html:
                    print(f"    ⚠️ 详情页获取失败")
                    continue

                # 提取详情
                detail_soup = self._make_soup(detail_html)

                # 详情页标题
                detail_title_el = detail_soup.find("div", class_="view-title")
                detail_title = detail_title_el.get_text(strip=True) if detail_title_el else title

                # 日期
                detail_date = pub_date
                date_el = detail_soup.find("div", class_="view-element")
                if date_el:
                    dm = re.search(r"(\d{4}-\d{2}-\d{2})", date_el.get_text())
                    if dm:
                        detail_date = dm.group(1)

                # 正文
                content_div = detail_soup.find("div", class_="n_content_c")
                content = str(content_div) if content_div else ""
                summary = content_div.get_text(strip=True)[:500] if content_div else title[:300]

                # 附件
                attachments = []
                for a_tag in detail_soup.find_all("a", href=re.compile(r"\.rar$|\.zip$|\.doc|\.pdf|pan\.baidu")):
                    attachments.append({
                        "name": a_tag.get_text(strip=True) or "附件",
                        "url": a_tag.get("href", "")
                    })
                for file_item in src.get("articleFiles", []):
                    if isinstance(file_item, dict) and file_item.get("url"):
                        attachments.append({
                            "name": file_item.get("fileName", "附件"),
                            "url": file_item["url"]
                        })

                self.store_item(
                    title=detail_title,
                    url=detail_url,
                    content=content,
                    date=detail_date,
                    summary=summary,
                )
                items_found += 1
                time.sleep(self.sync_delay)

            print(f"  📄 第{page}页: {items_found}条")
            time.sleep(self.sync_delay)

        print(f"\n  ✅ 完成: 新增{self._stats['new']}, 跳过{self._stats['skip']}, 错误{self._stats['errors']}")

    def _make_soup(self, html):
        from bs4 import BeautifulSoup
        return BeautifulSoup(html, "html.parser")


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--pages', type=int, default=3, help='页数')
    args = parser.parse_args()
    c = TlEnviroCrawler(
        site_name=SITE_NAME,
        domain=DOMAIN,
        url=BASE,
        sync_delay=0.3,
    )
    c.max_pages = args.pages
    c.run()