#!/usr/bin/env python3
"""批量生成爬虫脚本 Part2 — 从 gdkfq 骨架复制 + 精确替换"""
import os, re as _re

OUT = os.path.expanduser("~/Crawler/gov_crawler/gen_out")
SKEL = open(os.path.join(OUT, "crawl_gdkfq_ezhou_hbgs.py"), encoding="utf-8").read()

def write(name, content):
    path = os.path.join(OUT, name)
    with open(path, "w", encoding="utf-8") as f:
        f.write(content)
    print("WROTE", name)

def re_replace_block(s, start_marker, new_block):
    idx = s.find(start_marker)
    if idx < 0:
        return s
    nxt = s.find("\ndef ", idx + 1)
    nxt2 = s.find('\nif __name__ == "__main__":', idx + 1)
    if nxt2 > 0 and (nxt < 0 or nxt2 < nxt):
        nxt = nxt2
    if nxt < 0:
        return s
    return s[:idx] + new_block.rstrip("\n") + s[nxt:]

def gen_from_skeleton(name, desc, url, base, list_url, site_name, group, category,
                      page_tpl, list_selector, date_sel, detail_selector):
    s = SKEL
    s = s.replace('"""\n葛店开发区 - 环保公示\nURL: http://gdkfq.ezhou.gov.cn/zwdt/hbgs/\n"""',
                  '"""\n%s\nURL: %s\n"""' % (desc, url))
    s = s.replace('BASE_URL = "http://gdkfq.ezhou.gov.cn"', 'BASE_URL = "%s"' % base)
    s = s.replace('LIST_URL = "http://gdkfq.ezhou.gov.cn/zwdt/hbgs/"', 'LIST_URL = "%s"' % list_url)
    s = s.replace('SITE_NAME = "葛店开发区-环保公示"', 'SITE_NAME = "%s"' % site_name)
    s = s.replace('GROUP_NAME = "湖北"', 'GROUP_NAME = "%s"' % group)
    s = s.replace('SCRIPT_NAME = "crawl_gdkfq_ezhou_hbgs.py"', 'SCRIPT_NAME = "%s"' % name)
    s = s.replace('CATEGORY = "环境保护"', 'CATEGORY = "%s"' % category)

    # build_list_url
    blk = "def build_list_url(page):\n    if page == 1:\n        return LIST_URL\n    return BASE_URL + \"" + page_tpl + "\" % page\n"
    s = re_replace_block(s, "def build_list_url(page):", blk)

    # parse_list
    blk2 = ("def parse_list(html_text, page_url):\n"
            "    items = []\n"
            "    if not BeautifulSoup:\n"
            "        return items\n"
            "    soup = BeautifulSoup(html_text, \"html.parser\")\n"
            "    for li in soup.select(\"" + list_selector + "\"):\n"
            "        a = li.find(\"a\", href=True)\n"
            "        if not a:\n"
            "            continue\n"
            "        href = a[\"href\"].strip()\n"
            "        title = a.get(\"title\", \"\") or a.get_text(strip=True)\n"
            "        date = \"\"\n"
            "        d = li.select_one(\"" + date_sel + "\")\n"
            "        if d:\n"
            "            date = d.get_text(strip=True)\n"
            "        title = clean_title(title)\n"
            "        if not title or len(title) < 4:\n"
            "            continue\n"
            "        if not href.startswith(\"http\"):\n"
            "            href = urllib.parse.urljoin(page_url, href)\n"
            "        items.append((href, title, date))\n"
            "    return items\n")
    s = re_replace_block(s, "def parse_list(html_text, page_url):", blk2)

    # parse_detail
    blk3 = ("def parse_detail(html_text, page_url):\n"
            "    title = \"\"; date = \"\"; content_html = \"\"\n"
            "    if BeautifulSoup:\n"
            "        soup = BeautifulSoup(html_text, \"html.parser\")\n"
            "        d = soup.select_one(\"" + detail_selector + "\")\n"
            "        if d:\n"
            "            content_html = \"\".join(str(c) for c in d.contents)\n"
            "        t = soup.find(\"meta\", attrs={\"name\": re.compile(\"ArticleTitle\", re.I)})\n"
            "        if t and t.get(\"content\"):\n"
            "            title = clean_title(t[\"content\"])\n"
            "        md = soup.find(\"meta\", attrs={\"name\": re.compile(\"PubDate\", re.I)})\n"
            "        if md and md.get(\"content\"):\n"
            "            dm = re.search(r\"([0-9]{4}-[0-9]{2}-[0-9]{2})\", md[\"content\"])\n"
            "            if dm:\n"
            "                date = dm.group(1)\n"
            "        if not title:\n"
            "            h1 = soup.find(\"h1\") or soup.find(\"h2\")\n"
            "            if h1:\n"
            "                title = clean_title(h1.get_text())\n"
            "    return title, date, content_html\n")
    s = re_replace_block(s, "def parse_detail(html_text, page_url):", blk3)
    return s

if __name__ == "__main__":
    # ===== gxdmjkq 南宁 =====
    write("crawl_gxdmjkq_nanning.py", gen_from_skeleton(
        "crawl_gxdmjkq_nanning.py",
        "广西-东盟经开区 - 环境核查审批",
        "http://gxdmjkq.nanning.gov.cn/zwgk/hjbh/hjhcsp/",
        "http://gxdmjkq.nanning.gov.cn", "http://gxdmjkq.nanning.gov.cn/zwgk/hjbh/hjhcsp/",
        "广西东盟经开区-环境核查审批", "广西", "环境保护",
        '/zwgk/hjbh/hjhcsp/index_%d.html',
        "ul.has_line li", "span.time", "div.content, div.article, .TRS_Editor, #content"
    ))
    # ===== youyang 酉阳 =====
    write("crawl_youyang_gsgg.py", gen_from_skeleton(
        "crawl_youyang_gsgg.py",
        "酉阳县 - 公示公告",
        "https://youyang.gov.cn/sy_236/gsgg/",
        "https://youyang.gov.cn", "https://youyang.gov.cn/sy_236/gsgg/",
        "酉阳县-公示公告", "重庆", "公示公告",
        '/sy_236/gsgg/index_%d.html',
        "ul.list li", "span", "div.TRS_Editor, div.article-content, div.content, .content-box"
    ))
    # ===== glg 漳州古雷 =====
    write("crawl_glg_zhangzhou.py", gen_from_skeleton(
        "crawl_glg_zhangzhou.py",
        "漳州古雷开发区 - 公示公告",
        "http://glg.zhangzhou.gov.cn/cms/html/zzglkfq/gsgg/index.html",
        "http://glg.zhangzhou.gov.cn", "http://glg.zhangzhou.gov.cn/cms/html/zzglkfq/gsgg/index.html",
        "漳州古雷开发区-公示公告", "福建", "公示公告",
        '/cms/html/zzglkfq/gsgg/index_%d.html',
        "ul#resources li", "span.date, span.time, em", "div#Content, div.content#Content, div.content"
    ))
    print("DONE part2 (3 sites)")