Ë
    u6‚jÀ  ã                   ó|  — d Z ddlZddlZej                  j                  d«      Z eej                  j                  ed«      d¬«      j                  «       Z
d„ Zd„ Zd	„ Zed
k(  rW ed edddddddddddd«      «        ed edddddddddddd «      «        ed! ed!d"d#d$d#d%d&dd'd(d)d*«      «        ed+«       yy),uH   æ‰¹é‡�ç”Ÿæˆ�çˆ¬è™«è„šæœ¬ Part2 â€” ä»Ž gdkfq éª¨æž¶å¤�åˆ¶ + ç²¾ç¡®æ›¿æ�¢é    Nz~/Crawler/gov_crawler/gen_outzcrawl_gdkfq_ezhou_hbgs.pyúutf-8©Úencodingc                 óÌ   — t         j                  j                  t        | «      }t	        |dd¬«      5 }|j                  |«       d d d «       t        d| «       y # 1 sw Y   ŒxY w)NÚwr   r   ÚWROTE)ÚosÚpathÚjoinÚOUTÚopenÚwriteÚprint)ÚnameÚcontentr
   Úfs       úgen_crawlers_p2.pyr   r      sO   € Ü�7‰7�<‰<œ˜TÓ"€DÜ	ˆd�C 'Ô	*ð ¨aØ	�‰�Ô÷ä	ˆ'�4Õ÷ð ús   ³AÁA#c                 óð   — | j                  |«      }|dk  r| S | j                  d|dz   «      }| j                  d|dz   «      }|dkD  r|dk  s||k  r|}|dk  r| S | d | |j                  d«      z   | |d  z   S )Nr   z
def é   z
if __name__ == "__main__":ú
)ÚfindÚrstrip)ÚsÚstart_markerÚ	new_blockÚidxÚnxtÚnxt2s         r   Úre_replace_blockr      s�   € Ø
�&‰&�Ó
€CØ
ˆQ‚wØˆØ
�&‰&�˜3 ™7Ó
#€CØ�6‰6Ð0°#¸±'Ó:€DØˆa‚x�S˜1’W  s¢
ØˆØ
ˆQ‚wØˆØˆTˆcˆ7�Y×%Ñ% dÓ+Ñ+¨a°°¨gÑ5Ð5ó    c                 óÊ  — t         }|j                  dd|›d|›d�«      }|j                  dd|z  «      }|j                  dd|z  «      }|j                  d	d
|z  «      }|j                  dd|z  «      }|j                  dd| z  «      }|j                  dd|z  «      }d|z   dz   }t        |d|«      }d|	z   dz   |
z   dz   }t        |d|«      }d|z   dz   }t        |d|«      }|S )NuP   """
è‘›åº—å¼€å�‘åŒº - çŽ¯ä¿�å…¬ç¤º
URL: http://gdkfq.ezhou.gov.cn/zwdt/hbgs/
"""z"""
z
URL: z
"""z&BASE_URL = "http://gdkfq.ezhou.gov.cn"zBASE_URL = "%s"z1LIST_URL = "http://gdkfq.ezhou.gov.cn/zwdt/hbgs/"zLIST_URL = "%s"u*   SITE_NAME = "è‘›åº—å¼€å�‘åŒº-çŽ¯ä¿�å…¬ç¤º"zSITE_NAME = "%s"u   GROUP_NAME = "æ¹–åŒ—"zGROUP_NAME = "%s"z)SCRIPT_NAME = "crawl_gdkfq_ezhou_hbgs.py"zSCRIPT_NAME = "%s"u   CATEGORY = "çŽ¯å¢ƒä¿�æŠ¤"zCATEGORY = "%s"z[def build_list_url(page):
    if page == 1:
        return LIST_URL
    return BASE_URL + "z	" % page
zdef build_list_url(page):z±def parse_list(html_text, page_url):
    items = []
    if not BeautifulSoup:
        return items
    soup = BeautifulSoup(html_text, "html.parser")
    for li in soup.select("zÚ"):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"].strip()
        title = a.get("title", "") or a.get_text(strip=True)
        date = ""
        d = li.select_one("a6  ")
        if d:
            date = d.get_text(strip=True)
        title = clean_title(title)
        if not title or len(title) < 4:
            continue
        if not href.startswith("http"):
            href = urllib.parse.urljoin(page_url, href)
        items.append((href, title, date))
    return items
z$def parse_list(html_text, page_url):z¾def parse_detail(html_text, page_url):
    title = ""; date = ""; content_html = ""
    if BeautifulSoup:
        soup = BeautifulSoup(html_text, "html.parser")
        d = soup.select_one("a˜  ")
        if d:
            content_html = "".join(str(c) for c in d.contents)
        t = soup.find("meta", attrs={"name": re.compile("ArticleTitle", re.I)})
        if t and t.get("content"):
            title = clean_title(t["content"])
        md = soup.find("meta", attrs={"name": re.compile("PubDate", re.I)})
        if md and md.get("content"):
            dm = re.search(r"([0-9]{4}-[0-9]{2}-[0-9]{2})", md["content"])
            if dm:
                date = dm.group(1)
        if not title:
            h1 = soup.find("h1") or soup.find("h2")
            if h1:
                title = clean_title(h1.get_text())
    return title, date, content_html
z&def parse_detail(html_text, page_url):)ÚSKELÚreplacer   )r   ÚdescÚurlÚbaseÚlist_urlÚ	site_nameÚgroupÚcategoryÚpage_tplÚlist_selectorÚdate_selÚdetail_selectorr   ÚblkÚblk2Úblk3s                   r   Úgen_from_skeletonr2      sJ  € ä€AØ	�	‰	ÑgÚ-1²3Ð7ó	9€Aà	�	‰	Ð:Ð<MÐPTÑ<TÓU€AØ	�	‰	ÐEÐGXÐ[cÑGcÓd€AØ	�	‰	Ð>Ð@RÐU^Ñ@^Ó_€AØ	�	‰	Ð)Ð+>ÀÑ+FÓG€AØ	�	‰	Ð=Ð?SÐVZÑ?ZÓ[€AØ	�	‰	Ð-Ð/@À8Ñ/KÓL€Að lÐnvÑ
vð  zGñ  G€CÜ˜Ð7¸Ó=€Að+ð
 .;ñ;ð
>+ñ+ð .6ñ6ð	9!ñ!€Dô, 	˜ÐBÀDÓI€Að-ð 0?ñ	?ðB5ñ	5€Dô( 	˜ÐDÀdÓK€AØ€Hr    Ú__main__zcrawl_gxdmjkq_nanning.pyu+   å¹¿è¥¿-ä¸œç›Ÿç»�å¼€åŒº - çŽ¯å¢ƒæ ¸æŸ¥å®¡æ‰¹z/http://gxdmjkq.nanning.gov.cn/zwgk/hjbh/hjhcsp/zhttp://gxdmjkq.nanning.gov.cnu(   å¹¿è¥¿ä¸œç›Ÿç»�å¼€åŒº-çŽ¯å¢ƒæ ¸æŸ¥å®¡æ‰¹u   å¹¿è¥¿u   çŽ¯å¢ƒä¿�æŠ¤z/zwgk/hjbh/hjhcsp/index_%d.htmlzul.has_line liz	span.timez/div.content, div.article, .TRS_Editor, #contentzcrawl_youyang_gsgg.pyu   é…‰é˜³åŽ¿ - å…¬ç¤ºå…¬å‘Šz#https://youyang.gov.cn/sy_236/gsgg/zhttps://youyang.gov.cnu   é…‰é˜³åŽ¿-å…¬ç¤ºå…¬å‘Šu   é‡�åº†u   å…¬ç¤ºå…¬å‘Šz/sy_236/gsgg/index_%d.htmlz
ul.list liÚspanz>div.TRS_Editor, div.article-content, div.content, .content-boxzcrawl_glg_zhangzhou.pyu$   æ¼³å·žå�¤é›·å¼€å�‘åŒº - å…¬ç¤ºå…¬å‘Šz<http://glg.zhangzhou.gov.cn/cms/html/zzglkfq/gsgg/index.htmlzhttp://glg.zhangzhou.gov.cnu"   æ¼³å·žå�¤é›·å¼€å�‘åŒº-å…¬ç¤ºå…¬å‘Šu   ç¦�å»ºz$/cms/html/zzglkfq/gsgg/index_%d.htmlzul#resources lizspan.date, span.time, emz-div#Content, div.content#Content, div.contentzDONE part2 (3 sites))Ú__doc__r	   ÚreÚ_rer
   Ú
expanduserr   r   r   Úreadr"   r   r   r2   Ú__name__r   © r    r   ú<module>r<      s  ðá Nß à‡g�g×ÑÐ8Ó9€ÙˆB�G‰G�L‰L˜Ð9Ó:ÀWÔM×RÑRÓT€òò
6ò?ðB ˆzÒá	Ð
$Ñ&7Ø"Ø5Ø9Ø'Ð)ZØ2°H¸nØ)Ø˜+Ð'Xó'ô ñ 
Ð
!Ñ#4ØØ"Ø-Ø Ð"GØ  (¨NØ$Ø�fÐ^ó$ô ñ 
Ð
"Ñ$5Ø Ø.ØFØ%Ð'eØ,¨h¸Ø.ØÐ5Ð7fó%ô ñ 
Ð
 Õ!ð? r    