Ë
    u6‚jŒ  ã                   ó°  — d Z ddlZddlZej                  j                  d«      Z eej                  j                  ed«      d¬«      j                  «       Z
d„ Zd„ Zd	„ Zed
k(  rq ed edddddddddddd«      «        ed edddddddddddd «      «        ed! ed!d"d#d$d#d%d&dd'd(dd)«      «        ed* ed*d+d,d-d,d.d/dd0d1dd2«      «        ed3«       yy)4uA   æ‰¹é‡�ç”Ÿæˆ�çˆ¬è™«è„šæœ¬ Part3 â€” dataproxy/ç‰¹æ®Šåˆ—è¡¨/API ç«™é    Nz~/Crawler/gov_crawler/gen_outzcrawl_gdkfq_ezhou_hbgs.pyúutf-8©Úencodingc                 óÌ   — t         j                  j                  t        | «      }t	        |dd¬«      5 }|j                  |«       d d d «       t        d| «       y # 1 sw Y   ŒxY w)NÚwr   r   ÚWROTE)ÚosÚpathÚjoinÚOUTÚopenÚwriteÚprint)ÚnameÚcontentr
   Úfs       úgen_crawlers_p3.pyr   r      sO   € Ü�7‰7�<‰<œ˜TÓ"€DÜ	ˆd�C 'Ô	*ð ¨aØ	�‰�Ô÷ä	ˆ'�4Õ÷ð ús   ³AÁA#c                 óð   — | j                  |«      }|dk  r| S | j                  d|dz   «      }| j                  d|dz   «      }|dkD  r|dk  s||k  r|}|dk  r| S | d | |j                  d«      z   | |d  z   S )Nr   z
def é   z
if __name__ == "__main__":ú
)ÚfindÚrstrip)ÚsÚstart_markerÚ	new_blockÚidxÚnxtÚnxt2s         r   Úre_replace_blockr      s�   € Ø
�&‰&�Ó
€CØ
ˆQ‚wØˆØ
�&‰&�˜3 ™7Ó
#€CØ�6‰6Ð0°#¸±'Ó:€DØˆa‚x�S˜1’W  s¢
ØˆØ
ˆQ‚wØˆØˆTˆcˆ7�Y×%Ñ% dÓ+Ñ+¨a°°¨gÑ5Ð5ó    c                 óÊ  — t         }|j                  dd|›d|›d�«      }|j                  dd|z  «      }|j                  dd|z  «      }|j                  d	d
|z  «      }|j                  dd|z  «      }|j                  dd| z  «      }|j                  dd|z  «      }d|z   dz   }t        |d|«      }d|	z   dz   |
z   dz   }t        |d|«      }d|z   dz   }t        |d|«      }|S )u   æ™®é€š HTML åˆ—è¡¨ç«™uP   """
è‘›åº—å¼€å�‘åŒº - çŽ¯ä¿�å…¬ç¤º
URL: http://gdkfq.ezhou.gov.cn/zwdt/hbgs/
"""z"""
z
URL: z
"""z&BASE_URL = "http://gdkfq.ezhou.gov.cn"zBASE_URL = "%s"z1LIST_URL = "http://gdkfq.ezhou.gov.cn/zwdt/hbgs/"zLIST_URL = "%s"u*   SITE_NAME = "è‘›åº—å¼€å�‘åŒº-çŽ¯ä¿�å…¬ç¤º"zSITE_NAME = "%s"u   GROUP_NAME = "æ¹–åŒ—"zGROUP_NAME = "%s"z)SCRIPT_NAME = "crawl_gdkfq_ezhou_hbgs.py"zSCRIPT_NAME = "%s"u   CATEGORY = "çŽ¯å¢ƒä¿�æŠ¤"zCATEGORY = "%s"z[def build_list_url(page):
    if page == 1:
        return LIST_URL
    return BASE_URL + "z	" % page
zdef build_list_url(page):z±def parse_list(html_text, page_url):
    items = []
    if not BeautifulSoup:
        return items
    soup = BeautifulSoup(html_text, "html.parser")
    for li in soup.select("zÚ"):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"].strip()
        title = a.get("title", "") or a.get_text(strip=True)
        date = ""
        d = li.select_one("a6  ")
        if d:
            date = d.get_text(strip=True)
        title = clean_title(title)
        if not title or len(title) < 4:
            continue
        if not href.startswith("http"):
            href = urllib.parse.urljoin(page_url, href)
        items.append((href, title, date))
    return items
z$def parse_list(html_text, page_url):z¾def parse_detail(html_text, page_url):
    title = ""; date = ""; content_html = ""
    if BeautifulSoup:
        soup = BeautifulSoup(html_text, "html.parser")
        d = soup.select_one("a˜  ")
        if d:
            content_html = "".join(str(c) for c in d.contents)
        t = soup.find("meta", attrs={"name": re.compile("ArticleTitle", re.I)})
        if t and t.get("content"):
            title = clean_title(t["content"])
        md = soup.find("meta", attrs={"name": re.compile("PubDate", re.I)})
        if md and md.get("content"):
            dm = re.search(r"([0-9]{4}-[0-9]{2}-[0-9]{2})", md["content"])
            if dm:
                date = dm.group(1)
        if not title:
            h1 = soup.find("h1") or soup.find("h2")
            if h1:
                title = clean_title(h1.get_text())
    return title, date, content_html
z&def parse_detail(html_text, page_url):)ÚSKELÚreplacer   )r   ÚdescÚurlÚbaseÚlist_urlÚ	site_nameÚgroupÚcategoryÚpage_tplÚlist_selectorÚdate_selÚdetail_selectorr   ÚblkÚblk2Úblk3s                   r   Ú	gen_plainr2      sI  € ô 	€AØ	�	‰	ÑgÚ-1²3Ð7ó	9€Aà	�	‰	Ð:Ð<MÐPTÑ<TÓU€AØ	�	‰	ÐEÐGXÐ[cÑGcÓd€AØ	�	‰	Ð>Ð@RÐU^Ñ@^Ó_€AØ	�	‰	Ð)Ð+>ÀÑ+FÓG€AØ	�	‰	Ð=Ð?SÐVZÑ?ZÓ[€AØ	�	‰	Ð-Ð/@À8Ñ/KÓL€Aà
kÐnvÑ
vð  zGñ  G€CÜ˜Ð7¸Ó=€Að+ð
 .;ñ;ð
>+ñ+ð .6ñ6ð	9!ñ!€Dô, 	˜ÐBÀDÓI€Að-ð 0?ñ	?ðB5ñ	5€Dô( 	˜ÐDÀdÓK€AØ€Hr    Ú__main__zcrawl_fengning_gsgg.pyu$   ä¸°å®�æ»¡æ—�è‡ªæ²»åŽ¿ - å…¬å‘Šå…¬ç¤ºzAhttp://fengning.gov.cn/col/col4511/index.html?number=C20007C00011zhttp://fengning.gov.cnu"   ä¸°å®�æ»¡æ—�è‡ªæ²»åŽ¿-å…¬å‘Šå…¬ç¤ºu   æ²³åŒ—u   å…¬ç¤ºå…¬å‘Šz./col/col4511/index_%d.html?number=C20007C00011zul liÚspanz/div.article, div.content, div.TRS_Editor, #zoomzcrawl_sthjt_sc.pyuK   å››å·�çœ�ç”Ÿæ€�çŽ¯å¢ƒåŽ… - æ‹Ÿæ‰¹å‡†å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�æŠ¥å‘Šä¹¦å…¬ç¤ºz7https://sthjt.sc.gov.cn/sthjt/c103940/list_level2.shtmlzhttps://sthjt.sc.gov.cnu(   å››å·�çœ�ç”Ÿæ€�çŽ¯å¢ƒåŽ…-æ‹Ÿæ‰¹å‡†çŽ¯è¯„u   å››å·�u   çŽ¯è¯„å®¡æ‰¹z#/sthjt/c103940/list_level2_%d.shtmlzdiv.list dl ddzspan.frz=div.TRS_Editor, div.content, div.list_con, .Custom_UnionStylezcrawl_puershi_hjxx.pyu0   æ™®æ´±å¸‚ - å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·ä¿¡æ�¯zXhttps://www.puershi.gov.cn/zwgk/zfxxgk2/zfbmxxgk/pessthjj/fdzdgknr/hjbh/jsxmhjyxpjxx.htmzhttps://www.puershi.gov.cnu   æ™®æ´±å¸‚-å»ºè®¾é¡¹ç›®çŽ¯è¯„u   äº‘å�—zA/zwgk/zfxxgk2/zfbmxxgk/pessthjj/fdzdgknr/hjbh/jsxmhjyxpjxx/%d.htmzdiv.gkpt_rig ul liz-div#vsb_content, div.content, div#content_divzcrawl_yanshi_hjbh.pyu0   å�ƒå¸ˆåŒº - å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·ä¿¡æ�¯z7https://www.yanshi.gov.cn/zdlyxxgk/hjbhxx/jsxmhjyxpjxx/zhttps://www.yanshi.gov.cnu   å�ƒå¸ˆåŒº-å»ºè®¾é¡¹ç›®çŽ¯è¯„u   æ²³å�—z+/zdlyxxgk/hjbhxx/jsxmhjyxpjxx/index_%d.htmlzdiv.news_list ul li, ul liz6div#contents.newscontents, div.ny_content, div.contentzDONE part3 (4 sites))Ú__doc__r	   ÚreÚ_rer
   Ú
expanduserr   r   r   Úreadr"   r   r   r2   Ú__name__r   © r    r   ú<module>r<      sE  ðá Gß à‡g�g×ÑÐ8Ó9€ÙˆB�G‰G�L‰L˜Ð9Ó:ÀWÔM×RÑRÓT€òò
6ò=ð~ ˆzÒá	Ð
"¡IØ Ø.ØKØ Ð"eØ,¨h¸Ø8Ø�ÐJó%ô ñ 
Ð
™yØØUØAØ!Ð#\Ø2°H¸nØ-Ø˜)Ð%dó ô ñ 
Ð
!¡9ØØ:ØbØ$ð  'AØ&¨°.ØKØ˜fÐ&Uó$ô ñ 
Ð
 ¡)ØØ:ØAØ#Ð%^Ø&¨°.Ø5Ø$ fÐ.fó#ô ñ 
Ð
 Õ!ðS r    