xj#dZddlZddlZddlZddlZddlZddlZejdddlm Z ddl m Z m Z ddl m Z ejjdejj!ejj#eddlmZdZej,ddD]IZej1d r eej5d ddZ1ej9sBeeZKe ed ed Zd ZdedZ e jBe dz jEdZ#ddiZ$ddZ%dZ&dZ'dZ(ddZ)e*dk(re)exsdyy#e$rYwxYw)u_ 海南省生态环境厅 - 环境影响评价 (政务公开 xxgk) 站点: https://hnsthb.hainan.gov.cn 栏目: /xxgk/0200/0202/hjywgl/hjyxpj/xxgkindex.html 列表: TRS 静态页 xxgkindex.html + xxgkindex_{N}.html (20条/页, 声明78页实际77, P78→302/404) 详情: meta ArticleTitle/PubDate + div.view.TRS_UEDITOR 正文 + div.other-word 附件 Nignore)urljoin)datetime timedelta) BeautifulSoup)push_to_searchdbz--pages==z[AutoPg] max_pages=u+海南省生态环境厅-环境影响评价zhnsthb.hainan.gov.cnzhttps://z,/xxgk/0200/0202/hjywgl/hjyxpj/xxgkindex.htmliG)daysz%Y-%m-%dz User-AgentzkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36c ltdD]y} tj|td}|jdk(rd|_|j cStd|d|jd|d ztjd {y #t$r!}td|d |d|d zYd}~=d}~wwxYw) N)headerstimeoutzutf-8z [WARN] z HTTP u, 重试r u 异常: ) rangerequestsgetHEADERS status_codeencodingtextprint Exceptiontimesleep)urllabelretryres &/root/gov_crawler/crawl_hnsthb_xxgk.pyfetchr%&sq  F S'2>A}}#$ vv KwfQ]]O8E!G9M N 1    F Kwis(57)D E E Fs=B !B  B3B..B3cg}t|d}|jdD]}|jdxsdj}|s*t j dd|j j}|jddjddj}|st||}t jd |r t|vrd}|jd } | r2t jd t| } | r| jd }|j|||d t!g} } |D]/} | d| vs | j#| d| j| 1| S)uB解析列表页: tr > a.new_title_a 标题链接 + 独立日期 tdlxmlz a.new_title_ahrefr\s+ · z https?://trz$\s*(20\d{2}-\d{2}-\d{2})\s*r )titlerpub_dater)rselectrstripresubget_textreplacermatchDOMAIN find_parentsearchstrgroupappendsetadd)htmllist_urlitemssoupar(r.rr/r-mseenuniqits r$ parse_listrH4sW E v &D [[ )If #**, vsAJJL1779 j"-55hCIIK h% 88L# &6+<  ]]4   A3r7KA771: uShGH)I,$D e9D HHRY  KKO Kcltjd|}|r|jdjndxs|}|j ddj ddj}tj dd|j}|j ddj ddj}d}tjd |}|r|jd}d}tjd |tj }|s%tjd |tj }|r|jd}d }d } tjd |D]D} | jd jdr|dz}|d ks.| j} n|dz }F| r|d| n|}t||}g} tjd|tj }|rtjd|jdD]t} | \} }tjd| tjs.tj dd|j}| jdt|| d|dv| r%|jdzdj!| z}|||fS)uJ详情: meta ArticleTitle/PubDate + TRS_UEDITOR 正文 + other-word 附件z+]*>(.*?)\s*

z,

]*>(.*)rz]*>|
z(.*?)z&]+href="([^"]+)"[^>]*>([^<]+)z*\.(doc|docx|pdf|xls|xlsx|wps|rar|zip|txt)$r)r*z

z

 )r2r9r;r1r5r3Sfinditer startswithstart clean_contentfindallIr<rrstripjoin)r?r list_titlerDr.r/contentbodydepthendmmattsrCr(names r$ parse_detailr]VsZ @$GA#$QWWQZ   " ;E MM*b ) 1 1(B ? E E GE FF>2u - 3 3 5E MM(B ' / /" = C C EEH FMA771:G \^bdfdhdhiA IIEtRTT Rwwqz++2D9 Bxx{%%d+ A:((*C  !tDSzdc* D 8$EAEqwwqzR QAJD$yyFbddSvvfc40668 l73+=*>bhOP  Q  .."T)DIIdO; (G ##rIctjdfd|}tjdfd|}tjdd|}tjdd|tj }tjd d|tj }d }tjd ||tj }tjd d|}|jS)uG清洗正文: 绝对化图片/链接, 规范化

, 保留段落结构z(]*src=")([^"]+)"cd|jdt|jdzdzSNr r"r;rrDpage_urls r$zclean_content..(AGGAJ1771:)FFLrIz(]*href=")([^"]+)"cd|jdt|jdzdzSr`rbrcs r$rezclean_content..rfrIz]*>z

zr)flagszc|jd}d|vr|Stjdd|}|jddjddjdd}tjd d|}|sdS|S) Nr]+>rr,r*u　z r))r;r2r3r5)rDsegtxts r$_drop_blank_paraz'clean_content.._drop_blank_paraswggaj S=JffZS)kk(C(003?GGRUVffVR%r%#%rIz

.*?

z

\s*(?:)?\s*

)r2r3rLr1)rWrdrns ` r$rPrPs 66-Ld TD 66,Ld TD 66+ud +D 66("d"$$ ?D 66&D =D& 66-!14rtt DD 660"d ;D ::<rIrc tddtdttdtj}g}d} |dk(rt}ntj dd|d}t |d|d }|std |d nt||}|std |d |dz dn|ddxsd}|ddxsd}td |dt|d|d|d |j||ddr,|ddtkrtd|dddtdn?t|tk\rtdtdn|dz }tjd)tg} } |D]/} | d| vs | j| d| j!| 1tdt|dt| dg} t#| D]\} }|dr |dtkrt |dd | dz}|s1t%||d|d!\}}}|sd"|v}|std#|dd$bt'j(d%d&|}t'j(d'd(|j+}|dd)xs|}| j!t||xs|d|||d|dd*| dzd+zdk(r*td,| dzd-t| d.t| dtjd/td0t| d1t-| d23tj|z j/}td4|d5d6y)7NrKz2==================================================u🚀 r zxxgkindex.html xxgkindex_z.htmlu第u页u ⚠ 第u页抓取失败 -> 结束u 第u!页: 空/无条目 -> 结束 (共u页)rr/?u页: u条 (z ~ )z [CUTOFF] z < u -> 停止u [AutoPg] 达到 max_pages=g?ru 列表共 u 条, 去重后 u 条u详情r.rju [SKIP] 空正文: (rkrr)r*r) site_namer.r/rVsummary source_urlrz [/u ] 已解析 g?u 有效 u 条 -> push_to_searchdb hnsthb_xxgk) batch_labelu ✅ 完成, 耗时 z.0fs)r SITE_NAMErnowLIST_URLr5r%rHlenextendCUTOFF_MAX_PGrrr=r>r< enumerater]r2r3r1r total_seconds) max_pagesrO all_itemspgrr?rAfirst_dlast_drErFrGvalidiitemr.r/rVhas_imgrmrvels r$runrs Bvh- E)  VH LLNEI B  7C""#3z"U5KLCSCt3-( IbT!:; < 4% E">r!tfDI J (:&-#r:&-# bTs5zl%yF81EF 9Z U2Yz%:V%C Kb * 56c&L M   2= 0 D E  a 33 8$D e9D HHRY  KKO LY((8T 4 HI ET?4  Z 06 9 T%[F1Q3%.1 #/d5k4=#Q xnG,U3BZL9:ffZW-ffVS#&,,.ds)$u " 4D$4u+;   ER<1  C!uAc$i[ SZLE F 39< Ic%j\!9 :;U 6 ,,.5 / / 1B !"S +,rI__main__)r)r)r)+__doc__r2ossysrrwarningsfilterwarnings urllib.parserrrbs4rpathinsertdirnameabspath__file__ crawler_librrargv_arNintsplitrisdigitrr}r7rr~strftimerrr%rHr]rPr__name__rIr$rsk! (277??277??8#<=>(  ((12,B }}Z  "((3*1-.G b'  y )* 9  fXI J (,,.9%0 0 : :: F G H D,$^6M-` z'.S!q   s;EE$#E$