$j~D d Z ddlZddlZddlZddlZddlZddlZddlZ ddl
m
Z
m Z ddl m
Z
ej j! dd ZdZdZd Zd
Zd Zd Zed
z Z e
j2 e d z
j5 d ZdddedZ dZ e ej> D ]k \ Z Z!e!jE d r e#e!jI dd d Z 4e!dk( s:e dz e&ej> k sS e#ej> e dz Z m e dk\ re ndZ'd Z(d!dZ)d Z*d Z+d Z,d Z-e.d k( r e- yy# e$ r dZ
Y w xY w# e%$ r Y w xY w# e%$ r Y w xY w)"u
crawl_ynwd.py — 武定县（楚雄）项目环评审批结果
https://ynwd.gov.cn/zfxxgk/fdzdgknr/zdmsjgysyjsxx/sthj/xmhpspjg.htm
CMS: 拓尔思(TRS) 政务公开平台（静态 .htm，静态分页）
列表: 20 条/页
分页: 静态「下页」链接（数字倒序），当前第 1 页 = xmhpspjg.htm，
第 2 页 = xmhpspjg/7.htm …（共 151 条 / 8 页）→ 逐页跟随「下页」链接
详情: /info/2554/.htm
标题: div.contentwarp > h1（回退 去后缀）
日期: div.label 「日期：YYYY年MM月DD日」
正文: div#vsb_content_500（内含 div.v_news_content，保留 <p>/<table>）
N)datetime timedelta)
BeautifulSoup
JSONL_PATH utf-8u" 武定县-项目环评审批结果u 云南z
crawl_ynwd.pyu 项目环评审批结果zhttps://ynwd.gov.cnz0/zfxxgk/fdzdgknr/zdmsjgysyjsxx/sthj/xmhpspjg.htmiG )daysz%Y-%m-%dzkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-LanguageReferer z--pages== z--pagesc | syt j | } | j dd j dd j dd j dd } t j dd| } t j dd| } t j dd| } t j d d
| } | j S ) Nr u u u u <[^>]+>u3 ^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*u ^[•·]\s*\s+ )html_libunescapereplaceresubstrip)ts
crawl_ynwd.py clean_titler ; s ! A (B ''"5==hKSST\^`aA
z2q!A
Er1MA
~r1%A
vsA A 779 c t t }d }t d D ] } dd l}dd l}|j |j j |j | ||d }|j dk( rt |_ |j c S d|j z }t j d t# d | d d
d |d
y # t $ r}t | d d }Y d }~Jd }~ww xY w)N r F)headerstimeoutverify zHTTP %sP z [fetch fail] Z z -> Tflush)dictHEADERSrangerequestsurllib3disable_warnings
exceptionsInsecureRequestWarningget status_codeENCencodingtext Exceptionstrtimesleepprint) urlr hlast_errattemptr+ r, res r fetchr@ G s W
AH8 # $ $ $W%7%7%N%N O S!WU KA}}#
vv
1==0H
1
s3Bx
:$G #1vcr{H #s A'C C C)C$$C)c
. &'( | sddg fS t j dd| t j } t j dd| t j } t j dd| t j t j z } t j dd| t j t j z } g &i 't j dd| } t r t | d }|j d
d D ] }|d
}|j
d s;|j
d s*t j d|t j sd|v sU|xs dj }|j
d xs |j
d xs dj }|r(t j d|t j s|}n|xs |}|j d xs[ |j
d xsH |j
d xs5 t j j |j d d xs d}t j j# | } &j% | |f dt' ' z }
d| d |d '|
< |j) |
|j d D ]` } | j
d xs | j
d xs d} t j d | t j sQ| j+ b |j d! D ]@ }
|
j d }|s|
j d r+g }|D ] } | j
d xs | j
d xs d} | s.t j j# | } | j
d" xs | j
d xs dj xs5 t j j | j d d xs d#}|j% d| d |d |sd$t' ' z }
d%d&j- | z d'z '|
< |
j) |
C |j d D ] } | j
d xs | j
d xs d} | s.t j j# | } | j
d" xs | j
d xs dj xs5 t j j | j d d xs d#}d$t' ' z }
d(| d |d)'|
< | j) |
t/ | } &'fd*}t j d+|| t j t j z } t r t | d }|j d D ]% }d,D ] }|j2 j5 |d- ' |j d. D ]h }|j d sD|j7 d/ s3|j7 d s"|j7 d
s|j+ Y|j9 j t/ | } g ((fd0}t j d1| t j rd2nd}d-}|| k7 r@| }t j d3|| t j t j z } || k7 r@t j d'd4| t j } t j d5d6| t j } t j d7d| t j } g }| j d8 D ] }|j }|st j d9d| }t j d:d| }|j; d;d j; d<d= }t j d>d=| }t j d?d| }t= j> | }|j; d@d= j; dAd= }t j dBd| }|st j dC| r'jA D ] \ }} |j; || } tC ( D ] \ } } |j; dD| z | } t j dE|t j s*t j dF|t j rdG|z }|j% |j d8j- | } 'jA D ] \ }} | j; || } tC ( D ] \ } } | j; dD| z | } t j dHd| } t= j> | } t j dId6| } t j dJd8| } t j d>d=| } tE g }!} | j d8 D ]W }
t j dKdt j d?d|
}
|
r|
| v r6| jG |
|!j% |
Y d8j- |! j } g tE }#}"&D ]. \ }$}%|$|#v r |#jG |$ |"j% |$|%f 0 | ||"fS # t0 $ r Y w xY w# t0 $ r Y w xY w)LuR 正文 HTML 清洗：保留 <p>/<table>，附件转 <p><a>绝对URL</a></p> 段落r r z
<!--.*?-->)flagsz <meta[^>]*>z<script[^>]*>.*?</script>z<style[^>]*>.*?</style>u 【字号：[^】]*】 html.parseraThrefrF appendixz
data-appendixz&\.(pdf|docx?|xlsx?|zip|rar|wps|et|ofd)z/module/download/oldsrcOLDSRCz ^(?:javascript:|mailto:|tel:|#)r _titletitle? 附件z__ATTACH__%d__ <a href="" target="_blank"></a>imgsrcz (icons?/|fileTypeImages/icon_)paltu 图片z __IMG__%d__z<p>z<br/>z</p>z <p><a href="z</a></p>c | j d }t j d|t j r2|j j d rdS | j d S | j d }t j dd| }t j | j }t j dd| j }|s5t j j |j d d xs d
}t j j! | }j# ||f d t% z }d |d
|d|< |S )Nr z&^(?:javascript:|mailto:|tel:|s?ftp:|#)z javascript:r r r$ r u ^附件[:：]\s*rM rN z __LINK__%d__rO rP rQ )groupr matchIlower
startswithr r r r ospathbasenamespliturllibparseurljoinappendlen) mrF innertxtabs_urlkey attachments link_protectpage_urls r
_link_replz html_to_text.<locals>._link_repl s wwqz
88=tRTT J 00 ?2 OQWWQZ O
ffZU+ $**,ff("c288:''""4::c?1#56B(C,,&&x6GS>* s< 00ELc R S
r z&<a\s[^>]*href="([^"]+)"[^>]*>(.*?)</a>) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingNdivtablec f j | j d dt dz
z S )Nr __TBL__%d__r )rc rW rd )re
table_protects r _tbl_replz html_to_text.<locals>._tbl_repl s. QWWQZ( M 2Q 677r z <table[^>]*>r z <table(?:(?!<table).)*?</table>z</p>
z <br\s*/?>
z8</?(?:span|font|o:p|st1?:[a-z]+|strong|em|b|i|u)\b[^>]*>z
z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)
r z [ \t]{2,}r z z [\s\u200b\u200c\u200d\ufeff]+u ^[【\[（(]?\s*(?:返回顶部|回到顶部|返回列表|打印(?:本页)?|关闭(?:窗口)?|分享|扫一扫|上一篇|下一篇)\s*[】\]）)]?$r| <p[ >]<a[ >]z <p>%s</p>z __(?:LINK|TBL|IMG|ATTACH)__\d+__z[ \t]+\nz\n{3,}r )$r r SrY r find_allr0 searchr rX get_textr\ r] r^ r_ r` ra rb rc rd replace_with decomposejoinr6 r5 attrspopfindunwrapr r r items enumeratesetadd))contentrl _souprD rF _href_oldsrc real_hrefrg rh ri rR rS rT imgssrcsrm tagattrdr~ has_tableprevpartsblockbplainkvitbloutseenfinaldedupseen_uur rj rk r} s) ` @@@r html_to_textr Z s 1byff]Brtt<Gff^R=Gff12wbddRTTkRGff/WBDD244KPGKLff.G<G5 !'=9E^^Cd^3
(yEE*%)? 99%NPTVXVZVZ[.$6 !ZR . . 0E uuX G!%%/ GR N N PG RXX.PRWY[Y]Y]%^$) $+$4u ::D:1 TQUU8_ Tg T gg . .ys/CA/F G TKS $ll 2 28Y GG &&~6*S ->>CQXZ](^L%NN3'%
(& ~~e,
$wwu~@):@b99 >RTTJMMO
$ ^^C(
,zz%(
6 D # \ !ggen G0A GR " $"(,,"6"6x"E "wwu~ G1A GR N N P O !ww / / #q0A B OFN WVY$Z[\ +c,.? ?,1GLL4F,F,O S)s+
, ~~e,
&ggen?(9?R ,,..x=wwu~?)9?RFFH Gww'' #q(9:G>F #c,&77T[]`$a S! %
& %jG ff>
BDD244K1G !'=9E ~~d+
. O .DIIMM$-.
. ^^E*
zzz-affWo !u
affSkKKMHHJ
%jG M8 YY>AI D
'/&&;Y rtt 5 '/
ffWj'>Gff\4=GffPRT BDD*G
E v& KKM FF+R 3 FF*B 2
IIdB ' 'c 2 FF<a (z2q)!!%(
fc*228SA7UC
88 k mr s &&( DAq !QA
. 2FAs -!+S1A 2yyArtt,9a1NaA
QWWY / 2 ++e
C ""$ 1kk!Q M* 23kk-!+S12
&&4b#
>C
C
C
&&dC
(C
&&FC
(C
&& sC
(C%%D
YYv
ffVR
B!:;cTk
Q ++e
"
"
$C6E 1 ;
1
aV 5 ] F sG -A3k7 !F+k7
:k7 k7 Ck7 8E k7 >C l 7 ll llc ` g }d}t s||fS t | d }|j dt j d D ]T }|j dd }|s|j d xs d
j
}t |j d xs d
}|s t |j d d
}|rt | dk rd|vrt j j || } d
}
|j d } | r}t j d| j d
} | rU| j d d| j d j d d| j d j d }
|j! | ||
f W |j dt j d }
|
rk|
j dd }|rV|j d rE|d j# j% d s#t j j ||d }||fS )u 返回 (items, next_page_url)NrC liz ^line_u6_idrD TrE rF r rL r rJ r$ z/info/span' (\d{4})[年/-](\d{1,2})[月/-](\d{1,2})r -r p_nextclass_
javascript)r r r compiler r0 r r r rd r` ra rb r rW zfillrc rZ r[ ) html_textrl r next_urlsoupr rD rF rL rh datespdmnxts r
parse_listr s EH h M 2DmmDRZZ %=m> - GGCdG # f
#**,AEE'N 0b1
3d
;<EE
Q 4 ,,&&x6
WWV_
Er{{Y]{G^_B%'XXa["((1+2C2CA2FQR HYHYZ[H\]
gud+,'-* ))F2::i#8)
9C
HHStH $ v qy'8'C'CL'Q ||++Hai@H (?r c B dx}x}}d }t r t | d }|j ddt j dt j i }|r |j d rt
|d }|sR|j d }|r|j d n|j d }|rt
|j d
}|j d d
} | r|t j d| j d }
|
rU|
j d d|
j d j d d|
j d j d }|j d xs |j d } | r"dj d | j D }|syt j d| t j t j z } | rAt
| j d }
t j dd|
j# }
|
}|smt j d| }
|
rU|
j d d|
j d j d d|
j d j d }|||fS # t $ r Y w xY w)Nr rC metaname ArticleTitle)r r zdiv.contentwarph1TrJ ry labelr r r r r r$ r vsb_content_500r zdiv.v_news_contentc 3 2 K | ] }t | y w)N)r6 ).0cs r <genexpr>z parse_detail.<locals>.<genexpr>G s &C!s1v&Cs z<title>(.*?)u! [-|_].*(?:武定|人民政府).*$)r r r r rY r0 r
select_oner r rW r r contentsr5 r r r )r rl rL r content_htmlr mtcwr labr elre r s r parse_detailr 2 s& "$$E$D< D M:D6&"**_bdd2S)TUBbffY'#ByM2__%67&(RWWT]diio ' $ (? @E))E')2CYY I3< |r-tA jB |D cg c]
\ }}||d c}}d ndd } tD r|jG | n|jG | |dz
}t d|d d |||rdndfz d tI jJ d tD s|r ||t g }|
}tI jJ d f tD rltM tD d d 5 } |D ]+ } | jO tA jB | d d!z - d d d t d"tD t+ | fz d n|r ||t t d#|||fz d t d$t z d y # t& $ r } t d| | fz d Y d } ~ d } ~ ww xY wc c}}w # 1 sw Y xY w)%Nr )push_to_searchdbz"=== %s (%s) pages=%d cutoff=%s ===Tr& r zPage %d: no next link, stopzPage %d fetch error: %szPage %d: empty, stopzPage %d: no items, stopz Page %d: found %d items (%s)ir r r r r z skip empty/placeholder: %s( )rL r: F) ensure_ascii) site_name
source_urlr: rL pub_datesummaryr categorytags
group_namerj u + %s (%s, %d字%s)& u , 表格g?) batch_labelg333333?wr )r3 r zJSONL written: %s (%d rows)u* 新增: %d | 跳过: %d | CUTOFF截断: %dz=== %s done ===)(r+ sysr] insertr\ dirnameabspath__file__ crawler_libr r9 SITE_NAME SCRIPT_NAME_PAGESCUTOFFr LIST_URLr* r@ r5 r rd r r r r r r r rY CATEGORY
GROUP_NAMEjsondumpsr rc r7 r8 openwrite) r+ r new_count
skip_count cutoff_count seen_urls
jsonl_rowsbatchr: pager r? r r rh rL r dhtmldtitleddater r r atts plain_lenno_parar r itemfrows r mainr W s HHOOArwwrwwx'@AB,
.)[&RX1Y
Y -.-I-
\IJ E
Ca!$ > /$6d C c
I (4/t < $Y4x +d2$ ?
,c%j#cd)/LLTXY$) ' GUD)# MM' "v
! 'NEa
*6ug*F 'FE< '3L''J $GYBFF:r7 ; A A CDI 99Y>> U !yyGRTT BBUKT}
1}4vcr{B$Oa
&gg Ur"" (VZ $zz8<=1q+=E S`b
D !!$' T" NI (F3BK 9B:,L LSW
Y JJsOO' P e U <E
3}>@
*cG
4 D!
D
3U;dBC
D D +z3z?.KKSWX U <
6)ZQ]9^
^fjk
i
't4I +tQi7t D T >" D Ds* O 1O4"91O: O1O,,O1:P__main__) )/__doc__r r\ r r7 r htmlr urllib.parser` r r bs4r ImportErrorenvironr0 r r2 r r r r BASE_URLr nowstrftimer r)
_MAX_PAGESr argvr rD r[ intr_
ValueErrorrd r r r@ r r r r __name__ r r r s ? > > > (! ZZ^^L"
-
0
% H H
(,,.9'2
2 <