ҪjY. d Z ddlZddlZddlZddlZddlZddlZddlZddlm Z m
Z
ddl m Z m
Z
ddlmZ dZedz ZdZd Zed
d d Zd
ddddddddddddddZed d fd Zd$d Zd Zd! Zd" Zed#k( r e yy)%u 溧阳市人民政府 (www.liyang.gov.cn) 政府信息公开爬虫 — PHPCMS + 创宇盾
用法: python3 crawl_liyang_xxgk.py [--catid 42151,42170] [--pages N] [--out /tmp/liyang_xxgk.jsonl] [--start-page N]
列表: /content/xxgk/index?catid=XXXX&page=N (iframe 内容, 必须 Referer=catid=42149 父页面)
详情: /html/czly/{年}/{栏目}_{MMDD}/{ID}.html, meta ArticleTitle/PubDate/ContentSource + td#czfxcontent
过盾: 完整浏览器头 + Referer (创宇盾)
注意: 创宇盾频率敏感, 限速 1.5-2s + 403 退避重试
N)Requesturlopen) HTTPErrorURLError)urljoinzhttps://www.liyang.gov.cnz0/index.php?m=content&c=index&a=lists&catid=42149 liyang_xxgkzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-Languageu 生态环境u 规章行政规范性文件u 政府信息公开制度u! 政策解读图解及简明问答u 专项规划和区域规划u 财政预决算公开u 行政事业性收费u 原生态环境历史信息u 社会救助公益事业u 扶贫信息乡村振兴u 公共文化服务u 应急管理通知公告u 基层政务公开目录清单u 政府信息公开指南)iy i ik i i* i i i٤ iפ i iS iʤ i iݤ c
t t }||d< t | D ]? } t | | }t || }|j j
dd c S y# t $ r}|j dk( r9d|dz z } t d | d
| d d d
d t j | Y d }~|j dv rY d }~ yt d|j d| d d d t j d Y d }~d }~wt t t f$ rO}t dt | j d| d d d t j d|dz z Y d }~Cd }~ww xY w)u5 带完整头 + Referer 抓取, 403/超时退避重试Referer)headers)timeoututf-8ignore)errorsi u [403] 退避 zs (iN)Tflush)i i z [HTTP ] r z [ERR )dictHEADERSranger r readdecoder codeprinttimesleepr TimeoutErrorOSErrortype__name__)
urlrefererr retriesr attemptreqrespewaits
crawl_liyang_xxgk.pyfetchr0 + sK 7mG GI> ( (#w/C30D99;%%gh%? ? ($ vv}GbL(vSST 1=TJ
4 vv# HQVVHBs34yk2$ ? JJqMM,0 ( GDG , ,-RCD {;4 H JJq7Q; ' ' (s1 :A## E(,AD4D5DE(AE##E(c \ ddl m} |xs t } || d }|j d D ] }|j d xs d}|s%|j d }|r|j dd }|s)|j dd d
i }|r|j d d }|rK|j
d
r|n t || } |j d| }
d|
_ |j |
|j |g d D ] } | j |j d D ]J } dD ] } | | j v s| | = t | j D ] } | j
d s| | = L |j d D ] }
|
j d xs |
j d xs d}|rK|j
d
r|n t || } |j d| }
d|
_ |
j |
x|
j |j g d D ]3 } | j dk( r| j d r$| j! 5 |j d D ]] }
|
j dd }|r4|j
d s#|j
d
r|n t || |
d< K|sN|
j _ |j d D ]I }|j# d r|j d r(|j d r:|j K t% | S ) u_ 白名单清洗: 保留 /
/, unwrap 装饰标签, 清 inline style, 链接绝对化r
BeautifulSoup html.parservideosrcr sourceparamnamer' attrsvaluehttpahrefu 查看视频)scriptstyleiframeobjectembedT)rB classalignvalignborder cellpadding cellspacingwidthheightbgcolorfacecolorsizelangdir setedariatabindexrolezaria-imgzdata-srcu 查看图片)divspanfontcenterbstrongemiuslabelh1h2h3tdrf tabler@
javascriptpstrip)bs4r3 BASEfind_allgetfind
startswithr new_tagstring replace_with decomposer; listr9 find_parentunwrapget_textstr)htmlbase_urlr3 basesoupvr6 ra ri abs_srcr> tattrrW r@ s r/
clean_htmlr D s ! tD } -D
]]7
# eeEl bx AeeE2&wvuo6AeeGR( ^^F 3cs9KG Sw /A%AH
NN1
KKM!" B
C
]]4
6 D qwwdG
M Dw'dG
}}U# ggen9
39r ^^F 3cs9KG Sw /A%AH Q MMO ]] B C 66T>ammG 4
]]3
uuVR 5 $ 7WT4=PAfI
KKM ]]3
zzz%affWoaffSk
KKM t9 c D ddl m} || d }g }|j d D ] }|j dd }|s|d }t j
d | s5|j
d
xs |j d xs d j }|j d
d }|r|j d nd } |s|j |t t | | d t g } }
|D ]/ } | d |
vs |
j | d | j | 1 | S )u3 xxgk 列表: div.xxgkList > p > a[href] + span.timer r2 r4 zdiv.xxgkList pr> Tr? r@ z$/html/czly/\d{4}/\w+_\d{4}/\d+\.htmltitlerj r rY r! class_)r r' dater' )rl r3 selectrp researchro ry rk appendr rm setadd)
r{ r3 r~ itemsri r> r@ r rY r seenoutits
r/ parse_list_htmlr ~ s ! } -D E
[[)
* U
FF3TF " yyy@$G w ?1::D:#9?RFFHvvfVv,,0t}}4}(b LL5t1DdS T U r#D
e9D HHRY JJrN Jr c ddl m} || d fd} |d xs d} |d xs d} |d xs d}j d d
xs= j d d
xs( j d d
xs j d d }d}g } |r't |j dd }
|
D ] } t | dd | j
d xs dj } | s7t j d| sN| j d xs d}
| j d r| n t || }|
|f| vs| j |
|f |
D ] } t | dd | j " t t! | | }| r.dj# d | D }|dz |z j% }||r|dd nd||| D
cg c]
\ }
}|
|d c}}
d S c c}}
w ) u- 详情: meta 字段 + td#czfxcontent + 附件r r2 r4 c | j dd| i }|r#|j d xs dj S dS )Nmetar9 r: contentr )rp ro rk )r9 mr~ s r/ r z parse_detail_html..meta s= IIfVTNI 334i &B--/<" Tr? r; Nr@ zB\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps|et|jpg|jpeg|png|gif|mp4)$rj u 附件r= c 3 4 K | ] \ }}d | d| d yw)z z
N ).0r fus r/ z$parse_detail_html.. s# TBl2$b8 <Ts
)fileNamefileUrl)r publish_dater r7 attachments)rl r3 rp rv rn getattrro lowerr r ry rq r r ru r rz joinrk )r{ r' r3 r r pubr7 zoombody att_links_anchorsr> hr r att_pr~ s @r/ parse_detail_htmlr s ! } -D=
&BE
y/
RC
/
"
(bF 99Tm9 , T %M 0R T 99Uy9 ) T-1YYu]Y-S
DI
c
56
.Aq'4(0v $"++-Ayy^`abJJTJ*6h,,v.QGCOr7)+ $$aW-
. Aq'4(4
#d)S) GGT)TTE4K%'..0D$'CRRDMN51bQ26N
Os
G c
X
t j } | j ddd | j dt dd | j d d
| j d t d
| j }|j
j
d D cg c] }|j st | }}dddddd}t |j dd }t }t j j |j rDt |j d D ]* } |j t j | d , g }|D ]_ } t$ j' | t) | }
d} | dkD rt* d| d| n
t* d| } t- | }
|
st/ d|
d| dd nd |
vr t/ d|
d t1 |
d d nt3 |
}|j5 | t7 j8 d |
}|rt; d! |D nd}| |k rM|j< dk( s| |j< k r/| dz
} t? j@ tC jD d"d# t/ d|
d$t1 | d%| d&d t/ d|
d't1 | d(d b t1 | |d)< t/ d*t1 | d+t1 | d&d |D cg c] }|d, |vs | }}t/ d-t1 | d tG |d D ] \ }}t- |d, }
|
sR|d.xx dz
cc< t/ d/| d0t1 | d1|d, d |d. |jH k\ rt/ d2d nGitK |
|d, }|d3 s|d3 |d3< |d4 s|d5 |d4< |d6 }t1 | d7k r
|d8xx dz
cc< |d3 |d4 ||d, |d, tL t7 jN d9d:| j d d; d|d< |d< |d= d> }|jQ t jR |d?@ dAz |jU |dBxx dz
cc< |dCz dk( s|t1 | k( r(t/ d/| d0t1 | dD|dB dE|d. d t? j@ tC jD d#dF |jW t/ dGt jR |d?@ d y c c}w # t" $ r Y w xY wc c}w )HNz--catidzS43641,42151,40811,42170,42282,43445,42157,42201,42199,42238,42323,42186,42269,42205u 子栏目 catid 逗号分隔)defaulthelpz--pagesr u 0=全量)r% r r z--outz/tmp/liyang_xxgk.jsonl)r z--max-detail-fail )r% r ,)r detail_ok detail_failemptyskippedr> r )encodingpage_url Tz/content/xxgk/index?catid=z&page=[u ] 页u 抓取失败, 停止r xxgkListu ] 无xxgkList (len=u ), 停止z
page=(\d+)c 3 2 K | ] }t | y wN)int)r xs r/ r zmain.. s -!s1v-s g?g ?r u 条 (页r u ] 累计 u 条r u 列表完成: u 条 (已抓过 r' u 待抓详情: r u 详情 /u : 抓取失败 u 失败过多, 中止r r r r r r z<[^>]+> r7 r ) r r r r
source_url site_namesummary date_rankauthorcontent_sourcer F) ensure_asciir r r z: ok=z fail=g@u 完成: ),argparseArgumentParser add_argumentr
parse_argscatidsplitrk openr r ospathexistsr jsonloads ExceptionCATSro rz rm r0 r lenr extendr findallmaxpagesr! r" randomuniform enumeratemax_detail_failr SITE_NAMEsubwritedumpsr close)apargsr catidsstatsout_fp done_urlsline all_itemsr r9 page_nor' r{ r r maxpr new_itemsidxdr recs r/ mainr s, "BOOI'|7 9OOICODOOG%=O>OO'c2O>
==?D "jj . .s 3
Aqwwyc!f
AF
AaARS TE
$((C'
2FI ww~~dhh G4 D
djj .z :; I Cxxs5z*OVYZ{TF4UG6'Kcgbh iC DI CJ aKC:D$uWI-BC4P%$ 23t9+YGtT#D)E U #JJ}d3E163-u--AD~4::?g
6J1
6>>#s34 AdV2c%j\7)1=T J
$yY 05TB1C2 ^E'N N3y>**:3y>:J!
LTXY (F2e9I+EFIF N3y>*
+48 Y* #-RRY - A % GC5#i.!1 E LTX Y]#t';';;,D9 dBuI .zGAgJ "6
An I, w<" 'Na NwZ n - 5 U)"vvj#w7==?Ek k ]+
TZZ%84?@
ka 8q=C3y> 1 GC5#i.!1u[7I6J&QVWdQePfgos t
6>>#s+,G#-H LLN HTZZE:;
r s 1 0 0 0 + , "
F
F
vO' ( % . ( " " ( % % % + % $ "2q 27t2*ZZJz zF r