jg/ X d Z ddlZddlZddlZddlZddlZddlZddlZddlm Z m
Z
ddl m Z dZ
dZedz ZdZeZd d
edz d d d
dddedz dddddZej& j) dd ddlmZ eddfdZeddfdZd Zd d Zd Z d Z e d k( r e yy# e$ r dZY 2w xY w)!u 临川区人民政府 (www.jxlc.gov.cn) 多栏目爬虫 — 大汉 jcms dataproxy
栏目:
tzgg col1384 通知公告 (webid=5, columnid=1384, unitid=70893, 记录格式 href='..'title='..'+span)
jjkfq col1611 经济开发区 (webid=5, columnid=1611, unitid=46632, 记录格式 href=".."title=".."+b, 页面内嵌61条)
用法: python3 crawl_jxlc_tzgg.py [--col tzgg|jjkfq] [--pages N] [--db /path] [--host IP]
加速乐防护: 需 --resolve 强制 IP + Referer + X-Requested-With, 全部走 POST
N)Requesturlopen)urljoinzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zhttps://www.jxlc.gov.cnz/col/col1384/index.htmlz
218.98.26.222138470893 jxlc_tzgg single_quote )columnidunitidlist_url site_name
record_fmtpages161146632z+/col/col1611/index.html?number=D00002D00004
jxlc_jjkfq double_quote )tzggjjkfqz/root/gov_crawler)push_to_searchdb c 8 ddl }|j }d|_ |j |_ t
|dd}|'d|d< d|d < t
| |j |
}n
t
| | }t ||| }|j j d
d S )u& 强制IP直连 + Referer, POST 或 GETr NF*/*)
User-AgentRefererAcceptXMLHttpRequestX-Requested-With!application/x-www-form-urlencoded Content-Typedataheadersr% contexttimeoututf-8ignoreerrors) sslcreate_default_contextcheck_hostname CERT_NONE verify_modeUAr encoder readdecode) urlrefererpostr) r. ctxr% reqopeners crawl_jxlc_tzgg.pyfetchr> + s
$
$
&C CmmCOG
&6"#"E c
w?c7+
S#w
7F ;;= 99 c ddl }ddl}ddlm} |j }d|_ |j |_ || }t ||j dd} |'d| d< d | d
< t | |j | }
n
t | | }
ddl } | j j t d
|| } |j |j" rd|j" z ndz }
|dnd}||j nd}| j% ||
|| | j' }|j) j+ dd }| j- |S )u- 直接连 IP + Host header (--resolve 等价)r N)urlparseFr )r r Hostr r r r! r" r# r& i r' ? POSTGET)bodyr% r* r+ r, )r. socket urllib.parserA r/ r0 r1 r2 r3 netlocr r4 http.clientclientHTTPSConnectionHOSTpathqueryrequest getresponser5 r6 close)r7 r8 r9 r) r. rH rA r: pr% r; httpconnrO methodrG respr$ s r= fetch_iprY @ s- %
$
$
&C CmmCO
A G &6"#"E c
w?c7+ ;; & &tS#w & OD 66aggS177]2 6D'VUF ,4;;=$DLLD'L: D 99; gh 7DJJL Kr? c ddl m} t j dd| } t j dd| } t j dd| } t j dd| } || d } |g d D ] }|j |j d
D ] }|j
d xs |j
d xs d}|rO|j d
r|nt t | }|j d| }d|_ |j | ||j |j g d D ] }|j |j d D ]s }|j
dd }|r8|j d s'|j d
r|nt t | |d< O|sR|j d sd|j u |j d D ]I } | j d r| j d r(| j d
r:| j K t! | S )uQ 白名单清洗: 保留 p/table/a, 去 inline style, unwrap span, 链接绝对化r
BeautifulSoupzrD zz%z"? html.parser)scriptstyleiframeobjectimgsrczdata-srcrU a)hrefu 查看图片) divspanfontcenterbstrongemiuslabelre
javascriptrT Tstriptable)bs4r\ resub decomposefind_allget
startswithr BASEnew_tagstring replace_withunwrapget_textfindstr)
htmlr\ souptrb rc abs_srcrd re rT s
r=
clean_htmlr ` s !
66/T :D
66-r4 8D
66:B ED
667T BD } -D
9
:
}}U# ggen9
39r ^^F 3cs9KG Sw /A%AH Q MMO ]]i
j
]]3
uuVR 5 $ 7WT4=PAfI
dool3
KKM ]]3
zzz%affWoaffUm
KKM t9r? c g }|dk( rt j d| t j }|D ] }t j d| }|rd|j d vr/t j d| }|r |j d j nd}|j d }|j
d st t | }|j |j d j ||d
|S t j d| t j }|D ] }t j d | }|rd|j d vr/t j d | } | r | j d j nd}|j d }|j
d st t | }|j |j d j ||d
|S )
u 解析 dataproxy 响应 → [(title, url, date)]
record_fmt: single_quote=col1384格式(href='..'title='..'+span), double_quote=col1611格式(href=".."title=".."+b)
r z&z href='([^']+)'title='([^']*)'z/art/r z]*>([^<]*)rD rU r
)titler7 datez"href="([^"]+)"[^>]*title="([^"]*)"z\s*([^<]+?)\s*)
rv findallSsearchgrouprs r{ r r| append)
r r itemsrecsrecmspr r7 rj s
r=
parse_listr s
E^#zzCT244P RC :C@Aqwwqz17=B*,288A;$$&"D''!*C>>&) dC( LL1771:#3#3#5c4P Q R. L zzCT244P RC ?EAqwwqz1 2C8A)*1771:##%D''!*C>>&) dC( LL1771:#3#3#5c4P Q R Lr? c t ddl m} || d }|j d }|r|j d nd}|slt j
d| t j }|rE|j d j }|j d
d j d d }t j
d
| }|r"|j d j dd nd}|j d }|r>|j dd D ] } | j t t | }
nd}
|||
fS )u4 解析详情页 → (title, publish_date, body_html)r r[ r] h1Trr rD z(.*?)r u 通知公告 u 临川区人民政府z&z parse_detail.. s 8Pmq>P r? )r_ )ru r\ r r rv r r r rs splitry rx r r ) r r\ r r r r r pubr rf rG s r= parse_detailr s ! } -D
4 B')BKKdK #rE II-tRTT :
"AGG ,-b1778PQRTUE
;TBA%&!''!*
Sb
!BC 999 D ==.P=Q C MMO #d)$ #t r? c
t j } | j ddt t j | j dt d | j dd | j dt | j d d | j }|j t j d
< t |j }|j xs |d }|d }|d
}|d }d|d d|d d}g }t } |dk( rg t | }
t% |
| } | D ]/ }
|
d | vs | j' |
d |j) |
1 t! dt+ | d d nt- d |d z D ] }|j/ | } t t0 d | }
t% |
| } | D
cg c] }
|
d | vs |
}}
|D ]' }
| j' |
d |j) |
) t! d!| d#t+ | d$t+ | d t+ | d%k( r n+t3 j4 t7 j8 d&d' t! d(t+ | d d d%d l }t= j> d)d* }|jA |d+, }t d- |jC d.|f D }|jE t+ | }|D
cg c] }
|
d |vs |
}}
t! d/| d0t+ | d1t+ | d2d g }d%}tG | D ] \ }}
t |
d |5 }
tI |
\ }}}|r|s(|d z
}t! d6| d7|d d8 d9t+ | d Sd:D ]# } |jK | s|t+ | d } n |j) ||
d |
d |xs |
d; |xs |
d; |d|dd< |d z d8z d%k( r.t! d=|d z d>t+ | d?t+ | d@| d t3 j4 t7 j8 dAdB t! dCt+ | d@| d |jL rltO |jL dDdEF 5 } |D ]+ } | jQ tS jT | dGH dIz - d d d t! dJ|jL d tV r tW | } t! dK| d y t! dLd y # t $ r)} t! dt# | d d d d}
Y d } ~ d } ~ ww xY w# t $ r+} t! d!| d"t# | d d d Y d } ~ d } ~ ww xY wc c}
w c c}
w # t $ r'} t! d3t# | d d4 d Y d } ~ d } ~ ww xY w# t $ r/} |d z
}t! d6| d"t# | d d4 d Y d } ~ d } ~ ww xY w# 1 sw Y 8xY w)MNz--colr )defaultchoicesz--pages)typer z--db)r z--hostz--outrN r r
r r z*page={pg}&appid=1&webid=5&path=/&columnid=r z&unitid=r z&permissiontype=0r u 列表页: 失败 d T)flushrD r7 u 页面内嵌: u 条r )pgz /module/web/jpage/dataproxy.jsp)r9 u 页u : 失败 z: u
条, 累计 r g?g ?u 共 SEARCH_DBz/root/search.db< )r) c 3 & K | ] }|d yw)r Nr ).0rs r= zmain.. s MQAaD Ms z.SELECT page_url FROM gov_raw WHERE site_name=?u 预去重: u → u (库中已有 )u 预去重跳过: P )r8 u 详情u : 空 title= z body=)u, 临川区人民政府 临川经济开发区 u) 临川区人民政府 信息公开制度 u 临川区人民政府 r ) r r7
source_urlpub_date publish_datecontentauthorr attachmentsu 详情 /z: ok=z fail=g ?g ?u 完成: ok=wr* )encodingF) ensure_ascii
u JSONL 已写: u DB 导入: u0 警告: push_to_searchdb 不可用, 只写 JSONL),argparseArgumentParser add_argumentlistCOLSkeysint DEFAULT_HOST
parse_argshostrY __globals__colr set Exceptionprintr r addr lenrangeformatr| timesleeprandomuniformsqlite3osgetenvconnectexecuterS enumerater r{ outopenwritejsondumpsr ) apargsr r r
r r post_params all_itemsseenr er itr r9 new_sq_db_c_have_beforerowsfailrm r r rG prefr oks r= mainr s "BOOGVT$))+5FOGOOICO6OOFDO)OOHlO3OOGTO*
==?D#'99H
txx.C JJ &#g,E:HK I\"J@Z @QQYZ]^fZgYhhyzK I 5D^# H%D 4, %B%y$E# $ % s5zl$/t<519% 1B %%%,D
4&(G HtT tZ0E %?"E$)>2?C?
%E# $
%
Ct2c%j\ s9~6FGt T5zQ JJv~~c3/ 0 1
DY
%T2 = ii %67
[[b[
) M"** D {KSW X K C $ c#hi(
e9 U))r&z -2f:"
ER<1 GAaC5#i.!1s4ykvNVZ [
6>>#s+,= -@
KD {&
/t< xx
$((C'
2 Ba
B
159D@A
B B txxj)6
d
#
B4 -
@Mo &s1vds|n5T BD
B4yQ 7tD
@& G =
!#a&"+ /t<<= AID HQCyQ
6d C > B Bs T 'U
VVA/V 7 VV+V
V>1W9 U UU
U> U99U>V V; V66V;> W6$W11W69X__main__)r ) __doc__r r r rv sysr r urllib.requestr r rI r r3 r| COL_URLr rN r rO insert crawler_libr ImportErrorr> rY r r r r __name__r r? r= r s 1 0 0 0 + v
*
* g44 $
gHH!$ " & ' , T2 :* "b @ B B 6nN` zF K s 0B B)(B)