+|jx- d Z ddlZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl Z ddl m
Z
mZ dZedz ZdZdZdZej( j+ d d
Zej( j+ d d Zd
ZddddZdZdadZ e ej< D ]k \ Z Z e jC d r e"e jG dd d Z 4e dk( s:e dz e%ej< k sS e"ej< e dz Z m e dk\ re ndZ&d#dZ'd Z(d Z)d Z*d Z+d Z,d Z-d Z.d Z/d! Z0e1d"k( r e0 yy# e$$ r Y w xY w# e$$ r Y w xY w)$u
crawl_jiangjin_bmjz.py - 江津区-部门街镇
http://www.jiangjin.gov.cn/zwxx_180/bmjz/
列表: div.list > a[href=./YYYYMM/t...html] > span(标题) + span(日期)
分页: index_N.html (24 页 ~480 条, 每页 20 条; pageCount 写死25但P25=404)
详情: /zwxx_180/bmjz/YYYYMM/t...html
→ div.title(标题) + div.info(发布日期) + div.trs_editor_view.TRS_UEDITOR(正文)
附件: ⚠️ JS变量 hasFJ 内 document.write 动态生成, 正则提取
N)
BeautifulSoupCommentzhttp://www.jiangjin.gov.cnz/zwxx_180/bmjz/u 江津区-部门街镇u 重庆zcrawl_jiangjin_bmjz.pyDB_PATHz/root/search.db
JSONL_PATH utf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-Languageg333333?g z--pages==z--pagesc t j t z
}|t k r t j t |z
t j at | D ]V } t j | t |d }|j dk( r d|_ |c S t j d|dz z X y # t $ r Y )w xY w)NF)headerstimeoutverify r ) time _last_reqREQUEST_INTERVALsleeprangerequestsgetHEADERS status_codeencoding Exception)urlretriesr dtirs (/root/gov_crawler/crawl_jiangjin_bmjz.pyhttp_getr# : s
y B
#b() I
7^ S'75QA}}# $
$
1q1u9 s $4B:: CCc | syt j | } | j dd j dd j dd j dd } t j dd| } | j S )Nr u u u u \s+ )html_libunescapereplaceresubstrip)ts r" clean_titler. L so ! A (B ''"5==hKSST\^`aA
vsA A 779 c
| dk( rt }n
t d| dz }t | }|g S |j }g }t j d|t j
D ] }|j
d t |j
d |j
d }}}t j j t d| d |j
d
j d d dd } |j
d
}
t j d |
} | r3t j j t | j
d } |j | ||f |S ) u0 抓一页列表, 返回 [(abs_url, title, date)]r
index_z.htmlNzo\s*\s*(.*?)\s*\s*\s*(\d{4}-\d{2}-\d{2})\s*\s*r ./z/tr zhref="(\./[^"]+)")LIST_URLr# textr* finditerSgroupr. urllibparseurljoinsplitsearchappend) pager r! html_textitemsmhreftitledateabs_urlrawhms r"
fetch_listrI U sF qy6$u--
Ay I E
[[ L NW Y[ Y] Y] ^ - GGAJ AGGAJ(?Te,,&&x2dV2aggaj>N>Nt>TUV>WXZYZ>[=\1]^ggaj
YY+S
1
ll**8RXXa[AG
gud+,- Lr/ c t g }t j d| t j }|s|S |j d }|j dd j dd }t j
d|t j D ] }|j d t
|j d }}|s1|j d r%t j j t | }n8|j d
r%t j j t | }n|}|j ||f |S ) u3 从 JS 变量 hasFJ 提取附件 [(abs_url, title)]zvar hasFJ = '(.*?)';r
z\''z\""z ]*>(.*?)r r3 /)r* r= r7 r8 r) r6 r.
startswithr9 r: r; r4 BASE_URLr> )r@ attsrB blockamrC txtrF s r"
extract_hasfjrT m s
D
)9bdd;A
GGAJE MM% % - -eS 9Ekk=ubddK
$HHQKRXXa[!9c ??4 ll**8T:G
__S
! ll**8T:GG WcN#
$ Kr/ c 6 t | }|dddg fS |j }t |d }t j d| }|rt |j
d nd}|s.|j dd }|rt |j }t j d | }|r|j
d nd}|s+t j d
| } | r| j
d nd}|j dd }
|
s<|j d D ]( } | j d xs g } d
| v s d| v sd | v s&| }
n |
s|j dd }
t | }
|
r)|
j d D ] }|j n|j d }
|||
|
fS )u6 详情页: 返回 (title, pubdate, zoom_soup, fj_atts)Nz html.parserz+z parse_detail.. s
1g0F r/ )string)r# r5 r r* r= r. r8 findget_textfind_allr rT extractnew_tag)r r! r@ souptmrD tdivdmpubdatedm2r\ dclsfj_attscs r" parse_detailrr s
AyT4##I M 2D A9 MB(*K $E yywy/
0E G SB bhhqkRG ii@)L"%#))A,2 99U#49 5D u% A%%.&BC #3 :Ks:R
yy6y*I&G &F G A
IIK ||E" '4 ((r/ c | ddg fS | j g d D ] }|j t | j d }g }| j dd D ] }|d j }|j d r(t j j t | |d< n8|j d
r't j j t | |d< |j j d st |j }|s|j |d |f | j d d
D ] }|d j } | j d r(t j j t | |d< O| j d
sat j j t | |d< t! | }
|
||fS )Nr F)scriptstyleiframeobjecttableaT)rC rC r3 rM )z.pdfz.docz.docxz.xlsz.xlsxz.zipz.rarimg)srcr{ )re decomposeboolrc r, rN r9 r: r; r4 rO lowerendswithr. rd r> str) zoom_souprF tag has_table attachmentsry hrS rz r` contents r" html_to_textr s 5"} !!"IJ
Y^^G,-IK
$
/ 5
fIOO << ,,Xq9AfI
\\#
,,Xq9AfI 779 X Y ajjl+C""AfIs#34 5 !!%T!2 ; J << --h:CJ
\\#
--h:CJ ; )nG I{ **r/ c |s| S | j r| gng }|D ] \ }}|j d| d| d dj | S )Nz z
z
)r, r> join)r r partsrF rS s r" append_attachmentsr s[ WIbE# N
|G9,>se8LMN ;;u r/ c | j dt f }|j D ch c] }|d c}S c c}w # t $ r
t cY S w xY w)Nz.SELECT page_url FROM gov_raw WHERE site_name=?r )execute SITE_NAMEfetchallr set)conncurrows r" load_existing_urlsr sN llKi\Z"%,,.13A111 u s )? :? ? AAc
| j d|t f }|j ry d} ddlm} || }| j d|||||t t t ||f
}|j dkD ryt j dd| } t j | } t j d d
| j d d } | j d |j |t | f y
y# t
$ r Y w xY w# t j $ r=}
dt# |
v r*t% j& d t) | ||||| cY d }
~
S d }
~
ww xY w)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?Fotherr )classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)<[^>]+>r r% r& r zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)Tlockedr2 )r r fetchone crawler_libr r
GROUP_NAME SCRIPT_NAMErowcountr* r+ r' r( r, lastrowidsqlite3OperationalErrorr r r store_record) r r rD r rE r r industryr summaryes r" r r sF
,,PSVXaRb
cC
||~ 5(/H ll
+ #ugtY
KQZ\d e
< r r* r+ r, r r r r openwritejsondumpscommitclose)r existing new_count
skip_count
jsonl_rowsr? rA rF rD rE dtitleddater\ fjr r rP all_attsseenuniqur- plain_lenokfr s r" mainr sO D w3 01 ./%d+
3x=/)GHPTUIJJa!$ 2 4
TF'#e* 6dC /t < $) , GUD 'X"5a
&27&; #FE4|uSbzl#gY?tLa
'3D''B $GYbyH5DD
$19 QF#
$
)$7GBFF:r7 ; A A CDI2~&"7vcr{m3wi@Ma
!! '&W$) !**#. #
6#2;-r%:$GQ dGVWeYOBQ W%VCR[ME7!4DAa
Y,
2 h
*cG
4 D!
D
3U;dBC
D D
|2c*o->fETR
K {&
5TB D Ds 51LL__main__)r2 r )2__doc__sysosr* r htmlr' urllib.parser9 r r r bs4r r rO r4 r r r environr r r ENCr r r
_MAX_PAGES enumerateargvr ry rN intr<
ValueErrorr r r# r. rI rT rr r r r r r __name__ r/ r" r s & ( ' ' $
&
**..$5
6
ZZ^^L"
-
@O'
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA$0.#)L+6 DJCZ zF k
s$ ,D8(E8E ?E E
E