jj d Z ddlZddlZddlZddlZddlmZ ddlmZ dZdZ dZ
dZ d d
iZ h d Z
d Zd
Zd Zd Zd Zedk( r e yy)u
长武县人民政府 — 重大建设项目
https://www.changwu.gov.cn/zfxxgk/fdzdgk/zdjsxm2/
CMS: TRS/WCM
分页: index.html (0), index_1.html... (虚标31页)
详情: ./YYYYMM/tYYYYMMDD_XXXXXX.html
正文: div.article-content.article-content-body
N)datetime)
BeautifulSoupz/root/search.dbu 长武县-重大建设项目z;https://www.changwu.gov.cn/zfxxgk/fdzdgk/zdjsxm2/index.htmlz1https://www.changwu.gov.cn/zfxxgk/fdzdgk/zdjsxm2/z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36>) pdddldth1h2h3h4h5h6hrlioltdthtruldivnavpreformmainasidetabletbodytfoottheadcenterdialogfigurefooterheaderhgroupaddressarticlecaptiondetailssectionfieldset
blockquote
figcaptionc | yddl }ddlm g fd | dj }|j dd| }|j dd | }|j S )
ul 块级边界出换行、行内节点直接拼接、
出换行（≈ 浏览器看到的换行结构）。N r )NavigableStringc F | j D ] }t | rj t | *t |dd dk( rj d Lt |dd t
v r+j d | j d | y )Nnamebr
)children
isinstanceappendstrgetattr _BLOCK_TAGS)nodechr0 outwalks "/root/gov_crawler/crawl_changwu.pyr> zbody_text..walk, s --
B"o.
3r7#VT*d2
4 VT*k9
4 R
4 R
z[ \t\r\f\v]*\n[ \t\r\f\v]*r4 \n{3,}
)rebs4r0 joinsubstrip)el_retr0 r= r> s @@@r? body_textrK $ sb z#
C H
A -tQ7A 61%A 779r@ c | dk( rt }n t d| d}t j |t d }d|_ |j S )Nr index_z.html headerstimeoututf-8)LIST_URLBASE_URLrequestsgetHEADERSencodingtext)pageurlrs r?
fetch_pager] ? sB qy
&e, S'26AAJ 66Mr@ c t | d }g }|j d D ] }|j dd }|s|d }|j d }|rt | dk rAd |vrF|j d
r|}n>|j d r
t |d d z }n |j d
rd|z }n t |z }d}|j d } | r| j d }|j |||d |S )N html.parserr aT)hrefra rG t202httpz./ /zhttps://www.changwu.gov.cnr/ span)titler[ date)r find_allfindget_textlen
startswithrT r7 )
htmlsoupitemsr r` ra ri full_urldate_strrh s
r?
extract_itemsru H s } -D EmmD! J GGCdG # y
&E
Q ??6 "H
__T
" $qr(*H
__S
!3d:H $Hwwv }}4}0H
uXxHI3J4 Lr@ c t j | t d }d|_ t |j
d } |g d D ] }|j
d}|j dd }|r7t j d
|j }|r|j d }|j dd }|s|j dd
}|r|j d D ] }|j g } t }
|j ddg D ]> } | j d } | s| |
vs | j | |
j! | @ | rdj# | }
nt% | }
n t% | }
t j& dd|
}
t j& dd|
}
|
j) |fS # t* $ r}t- d| d Y d }~yd }~ww xY w)NrN rO rR r_ )scriptstyler r# r$ r r/ r c . | xr d| j v S )Nattrlowercs r? z fetch_detail..p s 1Jv7J r@ )class_z(\d{4}-\d{1,2}-\d{1,2}) c V | xr& d| j v xr d| j v S )Narticle-contentbodyr{ r} s r? r z fetch_detail..v s0 8t>OSTSZSZS\>\8tagklkrkrktat r@ c . | xr d| j v S )Nr r{ r} s r? r z fetch_detail..x s A<`BSWXW^W^W`B` r@ rh r r Trb rB rA z {2,} z [ERROR] detail: flush)r/ r/ )rU rV rW rX r rY decomposerl rC searchrm grouprk unwrapsetr7 addrE rK rF rG Exceptionprint)r[ r\ rq tag detail_daterz m content_divrh
para_parts
seen_textsrH rY contentes r? fetch_detailr g s * LLgr :
QVV]3OP C MMO yy'JyK 4dmmoFA ggaj ii.tiu ))E2`)aK #,,V4
J J!**C;7
){{{.D
2 %%d+ NN4(
)
++j1 $K0 oG&&FG4&&30}} ++
"1#&d3s% D/G 2G 7BG G)G$$G)c
t j t } | j }t }|j dt f D ] }|j |d d}|j dt f j }|r
|d r|d }| j t d| d t dt | d g }d}t d D ]e }t | }t | } t d
|d z d t | d
d t | dk( r |d z
}|dk\ r nd}|j | g t dt | d t j t } | j }dx}
x} } |D ]. }
|
d }|
d }|
d }||v r| d z
} t! | \ }}|s|}|rt | dk r(t d|d d dt | dd |s| d z
} g|s#t# j$ j' d }t) j* dd| d d }|j- d dk( rt/ |j1 d d nd}|j d t t2 |||||d d |f |
d z
}
|
d z dk( s | j5 t d |
d!| d"d 1 | j5 | j t d#d$ d t d%t d t dt | d t d&|
d t d'| d t d(| d t d$ d y ))Nz.SELECT page_url FROM gov_raw WHERE site_name=?r z
1900-01-01z7SELECT MAX(publish_date) FROM gov_raw WHERE site_name=?zLatest DB date: Tr zExisting URLs: z Page r z: z itemsrf zTotal list items: r[ ri rj 2 z
[SHORT] rN z... ()z%Y-%m-%dz[^\d-]r/
-z
INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, date_rank)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
i z Progress: z new / z skipr4 z2==================================================zSite: zNew: zSkipped (existing): zErrors: )sqlite3connectDB_PATHcursorr execute SITE_NAMEr fetchonecloser rn ranger] ru extendr r nowstrftimerC rF countintreplacerS commit)connr~ existingrowlatest_db_date all_items empty_pagesrZ rp rr new_count
skip_count error_countitemr[ ri rt r r drs r? r r sc ??7 #D
AuHyyII<X SV !N
))MPY|
\
e
e
gC
s1v QJJL ^ ,
-T: OCM?
+48IKb
$ d#
Qxr#e* V4DA u:? 1 Ka K
s9~ .
/t< ??7 #D
A+,,I,
[ R5kW
< (? !OJ +C 0 "K#g,+ JuSbzl%G ~Q?t Lq ",,.11*=KffYK8"= 2=2C2CC2HA2MS$$S"-
.ST 3{GWTc]TV
W Y Q r>Q KKM L 7:,eDD Q; R> KKMJJL Bvh-t$ F9+
d+ s9~ .
/t< E)
T*
-T: H[M
"$/ VHT"r@ __main__)__doc__rU r rC osr rD r r r rS rT rW r: rK r] ru r r __name__ r@ r? r sy
* H > D * 6 >+ZI#V zF r@