ujE d Z ddlZddlZddlZddlZddlZddlmZ ddlm Z ddl
m
Z
m Z ddl m
Z
mZ ej dd Zd Zd
Zd Ze d Zd
ddZdd
iZdZdZd Zd Zd Zd Z d Z d Z d Z d Z e!dk( r e yy) u?
crawl_chengwu.py — 成武县人民政府·环境影响评价公众参与
======================================================
CMS: ELS 服务系统，API 驱动
列表：POST /els-service/article/{pageNo}/{pageSize}
JSON body: {"catas": ["1535192418234798080"]}
分页：10条/页
详情：/10086/{dwid}/{xxid}.html
标题
日期
正文 embedded in
用法:
python3 crawl_chengwu.py # 全量
python3 crawl_chengwu.py 1 # 增量
N)
BeautifulSoup)urljoin)datetime timedelta)ThreadPoolExecutor as_completed SEARCH_DBz/root/search.dbu 成武县人民政府u 环境影响评价公众参与zhttp://www.chengwu.gov.cnz/els-service/articlezoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z application/json;charset=utf-8)
User-Agentz Content-Typer
c t d| y )Nz
[chengwu] )print)msgs "/root/gov_crawler/crawl_chengwu.pylogr 5 s Jse
c t d| dt }ddgi} t j |t |d }|j }|j
di j
dg }g }|D ]m }|j
d d } |j
d
d }
|j
dd j } |j
dd } |j | |
| | f o |S # t $ r }t d| d | g cY d
}~S d
}~ww xY w)zLFetch one list page from API. Returns list of (xxid, dwid, title, date_str)./catas1535192418234798080 )headersjsontimeoutdatacontentsz ERROR list page : Nxxid dwidsubjectfwdate) API_URL PAGE_SIZErequestspostHEADERSr get Exceptionr stripappend)
page_nourlpayloadrespr r eitemscr r titledate_strs
r fetch_list_pager5 9 s
IQwiq
,C./0G}}S'Lyy{88FB'++J;
E
4uuVR uuVR i$**,552&
dD%23 4 L A3 /0 s AC D'C?9D?Dc \ | j d }|dk ry| |d }|j d dz }|}|t | k r|| dk( r|dz
}nb|| dk( rU||dz |d z j }|r3|j d
s"|j d s|j d rn|dz
}n|dz
}|t | k r||| } |j j
d
}|j dd }|j dd }|j dd }|S # t $ r |}Y Ew xY w)z4Extract and decode var memo = '...' from script tag.z
var memo =r r N" \ ;
unicode_escapez\/r z\nz\t ) findindexlenr*
startswithencodedecoder) replace)htmlidxchunk quote_startendrestmemo_raw memo_decodeds r extract_memo_contentrP Q sN
))L
!C
Qw JE++c"Q&K
C
E
: 1HC
3Z3
Qs1u%++-D4??3/4??43HDOO\`La 1HC 1HC E
[%H (//0@A
''s3L ''t4L ''t4L s D D+*D+c r t d| d| d} t j |t d }d|_ |j
}t |d }|j d d
di }|r$|j d r|d j nd}d} |j d d
di }
|
rK|
j d r:t j d|
d j } | r| j d } t | } | st d| d | |fS | | |fS # t $ r}t d| d| Y d }~y
d }~ww xY w)z=Return (content_html, date_str, title) or (None, None, None)./10086/r .htmlr )r r zutf-8z ERROR r NNNNz html.parsermetaname ArticleTitle)attrscontentr PubDatez(\d{4}-\d{2}-\d{2})r8 z WARNING: no content for )BASE_URLr% r( HEADERS_HTMLencodingtextr) r r rA r* rematchgrouprP )
r r r- r/ rH r0 soup
meta_titler3 r4 meta_datem content_htmls
r fetch_detailrg v sI
JgdV1TF%
0C ||C rB
yy
} -D 6&.)ABJ-7JNN9 200 THEN substr(content, 1, 200) ELSE content END
FROM gov_raw WHERE category = ?z
FTS note: ) rt ru rv rx rz r{ r| r) r )rt r~ r0 s r rebuild_ftsr sf
w'
2 N
j s AA A4 A//A4c * t t j dkD xr t j d dk( } t j t d z
j
d }t d| g }d} t | }|s|dk( r t d y nt d| d t | d
|D cg c] \ }}}}|s ||k\ s||||f } }}}}|j | | r t d n/| r't | t | k rt d| d n|dz
}t d
t | |s t d y g }
t dz } t dt | | D ] } || | | z }
t d| dz d| t |
z dt | d t |
}|
D ]S \ }}}}|j ||fd \ }}}|xs |}|xs |}t d| d| d}|
j ||||xs df U t! |
}t d| dt |
|
rAt# |
D cg c]
}|d s |d c} }t d|r|d nd d |r|d nd t% t d y c c}}}}w c c}w ) Nr8 1iG )daysz%Y-%m-%dz3-year cutoff: zNo articles foundzPage r z articleszIncremental: page 1 onlyz': some articles before cutoff, stoppingzTotal articles to process: zNo articles within 3 years r z Details [-r z]...rT rR rS r zSaved: r: z Date range: zN/Az ~ zDone!)rC sysargvr nowr strftimer r5 extendrj rangerr r( r[ r+ r sortedr ) incrementalthree_years_ago all_itemspager1 r r r3 dfilteredrm
batch_size batch_startbatch
detail_mapr4 rf detail_date detail_title final_title
final_dater- savedrdatess r mainr s chh-!#: s(:K ||~ w(??II*UO//*+, I D
%qy'( eD6CJ? k+a- +c%j*@)A3y>BRRV WX'.
+0 O 'D$x6@nndD\Se6f 3L+|&/%K$0J JgdV1TF%8C NNCj,:L"M N O
O
w E'%#g, ()g61!67 lu58%8E"ITY;Z [\MLW lN 7s 9 J
J
J
:
JJ__main__)"__doc__osr_ r r r% bs4r urllib.parser r r concurrent.futuresr r getenvrv ry rz r[ r# r' r\ rj r$ r r5 rP rg rr r r r __name__ r r r s $
( ?
"))K!2
3 # . &
J*
+3 5 3 0"J!)H". ">B zF r