ujB d Z ddlZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl m Z dZdZdZdZdZd Zej( j+ d
d Zej( j+ d d
ZdZdZd
ZdZdZ ddddZ dZ e ej@ D ]k \ Z!Z"e"jG d r e$e"jK dd d Z 4e"dk( s:e!dz e'ej@ k sS e$ej@ e!dz Z m e dk\ re ndZ(d Z)d Z*d Z+d Z,d Z-d Z.d Z/d! Z0e1d"k( r e0 yy# e
$ r dZ Y w xY w# e&$ r Y w xY w# e&$ r Y w xY w)#u
crawl_tieling_hjbh.py - 铁岭市-环境保护
https://www.tieling.gov.cn/tieling/zwgk/zfxxgk/fdzdgknr/zdmsxx/hjbh/index.html
列表: eportal/huilan, .xxgkList ul>li
标题 日期
分页: 首页 index.html, 第N页 {hash}-{N}.html (hash=4a90f99c, 每页10条)
详情: meta ArticleTitle + meta PubDate + div.detailContent
N)
BeautifulSoupzhttps://www.tieling.gov.cnzNhttps://www.tieling.gov.cn/tieling/zwgk/zfxxgk/fdzdgknr/zdmsxx/hjbh/index.htmlzXhttps://www.tieling.gov.cn/tieling/zwgk/zfxxgk/fdzdgknr/zdmsxx/hjbh/4a90f99c-{page}.htmlu 铁岭市-环境保护u 辽宁zcrawl_tieling_hjbh.pyDB_PATHz/mnt/data/search.db
JSONL_PATH utf-8zmeta:ArticleTitlez.detailContentzl]*href="(/tieling/zwgk/zfxxgk/fdzdgknr/zdmsxx/hjbh/\d+/index\.html)"[^>]*title="([^"]+)"[^>]*>(.*?)zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-Language z--pages==z--pagesc 2 | syt j | } | j dd j dd j dd j dd } t j dd| } t j dd| } | j S ) Nr u u u u u3 ^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts '/root/gov_crawler/crawl_tieling_hjbh.py clean_titler > s ! A (B ''"5==hKSST\^`aA
Er1MA
vsA A 779 c @ |dk( rt }nt j | }| j |t dd }t
dk( r |j j d S d
|_ |j S # t $ r |j j dd cY S w xY w) Nr )page Fheaderstimeoutverifygbkgb18030r errorsr ) LIST_URL_P1 LIST_URL_PNformatgetHEADERSENCcontentdecode Exceptionencodingtext)sessionr urlrs r
fetch_listr1 H s qy d + C"U CA
e| A99##E* * AJ 66M A99##Ii#@ @ As A4 4&B B c g }t j t | t j t j z D ] }|j d }t
|j d }d}| |j |j dz }t j d| }|r|j d }|rt | dk rt j j t | }|j |||f |S )Nr r x z(\d{4}-\d{2}-\d{2}) )r finditer LIST_LINK_RESIgroupr endsearchlenurllibparseurljoinr# append) html_textitemsmhreftitledateafterdmabs_urls r
parse_listrK W s E
[[ y"$$+
> -wwqzAGGAJ'!%%'!%%'C-0
YY-u
5
88A;DE
Q ,,&&{D9
gud+, - Lr c V d}d}d}t rt | d }t j d rrt j dd d }|j ddt j |t
j i }|r|j d rt |d }nt d
k( rR|j d
}|r|j d } d
D ] }
|
| v s| j |
d } n t | }nBt dk( r9t r3|j t } | r t | j dd }|sX|j ddt j dt
j i }|r |j d rt |d }|r%t j dd| j }t dk( rt rt j d r-|j ddt j dd d i } n|j t } | rl| j dd } | j d D
cg c]/ }
t! |
j dkD s |
j 1 }}
|rt |d }t rXt j d r |j ddt j dd d i } | r| j# d }t! | dkD rg }|dd D ]_ }|j dd } t j$ d| rt j$ d| d d sF|j' t) | a |rdj+ | ndj+ d |dd D }n\dj+ d | j, D }n9|j t } | r"dj+ d | j, D }|j ddt j d t
j i }|rK|j d r:t j. d!|d j }|r|j1 d }|s)t j$ d!| }|r|j1 d }|||fS c c}
w )"Nr html.parserzmeta::r metaname)attrsr) rF Tr )_-u —|r selectorr
ArticleTitleu; (?:\s*(?:发布日期[:：]\s*)?\d{4}-\d{1,2}-\d{1,2})+\s*$ firstlineztable:tableergodic
tru5 发布(?:时间|日期)|信息来源|字号|分享到u [，。；]c 3 2 K | ] }t | y wNstr).0r] s r z parse_detail.. s Sn`bTWXZT[Sn c 3 2 K | ] }t | y wr` ra rc cs r rd z parse_detail.. s .K!s1v.Kre c 3 2 K | ] }t | y wr` ra rg s r rd z parse_detail.. s *Ga3q6*Gre zPubDate|publishdatez(\d{4})-(\d{2})-(\d{2}))r DTITLE_MODE
startswithsplitfindr compiler9 r& r get_text
DTITLE_SEL
select_oner r DCONTENT_SELr= find_allr< rA rb joincontentsmatchr: )rB page_urlrF rG content_htmlsouprP mtr rawsepeltxtllinestrs body_rowsr] mdrI s r parse_detailr i s E
DL Y
6 ! !' * $$S!,Q/D6&"**T2442H)IJBbffY'#ByM2
G
# '"Ajjtj,1 Ccz !iinQ / $C(
J
&: ,B#BKK4K$@A6&"**_bdd2S)TUBbffY'#ByM2 FFY[]_dekkmE + % **84 79l>P>PQTVW>XYZ>[2\]B 6B ++c+6C03 $ W13qwwy>TVCVQWWY WE W +E!H 5 &&x0YYwy,:L:LSRS:TUV:W.XYY ++d+C3x!|$& "%ab' 6B"$++c+">C !yy)acf gprpypy {I KN OQ OR KS qT ( % , ,SW 5 6
>Grwwy'9BGGSnfijkjlfmSnLn ')ww.Kr{{.K'K __\2#%77*G2;;*G#GL
YYvfbjj9OQSQUQU.V%WY
X
"&bm6I6I6KLBxx{5yABxx{ $ $$? Xs
!R&/R&c | sddg fS t j dd| t j } t j dd| t j } t j dd| t j t j z } t j dd| t j t j z } g i t j dd| t j } t j d d| t j } t j d
d| } t j d d| } t ra t | d }|j d
d D ] }|j
d sG|j
d s6t j d|j
dd t j s\|d }|j d xsD |j
d xs1 t j j |j d d }t j j | } j! ||f dt# d}d| d| d |< |j% | |j d D ] }|j d } | s|j d r+g }
| D ] } | j
d xs | j
d xs d} | s.t j j | }|
j! d | dt j j | j d d xs d! d" |
sd#t# d}d$d%j' |
z d&z |< |j% | t) | } fd'}
t j d(|
| t j t j z } t rQ t | d }|j d D ]% }d)D ] }|j, j/ |d ' t) | } g fd*}t j d+|| t j t j z } t j d,| t j rd-nd}t j d&d.| t j } t j d/d0| t j } t j d1d| t j } g }| j d2 D ] }|j1 }|st j d3d| }t3 j4 | }|j7 d4d5 j7 d6d5 }t j d7d| }|s~ j9 D ] \ }}|j7 || } t; D ] \ }}|j7 d8| d| } |j! |j1 d2j' | } j9 D ] \ }}|j7 || } t; D ] \ }}|j7 d8| d| } t j d9d| }t j d:d| }t j d;d| }t j dd2| }t j d?d5| }t= }g } |j d2 D ]X }t j d@dt j d3d| }|s2||v r7|j? | | j! | Z d2j' | j1 }|| fS # t* $ r Y :w xY w# t* $ r Y w xY w)ANr r z
)flagsz ]*>z]*>.*?z]*>.*?z;]*src="[^"]*fileTypeImages/icon_[a-z]+\.gif"[^>]*/?>z;]*src="[^"]*/images/20200907-wz-banner\.png"[^>]*/?>u 【字号：[^】]*】u' 发布(?:时间|日期)[:：][^<]{0,30}rM aT)rE appendixz
data-appendixz\.(pdf|docx?|xlsx?|zip|rar)rE rR _title?
__ATTACH____z z
pimgsrcoldsrc *\*+2. 'y0B3%t L S
r z&]*href="([^"]+)"[^>]*>(.*?)) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingc h j | j d dt dz
dS )Nr __TBL__r r )rA r: r= )rD
table_protects r _tbl_replz html_to_text.._tbl_repl s3 QWWQZ(]+A-.b11r z]*>.*?z ]*>r z
z r[ z(]*>z
r r
z z [\s\u200b\u200c\u200d\ufeff]+r z
__LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}z [ \t]{2,}r ) r r r8 r9 r rs r& r< ro r r r rl r> r? r@ rA r= replace_withrt rb r+ rQ popr r r r rC enumeratesetadd) r) rw _soupr rE r~ rJ r r imgssrcsr r r tagattrr has_tablepartsblockbplainkvitbloutseenfinalr r r s ` @@@r html_to_textr s 1byff]Brtt"))Ljlmlqlqrxz|l} @B @D @D CE V9D**4*0kAEE(OkrwwGWGWX\XbXbcfXghiXjGkC $ll 2 28T BG &&~6 &s<'8&9
G[][_[_bdbfbf[fgG !'=9E ~~d+
. L .DIIMM$-.
. %jG M2 ff/G244RTRVRV;WGYY>AIffWj'>Gff\4=Gff@"gUWUYUYZG E v& KKM z2q)!!%(
fc*228SA7UC &&( DAq !QA
. 0FAs GA3b/3/A 0
QWWY ++e
C ""$ 1kk!Q M* 03kkGA3b/3/0
&&!2s
+C
&& "c
*C
&& "c
*C
&&#R
-C
C
C
&&dC
(C
&&FC
(C
&& sC
(C 5D E
YYv
ffVR
B!:; $;
Q ++e
"
"
$C ; &&i 2 sG A:_
C'_ 5_ 1_ :A/_ *A_ 8A_0 _-,_-0 _=<_=c
| j d|t f }|j ry d} ddlm} || }| j d|||||t t t ||f
}|j dkD ryt j dd| } t j | } t j d d
| j d d } | j d |j |t | f y
y# t
$ r Y w xY w# t j $ r=}
dt# |
v r*t% j& d t) | ||||| cY d }
~
S d }
~
ww xY w)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?Fotherr )classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)r r r r
zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)Tlocked )execute SITE_NAMEfetchone crawler_libr r+
GROUP_NAME SCRIPT_NAMErowcountr r r r r lastrowidsqlite3OperationalErrorrb timesleep store_record) connr/ rF r) rG r curindustryr summaryes r r r 9 sF
,,PSVXaRb
cC
||~ 5(/H ll
+ #ugtY
KQZ\d e
< d 5 t= j> d t
rmtA t
d d 5 }|D ]+ }|jC tE jF |d ! d"z - d d d t d#t
dt# | d$d
| |jI |jK t d%| d
t d&| d
t d't4 d(d
y # t $ r } t d| d| d
Y d } ~ d } ~ ww xY w# t $ r#} t d| d| d
|dz
}Y d } ~ jd } ~ ww xY w# 1 sw Y xY w))Nr < )r zPRAGMA busy_timeout=60000zPRAGMA journal_mode=WALr zPage z fetch error: T)flushz: found z itemszEmpty page, stop paginationz detail error z: r r r\ z skip empty: r )rw rF r) publish_dater site_name
group_name script_namez
JSONL + ( z ()z + g ?g ?wr )r, F) ensure_asciir[ zJSONL written: z rows)u 新增: u 跳过: z=== z done ===)&requestsSessionr updater' r r connectr r r range_PAGESr1 r+ printrK r= r r r r r r r rA r r r r r r openwritejsondumpscommitclose)r sr new_count
skip_count seen_urls
jsonl_rowsr rB r rC rJ rF rG dhtmldtitleddaterx r) r atts plain_lenokfrows r mainr h sx AIIW D w3 01 ./IJIJa!$ 1 "1d+I 9%
dV8CJfETR
HYK
- HZL
!. D 9
%T2w E$~aS1 >
y1#6dCa
F D Ds< ! L L,1M L) L$$L), M5MMM$__main__)2__doc__sysr r r htmlr urllib.parser> r r bs4r ImportErrorBASE_URLr# r$ r r r environr& r r r( rj rp rr r7 r'
_MAX_PAGESr argvr r rk intrl
ValueErrorr= r r r1 rK r r r r r __name__ r r r s !
(^ h $
%
**..$9
:
ZZ^^L"
-
!
~ @O'
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA $G%TC'L D
0M3` zF E
M:
s5 D/ *D=&E/D:9D:=EEEE