uj d Z ddlZddlZddlZddlZddlZddlZddlZddl m Z m
Z
ddl m Z m
Z
ej j dd ZdZdZ e j& e
d
z
j) d Zd Zd
ddZddlZ ej2 Zej6 j9 e ddZ d Z d Z d Z d Z!e"dk( r e! yy)uP crawl_taihe_hp.py — 太和县生态环境分局-建设项目环评文件审批 N)datetime timedelta)ThreadPoolExecutor as_completed SEARCH_DBz/root/search.dbu: 太和县生态环境分局-建设项目环评文件审批zhttps://www.taihe.gov.cniG )daysz%Y-%m-%d zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36XMLHttpRequest)z
User-AgentzX-Requested-Withc t d D ]. } t j | | }d|_ |j c S y # t
$ r }t
j d Y d }~Ud }~ww xY w)N timeoutzutf-8 )rangesessgetencodingtext Exceptiontimesleep)urlr _res #/root/gov_crawler/crawl_taihe_hp.pyfetchr s`
1X g.A AJ66M JJqMM s )> A&A!!A&c T g }t j d| t j }|s|S t j d|j d t j D ] }|j d }t j d| }t j d| }|sC|sF|j t j j t |j d |j d j |j d j d |S )z"Parse articles from AJAX list HTMLzz(.*?)r z%([^<]+) )r titledate) researchDOTALLfinditergroupappendurllibparseurljoinBASEstrip)htmlitemsul_matchlicontentaspans r parse_list_htmlr4 $ s Eyy*D")) J ((1+ II> Hyy0': LL ||++D!''!*=))+
1
++-
L c t j d| }|r|j d nd}|sHt j d| t j }|r |j d j nd}d}t j d| t j }|r |j d j }t j
dd|t j }t j
dd|t j }t j
d d| }t j
d
d| }t j
d d| }||j fS ) z*Extract title and content from detail pagez/(.*?)zU]*>(.*?)
\s*\s*\s*z]*>.*?)flagsz]*>.*?z\s*zz ]*>\s*(?:\s*)*)r" r# r& r$ r, sub)r- title_mr tr1 ms r parse_detailr= 8 s iiJDQG 'GMM! RE IIEtRYY W&'
"RG
jlprtr{r{|A''!*""$ ff12wbiiPGff/WBIINGff&G4GffYG,Gff8"gFG '--/ !!r5 c t | d }|s
dd| d dd fS t | \ }}|xs | d }t || d | d | d |ddfS )z#Fetch detail and return insert datar Nz FETCH_ERR: r r! ) site_namer page_url publish_date
source_urlr1 )r r= SITE_NAME)itemr- dtr1 final_titles r process_itemrH P s e D {4="#5"6777 t$KB%W
K K V 5k
r5 c
t dt j j d dt t d } | s t d y t
j d| }|rt |j d nd}t d | t | }t d
|dz D ]h }t d | d }|r t | }|j | nt d
| d t j t j dd j t# }g }|D ]/ } | d |vs |j% | d |j' | 1 t dt) | |D
cg c] }
|
d t* k\ s|
} }
t dt) | dt) | t) | z
d t- j. t0 d } | j3 }
t# }|
j5 dt f D ] }|j% |d | D
cg c] }
|
d |vs |
}}
t dt) | dx}}t7 t8 5 }|D ci c] } |j; t< | | }} t? tA | d D ] \ }
}|| } |jC \ }}|s|s0t d |
d t) | d| d d d d |xs d! |dz
}R |
j5 d"|d# |d |d$ |d% |d& |d' f |
jD r|dz
}d(t) |d' d)}nd*}t d |
d t) | d| d d d d | | jI d d d | jK t d,| d-| y c c}
w c c}
w c c} w # tF $ r5}t d |
d t) | d| d d d d+| |dz
}Y d }~d }~ww xY w# 1 sw Y {xY w).N[z%H:%M:%Sz] z=https://www.taihe.gov.cn/OpennessTarget/350/29374/page_1.htmlz [ERROR] Failed to fetch page 1zpagecount="(\d+)"r ' z
Total pages: r z7https://www.taihe.gov.cn/OpennessTarget/350/29374/page_z.htmlz [WARN] Page z failedg?g?r zTotal unique: r! zWithin 3 years: z (filtered: )r? r
z.SELECT page_url FROM gov_raw WHERE site_name=?r z New items: ) max_workersz [/r z... FAILzINSERT OR IGNORE INTO gov_raw
(site_name, title, page_url, publish_date, source_url, content)
VALUES (?,?,?,?,?,?)r@ rA rB rC r1 zOK (zB)DUPz ... ERR: z
=== Done === New: z
, Errors: )&printr nowstrftimerD r r" r# intr& r4 r extendr r randomuniformsetaddr' lenCUTOFFsqlite3connectDB_PATHcursorexecuter MAX_WORKERSsubmitrH enumerater resultrowcountr commitclose)r- pc total_pages all_itemsphr. seenuniquerE irecentconncexistingrowto_fetch new_count error_countexecutorfuturesfuturerd errstatusr s r mainr} c s$ Ahlln%%j1
2"YK
@A P QD
./
' .B&(#bhhqk"bK M+
'( %I
1kAo
& - KA3eT U #A&E U # N1#W- .
6>>#s+,- 5D
F ;d " HHT%[ ! MM$
N3v;-
()
7A1V9#6a
7F
7 S[M c&k#f+6M5Na
PQ ??7B /D
AuHyyII<X SV ">aQuXX%=>H> KH
'( I 4 IQR8??<6<RR"<#8!< IAv6?D --/KFC&A3aH
bg s1C0DD PVXYq
! , K(&/6*;MN+VL-A6)CTVW
:: NI #Cy(9$:#;2 >F "FA3aH
bg s1C0DDQR
KKM3 : JJL :k]
CD] 8 ? S,
!A3aH
bg s1C0DIaSQRq
!/ sb 2N!N!# N&0N&!O1& N+A)O1.A6N0$O1+O10 O.9+O)$O1)O..O11O:__main__) )#__doc__osr" r\ r sysrV urllib.parser( r r concurrent.futuresr r environr r^ rD r+ rR rS r[ ra HEADERSrequestsSessionr headersupdater r4 r= rH r} __name__ r5 r r s W 7 7 7 7 ( ?
**..&7
8 H !
(,,.9%0
0 : :: F D(
x G ("0 &REj zF r5