uj d Z ddlZddlZddlZddlZddlZddlZddlZddl m Z m
Z
ddl m Z m
Z
ej j dd ZdZdZ e j& e
d
z
j) d Zd Zd
diZddlZ ej2 Zej6 j9 e ddZ d Z d Z d Z d Z!e"dk( r e! yy)uR crawl_kmsthjj.py — 昆明市生态环境局-建设项目环境影响评价信息 N)datetime timedelta)ThreadPoolExecutor as_completed SEARCH_DBz/root/search.dbu= 昆明市生态环境局-建设项目环境影响评价信息zhttps://sthjj.km.gov.cniG )daysz%Y-%m-%d z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c @ | j d s$t j j t | n| }t d D ]. } t j || }d|_ |j c S y # t $ r }t j d Y d }~Ud }~ww xY w)Nhttp timeoutzutf-8 )
startswithurllibparseurljoinBASErangesessgetencodingtext Exceptiontimesleep)urlr full_url_res "/root/gov_crawler/crawl_kmsthjj.pyfetchr# s 69nnV6Lv||##D#.RUH
1X 73A AJ66M JJqMM s )A55 B >BB c g }t j d| t j D ] }|j d }t j d| }t j d| }|sC|j t j j t |j d |j d j |r|j d ndd |S )z$Parse article items from a list pagezC(.*?)
\s*\s*r z#\s*([^<]+)\s*ztitle="(\d{4}-\d{2}-\d{2}) )r titledate) refinditerDOTALLgroupsearchappendr r r r strip)htmlitemsblockcontenta
date_matchs r" parse_list_pager6 $ s E ceikmktktu ++a. II(.*?)zN(.*?)
\s*\s*\s*\s*\s*z3(.*?)
\s*\s*z]*>.*?)flagsz]*>.*?z\s*zz ]*>\s*(?:\s*)*)r) r- r, r/ r+ sub)r0 title_mr' h1 content_mr3 s r" parse_detailr> 5 s( iiJDQG 'GMM! RE
YY($
/') !!#r kmqsus|s|}I II TVZ\^\e\ef ,5iooa &&(2G ff12wbiiPGff/WBIINGff&G4GffYG,Gff8"gFG '--/ !!r7 c t | d }|s
dd| d dd fS t | \ }}|xs | d }t || d | d | d |ddfS )z*Fetch detail and return data for insertionr Nz FETCH_ERR: r' r( ) site_namer' page_url publish_date
source_urlr3 )r# r> SITE_NAME)itemr0 dtr3 final_titles r" process_itemrI P s e D {4="#5"6777 t$KB%W
K K V 5k
r7 c $ t dt j j d dt dg} t dd D ] }| j
d| d g }| D ] }t | }|rJt | }|j | t d | j | d
z d t | d nt d
| t j t j dd t! }g }|D ]/ }|d |vs |j# |d |j
| 1 t dt | dt | d |D cg c] }|d t$ k\ s| } }t dt | dt | t | z
d t' j( t* d }
|
j- } t! } | j/ dt f D ] }
| j# |
d | D cg c] }|d | vs | }}t dt | d}d}t1 t2 5 }|D ci c] }|j5 t6 | | }}t9 t; | d
D ] \ }}|| }|j= \ }}|s|0t d | d t | d|d d d d |xs d! |d
z
}R | j/ d"|d# |d |d$ |d% |d& |d' f | j> r9|d
z
}t d | d t | d|d d d d(t |d' d) n$t d | d t | d|d d d d* |
jC d d d |
jE t d, t d-| d.| y c c}w c c}w c c}w # t@ $ r5}t d | d t | d|d d d d+| |d
z
}Y d }~d }~ww xY w# 1 sw Y xY w)/N[z%H:%M:%Sz] zNhttps://sthjj.km.gov.cn/zfxxgk/fdzdgknr/zdlyxxgk/hjbh/jsxmhjyxpjxx/index.shtmlr% zIhttps://sthjj.km.gov.cn/zfxxgk/fdzdgknr/zdlyxxgk/hjbh/jsxmhjyxpjxx/index_z.shtmlz List page r z: z itemsz [WARN] Failed: g333333?g ?r z
Total unique: z (deduped from )r( zWithin 3 years: z (filtered: r@ r
z.SELECT page_url FROM gov_raw WHERE site_name=?r zNew items to fetch: ) max_workersz [/r' z... z UNKNOWN ERRzINSERT OR IGNORE INTO gov_raw
(site_name, title, page_url, publish_date, source_url, content)
VALUES (?,?,?,?,?,?)rA rB rC rD r3 z... OK (zB)z... DUPz ... ERR: z
=== Done ===zNew: z
, Errors: )#printr nowstrftimerE r r. r# r6 extendindexlenr r randomuniformsetaddCUTOFFsqlite3connectDB_PATHcursorexecuter MAX_WORKERSsubmitrI enumerater resultrowcountr commitclose) list_urlsi all_itemsr r0 r1 seenuniquerF recentconnc
existing_urlsrowto_fetch new_count error_countexecutorfuturesfuturerc errr! s r" mainrx c s\ Ahlln%%j1
2"YK
@A YI 1a[ pdefdggmnop I -Sz #D)E U # L !5a!7 83u:,fM N %cU+ ,
6>>#s+,- 5D
F ;d " HHT%[ ! MM$
S[M Y 8H
JK
7A1V9#6a
7F
7 S[M c&k#f+6M5Na
PQ ??7B /D
AEMyyII<X "#a&!" "CaQuX]%BCHC X
01 IK 4 IQR8??<6<RR"<#8!< IAv6?D --/KFCfnA3aH
bg s1C0DDI]P]H^_`q
! , K(&/6*;MN+VL-A6)CTVW
:: NICs!CM?"T']3B5G4HQTU[\eUfQgPhhjklCs!CM?"T']3B5G4HPQ
KKM1 : JJL N E)J{m
45a 8 D S,
!A3aH
bg s1C0DIaSQRq
!/ sb N61N6 N; N;P O 1A)PBO.P P P+O>9P>PPP__main__) )#__doc__osr) r[ r sysrV urllib.parser r r concurrent.futuresr r environr r] rE r rQ rR rZ r` HEADERSrequestsSessionr headersupdater# r6 r> rI rx __name__ r7 r" r s Y 7 7 7 7 ( ?
**..&7
8 K
(,,.9%0
0 : :: F D x G ""6 &P6f zF r7