uj$" d Z ddlZddlZddlZddlZddlZddlZddlZdZ dZ
e
dZ dZ dZ
dZ ej Zd e_ ej$ e_ dd
Zd Zd Zd
Zd Zedk( r e yy)uS
招远市政府 - 建设项目环评审批 (zhaoyuan.gov.cn)
CMS: TRS/JCMS, AJAX 分页 (client-side only, 15 items)
API: /api-gateway/jpaas-publish-server/front/page/build/unit
webId=155, pageId=48655, tagId=右侧文章列表
List: ul.bt-main-r-ul > li.bt-main-r-ul-li > a[title] + span(date)
Detail: /col/col48655/art/2026/art_{hash}.html
Content: div.text#zoom (> p + table)
Title: div.title > h1
Date: span.time "发布日期：YYYY-MM-DD"
Attachments: inside content
15 items/page, only page 1 available (AJAX client-side pagination)
Nu" 招远市-建设项目环评审批zhttps://www.zhaoyuan.gov.cnz/api-gateway/jpaas-publish-server/front/page/build/unit?parseType=bulidstatic&webId=155&tplSetId=sW4TqQJo1K65B01yzYBRg&pageType=column&tagId=%E5%8F%B3%E4%BE%A7%E6%96%87%E7%AB%A0%E5%88%97%E8%A1%A8&pageId=48655&pageNo=1z/root/search.dbz
2023-01-01 Fc N t j j | ddi }|r|j d| t j j |dt
}|j
j dd S # t $ r}t d
| d | Y d }~y d }~ww xY w) Nz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36)headersReferer )timeoutcontextzutf-8replace)errorsz [FETCH ERROR] : )
urllibrequestRequest
add_headerurlopenssl_ctxreaddecode Exceptionprint)urlrefererreqrespes */root/gov_crawler/crawl_zhaoyuan_hpjpxz.pyfetchr ( s
..
H/
C y'*~~%%c2w%Gyy{!!')!<<
Rs+,s AB B$ B B$c t t d } | sg S t j | }|j di }t |t r|j dd }n t | }g }d}t j ||t j D ] }|j d j }|j d j }|j d j } |j d
s t |z }|j! ||| d |S # t j $ r g cY S w xY w) zFetch list via AJAX APIz3https://www.zhaoyuan.gov.cn/col/col48655/index.html)r datahtml z]*class="[^"]*bt-main-r-ul-li[^"]*">.*?]*href="([^"]*)"[^>]*title="([^"]*)"[^>]*>.*?]*>(\d{4}-\d{2}-\d{2})r http)r titledate)r API_URLjsonloadsget
isinstancedictstrJSONDecodeErrorrefinditerDOTALLgroupstrip
startswithBASE_URLappend)
r r d list_htmlitemspatternmatchhrefr% r& s
r fetch_list_apir= 6 s "W XD zz$ HHVR a fb)IAI
E VGWi; B{{1~##% A$$&{{1~##%v&d?D
TE4@A
B L s AD( (E ?E c
\ d}d}d}t j d| }|r4t j dd|j d j }t j d| }|r|j d }| j d }|dk r| j d }|dkD r| j
d d| }|dk r|}g d
}t | } |D ]% }
| j |
| } d| cxk r| k s!n $| } ' | || } t j d d| t j }
t j d
d|
t j }
t j dd|
}
t j dd|
}
|
j dd }
g }t j d|
t j D ]. }|j d }t j d|t j }g }|D ] }t j d|t j }|D cg c]' }t j dd| j ) }}|s]t | si|j ddj | z dz |st | dk\ r/|d j d dz
}|j! ddd|z z |j dj | 1 t j dd|
}t j d d | }t j d d| }t j d d| }|j }|r|gng }|r |j dj | dj | }g }t j d | D ]y }|j d } |j d j }
| j# d! s t$ | z } |j | |
xs | j' d" d# d$d% d& { ||||d'S c c}w )(z-Extract title, date, content from detail pager! z]*>(.*?)z<[^>]+>r u( 发布日期[：:]\s*(\d{4}-\d{2}-\d{2})z id="zoom"r z class="text"zr$ =N2 )r< text)r% r& content attachments)r/ searchsubr2 r3 findrfindlenr1 r
r0 findallanyr6 joincountinsertr4 r5 split) r r% r& rH h1_match
date_matchzoom_idxstart end_markersendmarkerpos content_htmlrG tables table_match
table_htmlrows table_linesrowcellsc
cell_textshdrcleanpartsrI r; r< s r parse_detailrj Q s E
DG yy/6Hz2x~~a'89??A FMJ " yy%H!|99^,!|
61h/ 19 Ek $i! F))FH-C3}} E# vv2B BIIVvv0"d"))L vvlD$/vvgvt,||D"% ;;'A4S
6K$**1-J:: 4j"))LD K
M
#?biiPIN OAbffZQ 7 = = ? O
O #j/ &&tejj.D'Dt'KL
M
{#q(%a...s3a7C &&q# *<=
dii 45
6 z2t,y#u-y$.y&%0
b LLV, -++e$ K UW[\ U{{1~{{1~##%v&d?DD$2R$**S/":Mcr:RST U DW[ YY= Ps ,P)c t j t d }|j }d}d}| D ] }|j dd }|j dd }|j dd }|j dd } d}
|j d g D ] } |
r|
d
z
}
|
d | d d
| d dz
}
| s|s|dz
} |j d||t || |
||f |j dkD r|dz
}n|dz
} |j |j ||fS # t $ r$} t d|d d d| |dz
}Y d } ~ d } ~ ww xY w)N r r r% r! r r& rH rI r@ [rG z](r< )r a
INSERT OR IGNORE INTO gov_raw
(title, page_url, site_name, publish_date, content, summary, attachments, source_url, date_rank)
VALUES (?, ?, ?, ?, ?, '', ?, ?, CAST(strftime('%s', ?) AS INTEGER))
z
[DB ERROR] r ) sqlite3connectDB_PATHcursorr* execute SITE_NAMErowcountr r commitclose)
r9 connre insertedskippeditemr% r r& rH
attach_strattr s
r insert_to_dbr sx ??7B /D
AHG "%hhub!xx#((9b)
88M2. !&>DL +Y $jj ;] ' * *SS S
VC5 23 i[
9+
>?1}w3 KL
m __main__)N)__doc__r r/ osr( rp urllib.requestr
sslru r5 r' rr r r create_default_contextr check_hostname CERT_NONE verify_moder r= rj r r __name__ r r r s
1 (Z $ %
$#
$
$
& mm 6OZd& R7 t zF r