{j8 f d Z ddlZddlZddlZddlZddlZddlZddlm Z m
Z
dZ dZ dZ
dZdZd Zd
Zd Zd d
e ddZdZej* j- ej* j/ ej* j1 e d ZdZdZ d Z d Z d Z d Z d Z!d Z"d Z#e$dk( r e# yy) u
温州瓯江口(海洋经济发展示范区) - 建设项目环境影响评价信息公示 (ojk.wenzhou.gov.cn/col/col1229874415/index.html)
CMS: 浙江政务 JCMS (ZJSJYH 单元构建) - 与新昌/洞头同款
列表: GET /api-gateway/jpaas-publish-server/front/page/build/unit
params: parseType=bulidstatic&webId=2070&tplSetId=fbSBcd1C5fiSI7LwwVOpW&pageType=column
&tagId=内页分页列表&editType=null&pageId=1229874415
¶mJson={"pageNo":N,"pageSize":20,"search":""}
响应 JSON data.html: 标题
count=131 总条数 -> 7页 (20条/页)
详情: /col/col1229874415/art/YYYY/art_hex.html
正文 div.zwnr_con > div#zoom > p
meta PubDate / 时间：YYYY-MM-DD HH:MM
样本: https://ojk.wenzhou.gov.cn/col/col1229874415/art/2026/art_e85b48cb6178421491d45cbc10041d87.html
用法: python3 crawl_ojk_hpgs.py [--pages=N | N] [--sync-only]
N)
BeautifulSoupTagu( 温州瓯江口-建设项目环评公示zhttps://ojk.wenzhou.gov.cnzQhttps://ojk.wenzhou.gov.cn/api-gateway/jpaas-publish-server/front/page/build/unit
1229874415u 内页分页列表fbSBcd1C5fiSI7LwwVOpW2070 zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z"https://ojk.wenzhou.gov.cn/col/colz /index.html)z
User-AgentRefererz
2023-01-01zquality_results.dbz/root/search.dbzojk.wenzhou.gov.cnc 0 dd l }|j t }|j d |j }|j d d}| D ] } |j d|j dd |j dd |j dd |j d d |j d
d |j d d |j d d |j d
d |j dd |j dd f
||j z
} |j |j t d| dt | |z
|S # t $ r,}t d|j dd d| Y d }~@d }~ww xY w)Nr PRAGMA busy_timeout=300000a CREATE TABLE IF NOT EXISTS crawl_results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
site_id INTEGER DEFAULT 0,
title TEXT, url TEXT, content TEXT,
publish_date TEXT, summary TEXT,
domain TEXT, category TEXT DEFAULT '',
content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
industry TEXT DEFAULT '', group_name TEXT DEFAULT '',
script_name TEXT DEFAULT '', attachments TEXT DEFAULT '[]',
UNIQUE(domain, url)
)zINSERT OR IGNORE INTO crawl_results (domain, title, url, content, publish_date, summary, industry, group_name, script_name, attachments) VALUES (?,?,?,?,?,?,?,?,?,?)domain titleurlcontentpub_datesummaryindustry
group_name script_name attachments[]z
[push] ERR : z[push_to_searchdb] new=z skip=) sqlite3connect
QUALITY_DBexecutecursorgetrowcount Exceptionprintcommitcloselen)records batch_labelr conncurn_newres #/root/gov_crawler/crawl_ojk_hpgs.pypush_to_searchdbr- - ss ??: &DLL-.
++-CKK
E
:
: KK xx$aeeGR&8!%%r:Jy"%quuZ'Rz2&lB(?}b)155 +EG
S\\ !E : KKMJJL #E7&W e1C0D
EF L : M!%%b/!2"QC8 9 9 :s C E F)!FFc dd l } | j t }|j d |j dt f j }|j
|s t d y | j t d }|j d |j d dx}} |D ] }|\ }} }
} } }
}}}|j d | f }|j } |rF|j d
||
xs d d d | xs d | xs d d d |
xs d |xs d |xs d |xs d
|d f |dz
}nH|j d|| |
xs d d d | xs d | xs d d d |
xs d |xs d |xs d |xs d
t | f |dz
} |j |j dt f |j |j dt f j d }|j
t d| d| d| d y # t $ r}t d| Y d }~y d }~ww xY w# t $ r }t d| d| Y d }~d }~ww xY w# |j
w xY w)Nr r zSELECT title, url, content, publish_date, summary, industry, group_name, script_name, attachments FROM crawl_results WHERE domain=? ORDER BY idz[sync] local read ERR: u [sync] 本地没有数据i, )timeoutzPRAGMA journal_mode=WALz'SELECT id FROM gov_raw WHERE page_url=?zUPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, industry=?, group_name=?, script_name=?, attachments=? WHERE id=?r
i r zINSERT INTO gov_raw (title, page_url, content, publish_date, summary, industry, group_name, script_name, attachments, site_name, source_url) VALUES (?,?,?,?,?,?,?,?,?,?,?)z
[sync] ERR r zINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z[sync] OK updated=z
inserted=u DB共u 条)
r r r r DOMAINfetchallr# r r! SEARCH_DBfetchone SITE_NAMEr" )r r' rowsr+ dstupdinsr* r r r r r indgrpsnattr( rowtotals r, sync_to_serverr@ P s
z* 12||
^
I
(*
)*
//)S/
1CKK)*KK,-MC# 2AHI EE37Cb#++ G#PC,,.C
2KK ^ B 8(.b7=VXZ^[^J_ CI2rxRc!fNO 1HCKK F gmWf%=x~2PWP][]_c`cOd CI2rxRiQTVW 1HC! 2&
Q L
Lyl[ddfghi se:cU&s
CDQ
's+,8
2
cU"QC011
2 sP AG9 -4I "BH 2A!I 9 HHH I&H=7I =II Ic t j dd| xs d } t j dd| } t j dd| j S )Nz$·| | |\u200b|\ufeffr
u ^[•·]\s*z\s+ )resubstrip)ts r, clean_titlerG sG
6AGDA
~r1%A
66&#q ! ' ' )) c L t j d| xs d }|r[t |j d t |j d t |j d }}}|dd|dd|dS t j d | xs d }|r|j d
S y) Nu+ (\d{4})[年\-/.](\d{1,2})[月\-/.](\d{1,2})r
r0 04d-02dz(\d{4})-(\d{2})-(\d{2})r )rC searchintgroup)smymods r, normalize_daterW s
@!'rJAqwwqz?C
OS_q2C"S1S'**
,ag26Awwqz
rH c 4 t j | t ddd }dt t dt
dt |d} t j t |t d
}|j |j }|j d s%t d | d
|j d g ddfS |d d }g }d}t j d| } | rt! | j# d }t% |d }
|
j' dd D ] } | d j) } d| vrt j d| s2t+ | j- d }
|
sP| j/ d r| n t0 | j/ d r| nd | z z }|j3 |
|dd t5 }g }|D ]/ }|d |v r |j7 |d |j3 | 1 |d|fS # t $ r }t d | d| g ddfcY d }~S d }~ww xY w)!Nr
)pageNopageSizerO F ensure_ascii bulidstaticcolumnnull) parseTypewebIdtplSetIdpageTypetagIdeditTypepageId paramJson )paramsheadersr/ successz
[list] Pz API fail: messager datahtml ERR: z
count="(\d+)"r0 html.parseraThrefrs z/art/z /art/\d{4}/art_[0-9a-f]+\.html$rE http/)r r r r ) jsondumps PAGE_SIZEWEB_IDTPL_SETTAG_IDPAGE_IDrequestsr API_URLHEADERSraise_for_statusr! r rC rO rP rQ r find_allrE rG get_text
startswithBASE_URLappendsetadd)page
param_jsonri r* jrn r+ itemscountrS souprq rs r full_urlseenuniqits r,
fetch_listr s tb Q`efJ" F
LL" M
FFHuuY JtfKi0@/AB Cua< y
E
E
"D)AAGGAJ } -D
]]3T]
*
Hy $ yy;TB AJJTJ 23 ??624DOO\_L`Dfilpfp8q
uX2FG
H 5D
D
e9 E B
u 7
4&qc*+5!| s$ A6G. 3G. . H7H HHc
t j | t d }|j d|_ t |j d }d}t j d|j t j }|r=t t j d d|j d
j }|s,|j! d }|rt |j# }|s6|j! d d
di }|r |j d rt |d }d}|j! d d
di }|r'|j d r|d j d d }|s3t j d|j }|r|j d
}|j% d xs$ |j% d xs |j% d } | -|j' t j( d D ] }
|
} n | |d|g dS g } g } | j' d D ]* }
t+ |
t, s|
j d xs dj }t j d |t j. s d |v sd |v sf|j1 d r|n4|j1 d rd!|z n t2 |j1 d" r|nd"|z z t |
j# d#$ xs' |
j d% xs |j5 d" d& }t7 fd'| D s| j9 |d( | j9 |
- | D ]& }
|
j; d) }|s|j= ( | j' dd#* D ] }
|
d j }|j1 d" s5|j1 d+ s$|j1 d rJ|j1 d, r\|j1 d rd!|z n t2 |j1 d" r|nd"|z z |
d< | j' d- D ]a }|j d.d }|s|j1 d r*|j1 d rd!|z n t2 |j1 d" r|nd"|z z |d.< c | j? d/ D ] }
|
j= | j' t j( d0 1 D ]9 }
|
j; d) xs |
j; d2 }|s*|j= ; tA | jC t j d3dt j t j. z 4 t j d5dt j t j. z 4 t j d6d7 t j d8d t j d9dt j 4 t j d:dt j 4 | rDt7 fd;| D }|s.djE d< | D }jG d=z |z tI t j d>d j }d?v }d@v xs dAv }|dBk r
|s |s | s|d|| dS ||| dS # t $ r}t d| d| Y d }~y d }~ww xY w)CNrh )rj r/ zutf-8rp z [detail] ro r
z(.*?)u
[-_—]\s*.*$r0 h1metaname ArticleTitle)attrsr PubDate
u@ (?:时间|发布日期|发布时间)[：:]\s*(\d{4}-\d{2}-\d{2})z div.zwnr_conzdiv#zoomzdiv.ny-contentz/zwnr_con|ny-content|article-content|TRS_UEDITOR)class_r r r r rq rs z)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$z/document/downloadz /attachment/ru z//zhttps:rv Trt downloadc 3 . K | ] }|d k( ywr N ).0xabs_urls r, z parse_detail.. s 9qqx7*9s )r r prr z./
javascriptimgsrczmdiv.share, div.ewm, div.qrcode, div.jzgov-qrcode-div, div.app-share, div.print, div.bshare-custom, div.fzxxzsu 扫一扫|分享到|打印本页)stringdivz]*>.*?)flagsz]*>.*?z]*>z
N)html_modescape)r rq s r, r z parse_detail.. s3 mcd ahZr(//!F):T9UU] ^ ms ,.
<[^>]+>z
"&&# 9
.EH & 3 4AQUU9
Y<%%', IIY[\[a[a b wwqzHoon-q 1LqPTP_P_`pPqG--rzz2c'd-e BG 28TVWW DI
c
" !S! f
#**,
99A4 NRfjnRn sA EI sI "oof 5dt_cOdHtOjr C N N OR Svz Y\ _c Yc kdG qzzz 56b!%%
:KbtzzZ]_aObD9D99 T' :; Q
MM#
KKM
c
- By ??3 4??4#8 QWAXaeapapq}a~,0OOD,AD x[_[j[jkn[oSWux{u HAAfIB & ~ggeR s~~f-,/NN4,@(S.hY\YgYghkYlRUrux{r{F|CJ~
nn M N
bjj1S&T U NN3 82>>%#8
KKM
G++-.I3R"$$QSQUQU+VI12yrtt TI{E95I}b)4I7YbddSI4b)244PI :T::ww mhl mmH!((*T1H'Y*>I!|GId 28TXYYyhW[ \\s
C5qc*+s A Z: : [ [[ c dt j v } d }t j dd D ]O }|j d r t |j dd d }4|j
sEt | }Q t d| d| | r t y g }d}t dd D ]V }|r||kD r nLt | \ }}}|r|}t d | d
t | d | d |s n|D ] } t | d
}
|
!t d| d d d | d d| d g d}
|
d r'|
d t k rt d|
d d|
d d d at t |
d xs | d | d
|
d |
d xs | d t! j" dd|
d d d ddt$ j& j) t j d |
d rt+ j, |
d d nd d } |j/ | |rW n t d t | d | d |rXt1 |t$ j& j) t j d d!t j v r t y y y # t
$ r Y \w xY w)"Nz --sync-onlyr0 z--pages==z[AutoPg] max_pages=z sync_only=r i z[list] Pz: items=z (count=)r z [fail] r 2 r
r r z [cutoff] rB ( r r u 浙江r Fr[ r ) r site_namer r r r r r r r r z[total] valid=z--sync)sysargvr rP r
ValueErrorisdigitr! r@ ranger r$ r CUTOFFr1 r5 rC rD ospathbasenamerw rx r r- ) sync_only max_pagesrq valid total_countpgr has_nextr r detrecs r, mainr * s )II
XXab\ <<
#
Q 2 3 YY[AI
{+i[
AB EKAtn i !+B x K
HSZLqAB B r%y)C{ "W+cr"2!345 "7 :gij:3z?V#; C
O#4Ac'l3B6G5HIJ & W 4G%y y>
O =r*~ 66*b#i.A$3G & !ww / / 0 0! => sxx
[
s I88 JJ__main__)%__doc__r~ rC r r rw rn r bs4r r r5 r r r} r| r{ rz ry r r r r dirnameabspath__file__r r3 r1 r- r@ rG rW r r r __name__r rH r, r s
" 6 '
]
! D3G9KH
WW\\"''//"''//(*CDFZ
[
F2Ej* / d_]D7 t zF rH