{jW8ndZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z dZ dZdZdZd Zd Zd Zd d eddZdZej,j/ej,j1ej,j3edZdZdZdZdZdZ dZ!dZ"dZ#dZ$e%dk(re$yy)u 新昌县人民政府 - 通知公告 (www.zjxc.gov.cn/col/col1229789309/index.html) CMS: 浙江政务 JCMS (ZJSJYH 单元构建) 列表: GET /api-gateway/jpaas-publish-server/front/page/build/unit params: parseType=bulidstatic&webId=2761&tplSetId=6pucDKzxQsGukk9oZp74P&pageType=column &tagId=当前栏目列表1a&editType=null&pageId=1229789309 ¶mJson={"pageNo":N,"pageSize":15,"search":""} 响应 JSON data.html: 标题 count=11459 总条数, 15条/页 -> 764页 详情: /col/col1229789309/art/2026/art_xxx.html 正文 div.ImporF_bd-main > div.artic_main > p 发布时间: 发布时间：YYYY-MM-DD HH:MM 附件: /api-gateway/jpaas-web-server/front/document/download?fileUrl=... 带 download 名 样本: https://www.zjxc.gov.cn/col/col1229789309/art/2026/art_f23e7d9d82777ae765b177327484ad8b.html 用法: python3 crawl_zjxc_tzgg.py [--pages=N | N] [--sync-only] N) BeautifulSoupTag)quoteu新昌县-通知公告zhttps://www.zjxc.gov.cnzNhttps://www.zjxc.gov.cn/api-gateway/jpaas-publish-server/front/page/build/unit 1229789309u当前栏目列表1a6pucDKzxQsGukk9oZp74P2761zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zhttps://www.zjxc.gov.cn/col/colz /index.html)z User-AgentRefererz 2023-01-01zquality_results.dbz/root/search.dbzwww.zjxc.gov.cnc0ddl}|jt}|jd|j }|jdd}|D]} |jd|j dd|j dd|j dd|j d d|j d d|j d d|j d d|j d d|j dd|j ddf ||j z }|j|jtd|dt||z |S#t$r,}td|j ddd|Yd}~@d}~wwxYw)NrPRAGMA busy_timeout=300000aCREATE TABLE IF NOT EXISTS crawl_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_id INTEGER DEFAULT 0, title TEXT, url TEXT, content TEXT, publish_date TEXT, summary TEXT, domain TEXT, category TEXT DEFAULT '', content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, industry TEXT DEFAULT '', group_name TEXT DEFAULT '', script_name TEXT DEFAULT '', attachments TEXT DEFAULT '[]', UNIQUE(domain, url) )zINSERT OR IGNORE INTO crawl_results (domain, title, url, content, publish_date, summary, industry, group_name, script_name, attachments) VALUES (?,?,?,?,?,?,?,?,?,?)domaintitleurlcontentpub_datesummaryindustry group_name script_name attachments[]z [push] ERR : z[push_to_searchdb] new=z skip=) sqlite3connect QUALITY_DBexecutecursorgetrowcount Exceptionprintcommitcloselen)records batch_labelrconncurn_newres $/root/gov_crawler/crawl_zjxc_tzgg.pypush_to_searchdbr-.ss ??: &DLL-. ++-CKK    E  : : KKxx$aeeGR&8!%%r:Jy"%quuZ'Rz2&lB(?}b)155+EG  S\\ !E : KKMJJL #E7&We1C0D EF L  : M!%%b/!2"QC8 9 9 :sC E  F)!FFcddl} |jt}|jd|jdtfj }|j |s tdy|jtd}|jd|jddx}} |D]}|\ }} } } } } }}}|jd | f}|j} |rF|jd || xsd dd | xsd | xsd dd| xsd |xsd |xsd |xsd |df |dz }nH|jd|| | xsd dd | xsd | xsd dd| xsd |xsd |xsd |xsd t| f |dz }|j|jdtf|j|jdtfjd}|j td|d|d|dy#t$r}td|Yd}~yd}~wwxYw#t$r}td| d|Yd}~d}~wwxYw#|j wxYw)Nrr zSELECT title, url, content, publish_date, summary, industry, group_name, script_name, attachments FROM crawl_results WHERE domain=? ORDER BY idz[sync] local read ERR: u[sync] 本地没有数据i,)timeoutzPRAGMA journal_mode=WALz'SELECT id FROM gov_raw WHERE page_url=?zUPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, industry=?, group_name=?, script_name=?, attachments=? WHERE id=?r i rzINSERT INTO gov_raw (title, page_url, content, publish_date, summary, industry, group_name, script_name, attachments, site_name, source_url) VALUES (?,?,?,?,?,?,?,?,?,?,?)z [sync] ERR rzINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z[sync] OK updated=z inserted=u DB共u条) rrrrDOMAINfetchallr#r r! SEARCH_DBfetchone SITE_NAMEr")rr'rowsr+dstupdinsr*rrrrrindgrpsnattr(rowtotals r,sync_to_serverr@Qs z* 12|| ^ I  (*    )* //)S/ 1CKK)*KK,-MC# 2AHI EE37Cb#++G#PC,,.C 2KK^B 8(.b7=VXZ^[^J_CI2rxRc!fNO1HCKKFgmWf%=x~2PWP][]_c`cOdCI2rxRiQTVW1HC! 2&   QL    Lyl[ddfghi  se:cU&s CDQ  's+,8 2 cU"QC011 2 sPAG9-4I"BH2A!I9 HHH I&H=7I=IIIctjdd|xsd}tjdd|}tjdd|jS)Nz$·| | |\u200b|\ufeffr u ^[•·]\s*z\s+ )resubstrip)ts r, clean_titlerGsG 6AGDA ~r1%A 66&#q ! ' ' ))cLtjd|xsd}|r[t|jdt|jdt|jd}}}|dd|dd|dStjd |xsd}|r|jd Sy) Nu+(\d{4})[年\-/.](\d{1,2})[月\-/.](\d{1,2})r r004d-02dz(\d{4})-(\d{2})-(\d{2})r)rCsearchintgroup)smymods r,normalize_daterWs @!'rJAqwwqz?C OS_q2C"S1S'** ,ag26Awwqz rHc ,tj|dddd}dttdtdt |d } t jt|td }|j|j}|jd s%td |d|jdgddfS|dd}g}d}tjd|} | rt| j!d}t#|d} | j%ddD]} | dj'} d| vrtjd| s2t)| j+d} | sP| j-dr| nt.| j-dr| nd| zz}|j1| |dd t3}g}|D]/}|d!|vr |j5|d!|j1|1|d|fS#t$r }td |d|gddfcYd}~Sd}~wwxYw)"Nr )pageNopageSizerOF ensure_ascii bulidstaticcolumnnull) parseTypewebIdtplSetIdpageTypetagIdeditTypepageId paramJson)paramsheadersr/successz [list] Pz API fail: messagerdatahtml ERR: z count="(\d+)"r0 html.parseraThrefrtz/art/z/art/\d{4}/art_[0-9a-f]+\.html$rEhttp/)rrrr)jsondumpsWEB_IDTPL_SETTAG_IDPAGE_IDrequestsrAPI_URLHEADERSraise_for_statusr!r rCrOrPrQrfind_allrErGget_text startswithBASE_URLappendsetadd)page param_jsonrjr*jror+itemscountrSsouprrrtrfull_urlseenuniqits r, fetch_listrstrJY^_J" F  LL" M  FFHuuY JtfKi0@/AB Cua< y  E E "D)AAGGAJ } -D ]]3T] * Hy  $  yy;TB AJJTJ23 ??624DOO\_L`Dfilpfp8q uX2FG H 5D D e9   E B  u 9  4&qc*+5!|s$A6G*/G** H3HHHc *  tj|td}|jd|_t |j d}d}|jd}|rt|j}|sntjd |j tj}|r=ttjd d|j!d j#}|s6|jd d di}|r|jdrt|d}d}tjd|j }|r|j!d }|j%dxs$|j%dxs|j%d} | -|j'tj(dD]} | } n| |d|gdSg} g} | j'dD]*} t+| t,s| jdxsdj#}tjd|tj.s d|vsd|vsf|j1dr|n4|j1drd|znt2|j1d r|nd |zzt| jd!"xs'| jd#xs|j5d d$}t7fd%| Ds| j9|d&| j9| -| D]&} | j;d'}|s|j=(| j'dd!(D]} | dj#}|j1d s5|j1d)s$|j1drJ|j1d*r\|j1drd|znt2|j1d r|nd |zz| d<| j'd+D]a}|jd,d}|s|j1dr*|j1drd|znt2|j1d r|nd |zz|d,<c| j?d-D]} | j=| j'tj(d./D]9} | j;d'xs| j;d0}|s*|j=;tA| jCtjd1dtjtj.z2tjd3dtjtj.z2tjd4d5tjd6dtjd7dtj2tjd8dtj2| rDt7fd9| D}|s.djEd:| D}jGd;z|ztItjdkr |s | s|d|| dS||| dS#t$r}td|d|Yd}~yd}~wwxYw)?Nri)rkr/zutf-8rqz [detail] rpr h1z(.*?)u_\s*新昌.*$|_\s*通知公告$r0metaname ArticleTitle)attrsru(发布时间[：:]\s*(\d{4}-\d{2}-\d{2})zdiv.ImporF_bd-mainzdiv.artic_mainzdiv.main_sectionz5ImporF_bd-main|artic_main|article-content|TRS_UEDITOR)class_rrrrrrrtz)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$z/document/downloadz /attachment/rvz//zhttps:rwTrudownloadc3.K|] }|dk(ywrN).0xabs_urls r, zparse_detail..s9qqx7*9s)rrprsz./ javascriptimgsrczadiv.share, div.ewm, div.qrcode, div.jzgov-qrcode-div, div.app-share, div.print, div.bshare-customu 扫一扫|分享到|打印本页)stringdivz]*>.*?)flagsz]*>.*?z]*>z

z

\s*

z\s*
.s:ah)+:sc3`K|]&}d|ddtj|dd(yw)z

rz

N)html_modescape)rrrs r,rzparse_detail..s3mcdahZr(//!F):T9UU]^ms,. <[^>]+>z "&&#9 .EH =qvvFA771:oo23tGW7X\`\k\kl~\G--rzz2i'j-k BG  28TVWW DI   c "  !S! f #**, 99A4 NRfjnRnsAEIsI"oof5dt_cOdHtOjrCNNORSvzY\_cYckdGqzzz56b!%% :KbtzzZ]_aObD9D99 T':;   Q    MM#  KKM   c  -By  ??3 4??4#8QWAXaeapapq}a~,0OOD,ADx[_[j[jkn[oSWux{uHAAfIB&~ggeR  s~~f-,/NN4,@(S.hY\YgYghkYlRUrux{r{F|CJ~ nnAB bjj1S&TU NN3  82>>%#8 KKM G++-.I3R"$$QSQUQU+VI12yrtt TI{E95I}b)4I7YbddSI4b)244PI :T::wwmhlmmH!((*T1H O=r*~66*b#i.A$3G&!ww//   0 0! => sxx     [  sI88 JJ__main__)&__doc__r~rCrrrxrorbs4rr urllib.parserr5rrr}r|r{rzrrrrdirnameabspath__file__rr3r1r-r@rGrWrrr__name__rrHr,rs  " $ $ Z   ! D0 E   WW\\"''//"''//(*CDFZ [   F2Ej* 0fZ]z7t zFrH