{j=bdZddlZddlZddlZddlZddlZddlZddlm Z dZ dZ dZ dZ dZd Zd d d ed e ddZdZej$j'ej$j)ej$j+edZdZdZdZdZdZdZdZdZdZ e!dk(re yy)uO 黄冈市人民政府 - 政务公开栏目 (www.hg.gov.cn/zwgk/public/column/6636765?type=4&catId=7025318&action=list) CMS: 龙讯科技 lonsun 政务公开平台 列表: AJAX POST /zwgk/site/label/8888 data: labelName=publicInfoList, siteId=6792345, organId=6636765, pageSize=15, pageIndex=N, catId=7025318, type=4, action=list, length=50, dateFormat=yyyy年MM月dd日, file=/c2/xxgk/publicInfoList_newest_zc01 响应 HTML: tr.xxgk_nav_con > td.info > a.title (href+title), td.fbrq 日期 pageCount:234 (从响应 JSON 提取) 详情: https://www.hg.gov.cn/zwgk/public/6636765/{contentId}.html 正文 div.wzcon.j-fontContent (保留

/), meta ArticleTitle/PubDate 样本: https://www.hg.gov.cn/zwgk/public/6636765/1766719.html 用法: python3 crawl_hg_zf_hgzq.py [--pages=N | N] [--sync] N) BeautifulSoupu黄冈市-回应关切zhttps://www.hg.gov.cnz*https://www.hg.gov.cn/zwgk/site/label/8888702531866367656792345zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36XMLHttpRequestz)https://www.hg.gov.cn/zwgk/public/column/z?type=4&catId=z &action=list)z User-AgentzX-Requested-WithRefererz 2023-01-01zquality_results.dbz/root/search.dbz www.hg.gov.cnc0ddl}|jt}|jd|j }|jdd}|D]} |jd|j dd|j dd|j dd|j d d|j d d|j d d|j d d|j d d|j dd|j ddf ||j z }|j|jtd|dt||z |S#t$r,}td|j ddd|Yd}~@d}~wwxYw)NrPRAGMA busy_timeout=300000aCREATE TABLE IF NOT EXISTS crawl_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_id INTEGER DEFAULT 0, title TEXT, url TEXT, content TEXT, publish_date TEXT, summary TEXT, domain TEXT, category TEXT DEFAULT '', content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, industry TEXT DEFAULT '', group_name TEXT DEFAULT '', script_name TEXT DEFAULT '', attachments TEXT DEFAULT '[]', UNIQUE(domain, url) )zINSERT OR IGNORE INTO crawl_results (domain, title, url, content, publish_date, summary, industry, group_name, script_name, attachments) VALUES (?,?,?,?,?,?,?,?,?,?)domaintitleurlcontentpub_datesummaryindustry group_name script_name attachments[]z [push] ERR : z[push_to_searchdb] new=z skip=) sqlite3connect QUALITY_DBexecutecursorgetrowcount Exceptionprintcommitcloselen)records batch_labelrconncurn_newres %/root/gov_crawler/crawl_hg_zf_hgzq.pypush_to_searchdbr,+ss ??: &DLL-. ++-CKK    E  : : KKxx$aeeGR&8!%%r:Jy"%quuZ'Rz2&lB(?}b)155+EG  S\\ !E : KKMJJL #E7&We1C0D EF L  : M!%%b/!2"QC8 9 9 :sC E  F)!FFcddl} |jt}|jd|jdtfj }|j |s tdy|jtd}|jd|jddx}} |D]}|\ }} } } } } }}}|jd | f}|j} |rF|jd || xsd dd | xsd | xsd dd| xsd |xsd |xsd |xsd |df |dz }nH|jd|| | xsd dd | xsd | xsd dd| xsd |xsd |xsd |xsd t| f |dz }|j|jdtf|j|jdtfjd}|j td|d|d|dy#t$r}td|Yd}~yd}~wwxYw#t$r}td| d|Yd}~d}~wwxYw#|j wxYw)Nrr zSELECT title, url, content, publish_date, summary, industry, group_name, script_name, attachments FROM crawl_results WHERE domain=? ORDER BY idz[sync] local read ERR: u[sync] 本地没有数据i,)timeoutzPRAGMA journal_mode=WALz'SELECT id FROM gov_raw WHERE page_url=?zUPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, industry=?, group_name=?, script_name=?, attachments=? WHERE id=?r i rzINSERT INTO gov_raw (title, page_url, content, publish_date, summary, industry, group_name, script_name, attachments, site_name, source_url) VALUES (?,?,?,?,?,?,?,?,?,?,?)z [sync] ERR rzINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z[sync] OK updated=z inserted=u DB共u条) rrrrDOMAINfetchallr"rr SEARCH_DBfetchone SITE_NAMEr!)rr&rowsr*dstupdinsr)r rrrrindgrpsnattr'rowtotals r+sync_to_serverr?Ns z* 12|| ^ I  (*    )* //)S/ 1CKK)*KK,-MC# 2AHI EE37Cb#++G#PC,,.C 2KK^B 8(.b7=VXZ^[^J_CI2rxRc!fNO1HCKKFgmWf%=x~2PWP][]_c`cOdCI2rxRiQTVW1HC! 2&   QL    Lyl[ddfghi  se:cU&s CDQ  's+,8 2 cU"QC011 2 sPAG9-4I"BH2A!I9 HHH I&H=7I=IIIctjdd|xsd}tjdd|}tjdd|jS)Nz$·| | |\u200b|\ufeffr u ^[•·]\s*z\s+ )resubstrip)ts r+ clean_titlerFsG 6AGDA ~r1%A 66&#q ! ' ' ))cLtjd|xsd}|r[t|jdt|jdt|jd}}}|dd|dd|dStjd |xsd}|r|jd Sy) Nu+(\d{4})[年\-/.](\d{1,2})[月\-/.](\d{1,2})r r/04d-02dz(\d{4})-(\d{2})-(\d{2})r)rBsearchintgroup)smymods r+normalize_daterVs @!'rJAqwwqz?C OS_q2C"S1S'** ,ag26Awwqz rGc zdttdt|tdddddd } t j t |td }|jd |_ |j}g}d}tjd|}|rt|j!d}t#|d}|j%ddD]} | j'dd} | s| dj)} tjd| } | sFt| vr dtd| vr[t+| j-d} | syd}| j%d }|D]4}d!|j/d"xsgvst1|j-}n|s}| j-}tjd#|}|rUt|j!dd$d%t|j!d&d'd%t|j!d(d'}| j3d)r| nt4| j3dr| nd| zz}|j7| ||d*t9}g}|D]/}|d+|vr |j;|d+|j7|1|d|fS#t$r }td |d |gddfcYd}~Sd}~wwxYw),uPOST AJAX 接口拿列表页publicInfoList154list50uyyyy年MM月dd日z#/c2/xxgk/publicInfoList_newest_zc01) labelNamesiteIdorganIdpageSize pageIndexcatIdtypeactionlength dateFormatfile)dataheadersr.utf-8z [list] P ERR: FrNzpageCount:(\d+)r/ html.parsertr xxgk_nav_conclass_aThrefrtz/public/\d+/(\d+)\.html$z/public//rDr tdfbrqclassu"(\d{4})年(\d{1,2})月(\d{1,2})日rKrLrIrMrJhttp)r rrr)SITE_IDORGAN_IDstrCAT_IDrequestspostAJAX_URLHEADERSraise_for_statusencodingtextrr rBrNrOrPrfind_allfindrDrFget_textrrV startswithBASE_URLappendsetadd)pagerir)rr*items page_countrRsouprnrrrtm2r rtdsrwtxtm3full_urlseenuniqits r+ fetch_listrs&Y)5 D MM(w K  vv EJ $d+A_ } -DmmDm8N GGCdG # y  YY2D 9  4 hxj$:$$F AJJTJ23 kk$ B"&&//R0)"++-8 ++-C@#FB!"((1+.s31S!5Ec4J!CPRPXPXYZP[L\]`Kab??624DOO\_L`Dfilpfp8q uX8LM5N8 5D D e9   E B  z !!Y  4&qc*+5!|sAJ J:J5/J:5J:c  tj|td}|jd|_t |j d}d}|jdd d i }|r|jd rt|d }|s,|jd }|rt|j}|sntjd|j tj}|r=ttjdd|j!dj#}d}|jdd di }|r'|jd r|d j#dd}|s3tjd|j }|r|j!d}|j%dxs$|j%dxs|j%d}|-|j'tj(dD]} | }n||d|gdSg} |j'ddD]} | dj#} tjd| tj*sd| vsd | vsd!| j-vsV| j/d"r| n4| j/d#rd$| znt0| j/d%r| nd%| zzt| jd&xs| j3d%d'} t5fd(| Dr| j7| d)|j%d*xs$|j%d+xs|j%d,}|r|j'ddD]} | dj#} | s| j/d"r| n4| j/d#rd$| znt0| j/d%r| nd%| zzt| jd&xs| j3d%d'} t5fd-| Dr| j7| d)|j'ddD]g} | dj#} tjd| tj*s d| vsd | vsD| j9d.}|sX|j;i|j'ddD]} | dj#} | j/d%s5| j/d/s$| j/d"rJ| j/d0r\| j/d#rd$| znt0| j/d%r| nd%| zz| d<|j'd1D]a}|jd2d}|s|j/d"r*|j/d#rd$|znt0|j/d%r|nd%|zz|d2<c|j=d3D]} | j;|j'tj(d45D]9} | j9d.xs| j9d6}|s*|j;;t?|jAtjd7dtjtj*z8tjd9dtjtj*z8tjd:d;tjddtj8| rDt5fd?| D}|s.djCd@| D}jEdAz|ztGtjdBdj#}dCv}|dDkr |s | s|d|| dS||| dS#t$r}td|d|Yd}~yd}~wwxYw)ENrh)rjr.rkrmz [detail] rlr metaname ArticleTitle)attrsrh1z(.*?)u_黄冈市人民政府$r/PubDate u(发布时间[：:]\s*(\d{4}-\d{2}-\d{2})z div.wzconzdiv.j-fontContentzdiv.ls-article-infoz/wzcon|j-fontContent|article-content|TRS_UEDITORrpr rrrrrTrsrtz)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$z /attachment/z/fj/downloadrzz//zhttps:rurvc3.K|] }|dk(ywrN.0xabs_urls r+ zparse_detail..s9qqx7*9)rrzdiv.attachmentzdiv.attachmentszdiv.fjc3.K|] }|dk(ywrrrs r+rzparse_detail..s=11U8w.=rpz./ javascriptimgsrcz`div.share, div.ewm, div.qrcode, div.jzgov-qrcode-div, div.app-share, div.print, div.wzy_positionu 扫一扫|分享到|打印本页)stringdivz]*>.*?)flagsz]*>.*?z]*>z

z

\s*

z\s*
.3s:ah)+:sc3`K|]&}d|ddtj|dd(yw)z

rz

N)html_modescape)rrrs r+rzparse_detail..5s3mcdahZr(//!F):T9UU]^ms,. <[^>]+>zQ.R|VZVeVef{V|G--rzz2c'd-e BG  28TVWW D   c  -<y  99A4 NR`dhRhlrvzlzIMQMWMWMYY"oof5dt_cOdHtOjrCNNORSvzY\_cYckdGqzzz56M$**S/":MD9D99 T':; </0sDOODU4VsZ^ZiZijrZsH""3T"2 @AV9??$D"&//&"9$SWSbSbcgSh4nvCGCRCRSVCWz~]`cg]goh"1::D:#9:Qdjjob>Q===KKg >?  @  c  -y  99A4 NR`dhRhlrvzlz c"A   c  -By  ??3 4??4#8QWAXaeapapq}a~,0OOD,ADx[_[j[jkn[oSWux{uHAAfIB&~ggeR  s~~f-,/NN4,@(S.hY\YgYghkYlRUrux{r{F|CJ~ nnA bjj1S&TU NN3  82>>%#8 KKM G++-.I3R"$$QSQUQU+VI12yrtt TI{E95I}b)4I7YbddSI4b)244PI :T::wwmhlmmH!((*T1H O=r*~66*b#i.A$3G&!ww//  sxx     a  sJ JJ__main__)"__doc__rrBrrrhtmlrbs4rr4rrr~r|r{rrrrdirnameabspath__file__rr2r0r,r?rFrVrrr__name__rrGr+rs   $ " 7   D(:8*NSYRZZfg   WW\\"''//"''//(*CDFZ [   F2Ej* @"Fe]P;| zFrG