3zj;HdZddlZddlZddlZddlZddlZddlZddlm Z dZ dZ dZ ddd d d Z d Zd Zej j#ej j%ej j'edZdZdZdZdZdZdZdZdZedk(reyy)u 枣庄国家高新技术产业开发区 -- 生态环境分局 -- 建设项目环境影响评价 列表: POST http://www.zzctp.gov.cn/irs/front/search (TRS 政务公开 IRS 检索 API) body: code=18e334699d2, dataTypeId=2857, filters=[{aggrName:f_2024327849703, aggrValues:建设项目环境影响评价}, {aggrName:f_2024327355010, aggrValues:生态环境分局}] pageNo/pageSize=20, orderBy=time 详情: meta ArticleTitle/PubDate + div.view.TRS_UEDITOR 正文 + div.attachment 附件 样本: http://www.zzctp.gov.cn/zwgk/xxgkml/sgdw/sthjfj/202608/t20260807_2314926.html 用法: python3 crawl_zzctp_hpgs.py [--pages=N | N] N) BeautifulSoupu.枣庄高新区-建设项目环境影响评价http://www.zzctp.gov.cnz(http://www.zzctp.gov.cn/irs/front/searchzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zBhttp://www.zzctp.gov.cn/list_6548.html?siteid=179&classinfoid=2627zapplication/jsonXMLHttpRequest)z User-AgentRefererz Content-TypezX-Requested-Withz 2023-01-01zquality_results.dbz/root/search.dbc0ddl}|jt}|jd|j }|jdd}|D]} |jd|j dd|j dd|j d d|j d d|j d d|j d d|j d d|j dd|j dd|j ddf ||j z }|j|jtd|dt||z |S#t$r,}td|j d dd|Yd}~@d}~wwxYw)u[写入本地 quality_results.db（domain 隔离），供 sync_to_server 同步到 search.dbrNPRAGMA busy_timeout=300000aCREATE TABLE IF NOT EXISTS crawl_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_id INTEGER DEFAULT 0, title TEXT, url TEXT, content TEXT, publish_date TEXT, summary TEXT, domain TEXT, category TEXT DEFAULT '', content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, industry TEXT DEFAULT '', group_name TEXT DEFAULT '', script_name TEXT DEFAULT '', attachments TEXT DEFAULT '[]', UNIQUE(domain, url) )zINSERT OR IGNORE INTO crawl_results (domain, title, url, content, publish_date, summary, industry, group_name, script_name, attachments) VALUES (?,?,?,?,?,?,?,?,?,?)domaintitleurlcontentpub_datesummaryindustry group_name script_name attachments[]z [push] ERR : z[push_to_searchdb] new=z skip=) sqlite3connect QUALITY_DBexecutecursorgetrowcount Exceptionprintcommitcloselen)records batch_labelrconncurn_newres %/root/gov_crawler/crawl_zzctp_hpgs.pypush_to_searchdbr+&ss ??: &DLL-. ++-CKK    E  : : KKxx$aeeGR&8!%%r:Jy"%quuZ'Rz2&lB(?}b)155+EG  S\\ !E : KKMJJL #E7&We1C0D EF L  : M!%%b/!2"QC8 9 9 :sC E  F)!FFcddl} |jt}|jd|jdtfj }|j |s tdy|jtd}|jd |jddx}} |D]}|\ }} } } } } }}}|jd | f}|j} |rF|jd || xsd dd | xsd | xsd dd| xsd |xsd |xsd |xsd|df |dz }nH|jd|| | xsd dd | xsd | xsd dd| xsd |xsd |xsd |xsdt| f |dz }|j|jdtf|j|jdtfjd}|j td|d|d|dy#t$r}td|Yd}~yd}~wwxYw#t$r}td| d|Yd}~d}~wwxYw#|j wxYw)un把本地 quality_results.db 该 domain 的记录同步到 search.db（UPDATE-or-INSERT 保 id + FTS 增量）rNr zSELECT title, url, content, publish_date, summary, industry, group_name, script_name, attachments FROM crawl_results WHERE domain=? ORDER BY idz[sync] local read ERR: u[sync] 本地没有数据i,)timeoutzPRAGMA journal_mode=WALz'SELECT id FROM gov_raw WHERE page_url=?zUPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, industry=?, group_name=?, script_name=?, attachments=? WHERE id=?r i rzINSERT INTO gov_raw (title, page_url, content, publish_date, summary, industry, group_name, script_name, attachments, site_name, source_url) VALUES (?,?,?,?,?,?,?,?,?,?,?)z [sync] ERR rzINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z[sync] OK updated=z inserted=u DB共u条) rrrrDOMAINfetchallr!rr SEARCH_DBfetchone SITE_NAMEr )rr%rowsr)dstupdinsr(r r rrrindgrpsnattr&rowtotals r*sync_to_serverr>Js z* 12|| ^ I  (*    )* //)S/ 1CKK)*KK,-MC# 2AHI EE37Cb#++G#PC,,.C 2KK^B 8(.b7=VXZ^[^J_CI2rxRc!fNO1HCKKFgmWf%=x~2PWP][]_c`cOdCI2rxRiQTVW1HC! 2&   QL    Lyl[ddfghi  se:cU&s CDU  's+,: 2 cU"QC011 2 sPAG9-4I"BH2A!I9 HHH I&H=7I=IIIctjdd|}tjdd|}tjdd|jS)Nz$·| | |\u200b|\ufeffr u ^[•·]\s*z\s+ )resubstrip)ts r* clean_titlerEsC 6A>A ~r1%A 66&#q ! ' ' ))cddddddddddddd ddgd d dgd |td d} tjttt j |djdd}|j|j }|jdxsi}|jdxsi}|jdxs|jdxsi}|jdxsg}g}|D]} | jddk7r| jdxsi} | jds?|jttjddt| jdd| jdxsdt| jd xsdddd|jdxsd } |jd xsd!} || | fS#t$r } t!d"|d#| gd d!fcYd} ~ Sd} ~ wwxYw)$Nr 18e334699d22857f_2024327849703u建设项目环境影响评价1)aggrName aggrValuesfilterIdf_2024327355010u生态环境分局ALLrtimeall) appendixType beginDateTimecodecodes configCode dataTypeId endDateTimefilters granularityisSearchForcedhistorySearchWordsorderBypageNopageSizesearchByF ensure_asciiutf-8)headersdatar-rgmiddlepager listAndBoxtypeDATAr zr )r r rr= pageCountr.z [list] page  ERR: ) PAGE_SIZErequestspost SEARCH_URLHEADERSjsondumpsencoderaise_for_statusrappendrErArBstrrr)pagebodyr(drgrhrilstitemsitddr= page_countr)s r* fetch_listrsR2VB*:Zhk l*:N\_ ` ?@Tye D MM*g#zz$UCJJ7S]_ a  FFHuuV}"(#)r!>VZZ%8>Bjj&," Bvvf~'%2B66'? LL$RVVIr3rvvgr?R;S%TUvve}*v 4"5cr:   '"'aYY{+0q eZ''  tfF1#./1axsF=G!! H *H?H H c  tj|td}|jd|_t |j d}d}|jd d d i }|r|jd rt|d }|s,|jd}|rt|j}|srtjd|j tj}|rA|jdj!}tj"dd|}t|}d}|jd d di }|r'|jd r|d j!dd}|s3tjd|j }|r|jd}|j%dxs$|j%dxs|j%d} | -|j'tj(dD]} | } n| |d|gdSg} | j+d} |j%d} | r| j'ddD]}|d j!}tjd!|tj,s/) + 附件(div.attachment)re)rfr-rdz html.parserz [detail] roNr metaname ArticleTitle)attrsrh1z(.*?)r.u+^枣庄国家高新技术产业开发区--?PubDatermz(\d{4}-\d{2}-\d{2})zdiv.view.TRS_UEDITORzdiv.viewzdiv.TRS_UEDITOR TRS_UEDITOR)class_r rrrdivzdiv.attachmentaT)hrefrz)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$)rC/)rr httprc3.K|] }|dk(yw)r N).0xabs_urls r* zparse_detail..s9qqx7*9spz]*>.*?)flagsz]*>.*?z]*>z

imgsrcz

\s*

z\s*
rz

N)html_modescape)rrs r*rzparse_detail..s8[PQl1U8*Bxqy'A&B( K[s,. <[^>]+>z) max_pagesrvalidpgrr=rrdetrecs r*mainr,sI XXab\ << # Q 23 YY[AI  { +, EArl i #-b> uj RDU GE7+j\Z[  B*~"Z.6"9 BzN#31R["5E4FGHr%y)C{ "W+cr"2!345 "7 :gij &W4G%yy> O=r*~66*b#i.A$3G')yy1TVYZaVbVqfhipfq'rGxz&!ww//  sxx     Q  sI33 J?Jzwww.zzctp.gov.cn__main__)__doc__rqrurArrhtmlrbs4rr3BASErsrtrprrrdirnameabspath__file__rr1r+r>rErrrr/__name__rrFr*rs   <   7 DS&(     WW\\"''//"''//(*CDFZ [  !H5Er* &Rw]t2j  zFrF