({j[4dZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z dZ dZ dZdZdZd Zd Zej&j)d d Zej,j/ej,j1ej,j3ed ZdZdZdddddZdZdZ dZ!dZ"dZ#dZ$e%dk(re$yy)uj 达州高新技术产业园区 - 环境保护 (www.dzgxq.gov.cn 注意: dzgxq=达州高新非德州!, 四川) 栏目: 政府信息公开 > 环境保护 /xxgk-list-huanjingbaohu.html (catid=53, 257条/13页, 20条/页) CMS: 自建 PHP (xxgk 信息公开系统, API 动态加载) 列表: POST http://www.dzgxq.gov.cn/api-ajax_list-{page}.html body (PHP风格序列化!): ajax_type[]=39_xxgk&ajax_type[]=53&ajax_type[]=39&ajax_type[]=xxgk&ajax_type[]=Y-m-d&ajax_type[]=40&ajax_type[]=20&ajax_type[7][]=inputtime+DESC&ajax_type[7][]=displayorder+DESC&ajax_type[7][]=is_top+DESC&is_ds=1 响应 JSON: data[] (id/catid/title/url/inputtime/description) 详情: /xxgk-show-{id}.html 标题 div.cont_tit > h2, 日期 div.msg "发布日期：YYYY-MM-DD", 正文 div.contM (含表格/附件) 用法: python3 crawl_dzgxq_hjbh.py [--pages=N | N] [--sync] [--sync-only] N)ThreadPoolExecutor as_completedu达州高新区-环境保护zhttp://www.dzgxq.gov.cnz1http://www.dzgxq.gov.cn/api-ajax_list-{page}.html53zwww.dzgxq.gov.cnu四川z 2020-01-01DB_PATHz/root/search.dbzquality_results.dbzajax_type[]=39_xxgk&ajax_type[]=53&ajax_type[]=39&ajax_type[]=xxgk&ajax_type[]=Y-m-d&ajax_type[]=40&ajax_type[]=20&ajax_type[7][]=inputtime+DESC&ajax_type[7][]=displayorder+DESC&ajax_type[7][]=is_top+DESC&is_ds=1zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z!application/x-www-form-urlencodedXMLHttpRequestz4http://www.dzgxq.gov.cn/xxgk-list-huanjingbaohu.html) User-Agentz Content-TypezX-Requested-WithRefererctjdd|xsd}tjdd|}tjdd|jS)Nz$·| | |\u200b|\ufeffu ^[•·]\s*z\s+ )resubstrip)ts %/root/gov_crawler/crawl_dzgxq_hjbh.py clean_titler.sG 6AGDA ~r1%A 66&#q ! ' ' ))c   tjtj|tt d}|j |j}d|vr!td|dt|ddgdfSt|jd xsd}g}|jdxsgD]}t|jd xsd }|jd xsd j}|jd xsd jdd}|r|so|j|||t|jdxsd ddd||fS#t$r} td|d| gdfcYd} ~ Sd} ~ wwxYw)N)page)dataheaderstimeoutrz [list] Pz fail: drtotaltitler url inputtime description)rrpub_dater! ERR: )requestspostLIST_APIformat LIST_DATAHEADERSraise_for_statusjsonprintstrintgetrrappend Exception) rrjritemscrrres r fetch_listr84ss MM(//t/49g_a b  FFH ? JtfGCF4CL>: ;q5LAEE'N'a(v$" Ag 4"5E55<%2,,.C{+1r88:3B?I LL%*155+?+E2FtL   e|  4&qc*+1u s%A9E8F>Fc   tj|dtdid}|jd|_|j }d}tjd|tj}|r/ttjd d|jd }|sdtjd |tj}|r=ttjd d|jd j}d}tjd |tj}|r8tjd|jd }|r|jd}|s)tjd|}|r|jd}dtjd|tj}|r|jd s8tjd|tj}|r|jd tjddtjtj ztjddtjtj zg}tj"dD]}|jd jt|jd} } tjd| tj sd| vsd| j%vsy| j'dr| n1| j'dr| nt(| j'dr| nd| zzt+fd|Dr|j-| xs&| j/ddj/ddd d!} tjd"| tjd#| tjd$d%tjd&dtjd'dtjtj z|rDt+fd(|D} | s.dj1d)|D} j3d*z| zt5tjd dj}d+v}d,vxsd-v}|d.kr |s |s |s|d|gd/S|||d/S#t $r}td|d|Yd}~yd}~wwxYw)0Nr r)rrzutf-8z [detail] r$r z,
\s*]*>(.*?)<[^>]+>z(.*?)z^[^-]+-z
(.*?)
z(\d{4})-(\d{2})-(\d{2})rz1
]*>(.*?)
\s*(?:
)?z
]*>(.*)z]*>.*?)flagsz]*>.*?z,]*href="([^"]+)"[^>]*>([^<]{0,120})z0\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z|et|dps)$z /uploadfile/z /attachment/http///c3.K|] }|dk(ywrN).0aabs_urls r zparse_detail..s9qqx7*9s?)namerc|jdj}|jds"|jds|jdr|jdS|jdr$|jdj|d|zS|jdj||jdr t|zStdz|zS) Nr;r> javascript#rr?zhttp:r@)groupr startswithreplaceBASE_URL)mhrefs rabs_linkzparse_detail..abs_linkswwqz! ??6 "dool&CtWZG[771:  ??4 771:%%dGdN; ;wwqz!!$4??3;O4kkU]`cUcfjUjkkrzhref="([^"]+)"z src="([^"]+)"z]*>z

z

\s*

zI]*class="[^"]*(share|ewm|qrcode|print|bshare)[^"]*"[^>]*>.*?
c3,K|] }|dv ywrBrC)rDrE body_htmls rrGzparse_detail..s:ah)+:sc3`K|]&}d|ddtj|dd(yw)z

rJz

N)html_modescape)rDrEs rrGzparse_detail..s3mcdahZr(//!F):T9UU]^ms,. z?  II-y")) D z2qwwqz7G7G7I JKEH 19biiHA II0!''!* = wwqzH  II0) < wwqzHI F SUS\S\]AGGAJ  II5y")) L  I3R"$$QSQUQU+VI12yrtt TI D [[H) TaWWQZ%%'QWWQZ)@d 99H$PRPTPT UYgkoYotBFJFPFPFRtR"oof5dDOOTXDYD_gswtCtCDGtHkoNQTXNX`YG9D99 T%NTZZ_R-@-F-Fs-KA-NW^_` al((I>I'9=I{E95I}b)4Icegirz|z~z~BDBFBF{FGI :T::wwmhlmmH!((*T1H'Y*>I!|GId28TVWWyhW[ \\S  C5qc*+sAS S6S11S6c0tjt}|jd|j }|jdd}|D]} |jd|j dd|j dd|j dd|j d d|j d d|j d d|j d d|j d d|j dd|j ddf ||j z }|j|jtd|dt||z |S#t$r,}td|j ddd|Yd}~@d}~wwxYw)NPRAGMA busy_timeout=300000aCREATE TABLE IF NOT EXISTS crawl_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_id INTEGER DEFAULT 0, title TEXT, url TEXT, content TEXT, publish_date TEXT, summary TEXT, domain TEXT, category TEXT DEFAULT '', content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, industry TEXT DEFAULT '', group_name TEXT DEFAULT '', script_name TEXT DEFAULT '', attachments TEXT DEFAULT '[]', UNIQUE(domain, url) )rzINSERT OR IGNORE INTO crawl_results (domain, title, url, content, publish_date, summary, industry, group_name, script_name, attachments) VALUES (?,?,?,?,?,?,?,?,?,?)domainr rrr^r#summaryindustry group_name script_namer_[]z [push] ERR : z[push_local] new=z skip=) sqlite3connect QUALITY_DBexecutecursorr0rowcountr2r-commitcloserl)recordsr|conncurn_newr3r7s r push_localrsp ??: &DLL-. ++-CKK    E  : : KKxx$aeeGR&8!%%r:Jy"%quuZ'Rz2&lB(?}b)155+EG  S\\ !E : KKMJJL eWF3w<+=*> ?@ L  : M!%%b/!2"QC8 9 9 :sC E  F)!FFc tjt}|jd|jdtfj }|j |s tdytjtd}|jd|jddx}} |D]}|\ }}} } } } } }}|jd |f}|j} |rF|jd || xsd dd | xsd | xsd dd| xsd | xsd |xsd |xsd |df |dz }nH|jd||| xsd dd | xsd | xsd dd| xsd | xsd |xsd |xsd t|f |dz }|j|jdtf|j|jdtfjd}|j td|d|d|dy#t$r}td|Yd}~yd}~wwxYw#t$r}td|d|Yd}~d}~wwxYw#|j wxYw)NrwzSELECT title, url, content, publish_date, summary, industry, group_name, script_name, attachments FROM crawl_results WHERE domain=? ORDER BY idz[sync] local read ERR: u[sync] 本地没有数据i,)rzPRAGMA journal_mode=WALrz'SELECT id FROM gov_raw WHERE page_url=?zUPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, industry=?, group_name=?, script_name=?, attachments=? WHERE id=?r i r}r;zINSERT INTO gov_raw (title, page_url, content, publish_date, summary, industry, group_name, script_name, attachments, site_name, source_url) VALUES (?,?,?,?,?,?,?,?,?,?,?)z [sync] ERR r~zINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z[sync] OK updated=z inserted=u DB共u条) rrrrDOMAINfetchallrr2r-rfetchone SITE_NAMEr)rrowsr7dstupdinsr3rrr^r#ryindgrpsnattrrowrs rsync_to_serverrs z* 12|| ^ I  (*    )* //'3 /CKK)*KK,-MC# 2AHI EE37Cb#++G#PC,,.C 2KK^B 8(.b7=VXZ^[^J_CI2rxRc!fNO1HCKKFgmWf%=x~2PWP][]_c`cOdCI2rxRiQTVW1HC! 2&   QL    Lyl[ddfghi  se:cU&s CDQ  's+,8 2 cU"QC011 2 sPAG=14I &BH!6A!I = HHH! I*I;I II Icdtjv}d}dtjv}tjddD]O}|jdr t|j ddd}4|j sEt|}Qtd|d||r tyg}d}tdd D]b}|r||kDrnYt|\}}td |d t|d |d |sn)|j||P|sSt||k\sbntdt|d |d g}tt5} |D cic]} | jt | d| } } d} t#| D]R} | | } | j%}| dz } | dzdk(rtd| dt|dd|!td| ddd| dd| dgd}|dr'|dt&krtd|dd|dddt(t*|dxs| d| d|d|dxs| dt-j.d d|ddd!xs| j1d"ddt2t4j6j9tjd|d#rt;j<|d#d$%nd&d' }|j?|U dddtd(t||rHtA|t4j6j9tjd|r tyyy#t $rYwxYwcc} w#1swYxYw))Nz --sync-onlyz--syncr;z--pages==z[DZGXQ] max_pages=z sync_only=ri'z[list] Pr~u 条 (total=)u[total] 列表共 ) max_workersr2u 详情 r@z...T)flushz [fail] r(r r#r]z [cutoff] r r^r:r"r!r_F) ensure_asciir}) rx site_namerrr^r#ryrzr{r|r_z[total] valid=)!sysargvrOr/ri ValueErrorisdigitr-rranger8rlextendr MAX_WORKERSsubmitrurresultCUTOFFrrrrr0GROUPospathbasenamer,dumpsr1r) sync_only max_pagesdo_syncrE all_itemsrpgr5validexecutoritfut_mapdonefutdetrecs rmainrsv)II#(("G XXab\ << # Q 23 YY[AI ykYK @AI EAuo  i !"~ u Bs5zl,ugQ?@   3y>U+B   s9~.l5' CD E  4JSTB8??<E;R?TT( CB**,C AIDbyA~ $qY(8 O=r*~66*b#i.A$3Gd266R_acKd#!ww// rs  ? * $ >    **..$5 6 WW\\"''//"''//(*CDFZ [  3 D7(E  * 8O]dD1EhAH zFr