({j3dZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z dZ dZ dZdZdZd Zd Zd Zd Zej*j-d dZej0j3ej0j5ej0j7edZdZdddZdZ dZ!dZ"dZ#dZ$dZ%e&dk(re%yy)u_ 昌邑市人民政府 - 业务信息 (changyi.gov.cn/CYSXXGK/ZTZL/fdzdgkall.html -> 业务信息栏目) CMS: 潍坊市政府信息公开平台 (ELS/SYS-service API) 栏目树: GET http://www.changyi.gov.cn/sys-service/webcata/catas/108 (业务信息 cataid=4216, 6221条) 列表: POST http://www.changyi.gov.cn/els-service/article/{page}/13 body {"catas":["4216"],"dq":"108"} → data.contents[]: subject/fwdate/url/dwname 详情: /CYSXXGK/.../t{date}_{xxid}.htm 正文 div#ivs_content, meta ArticleTitle/PubDate 用法: python3 crawl_changyi_ywxx.py [--pages=N | N] [--sync] [--sync-only] N)ThreadPoolExecutor as_completedu"昌邑市人民政府-业务信息zhttp://www.changyi.gov.cnz7http://www.changyi.gov.cn/sys-service/webcata/catas/108z7http://www.changyi.gov.cn/els-service/article/{page}/134216108zwww.changyi.gov.cnu山东z 2020-01-01DB_PATHz/root/search.dbzquality_results.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zapplication/json;charset=UTF-8) User-Agentz Content-Typectjdd|xsd}tjdd|}tjdd|jS)Nz$·| | |\u200b|\ufeffu ^[•·]\s*z\s+ )resubstrip)ts '/root/gov_crawler/crawl_changyi_ywxx.py clean_titler'sG 6AGDA ~r1%A 66&#q ! ' ' ))c  tjtj|tt j tgtdd}|j|j }|jds$td|d|jdgd fS|jd i}t|jd xsd }g}|jd xsgD]}t|jd xsd}|jdxsdj}|jdxsdjdd} |r|so|j||| |jdxsdjd||fS#t $r} td|d| gd fcYd} ~ Sd} ~ wwxYw)u2POST 获取列表页，返回 (items, total_pages))page)catasdq)headersdatatimeoutsuccessz [list] Pz API fail: messagerr totalPagescontentssubjectr urlfwdateN dwname)titler!pub_dater$ ERR: )requestspostLIST_APIformatHEADERSjsondumpsCATA_IDDQraise_for_statusgetprintintrrappend Exception) rrjd total_pagesitemscr r!r"es r fetch_listr>-s MM(//t/4g#zzWIR*HISU W  FFHuuY JtfKi0@/AB Cq5L EE&" !%% -23 z"(b A!!%% "2"8b9G55<%2,,.CeeHo+224Sb9F# LL "55?0b779   k!!  4&qc*+1u s%B$F'C2F G#F=7G=Gc   tj|dtdid}|jd|_|j }d}tjd|}|rt|jd }|stjd |tj}|rbttjd d|jd j}tjd d|j}d}tjd |}|r"|jd jdd}|s)tjd|}|r|jd}dtjd|tj}|r|jd n8tjd|tj}|r|jd s8tjd|tj}|r|jd tjddtjtj ztjddtjtj zg}tj"dD] }|jd jt|jd} }tjd|tj s%d|j%vsd|j%vs|j'dr|n1|j'dr|nt(|j'dr|nd|zzt+fd|Dr|j-| xs|j/dd d! d"} tjd#| tjd$| tjd%d&tjd'dtjd(dtjtj ztjd)dtjtj z|rDt+fd*|D} | s.dj1d+|D} j3d,z| zt5tjd-dj} d.v}d/vxsd0v}| d1kr |s |s |s|d|gd2S|||d2S#t $r}td|d|Yd}~yd}~wwxYw)3Nr r)rrzutf-8z [detail] r'r z/(.*?)z^\S+--u [-_—]\s*.*$z*]*id="ivs_content"[^>]*>(.*?)\s*(?:)?zD]*class="Article_content[^"]*"[^>]*>(.*?)\s*(?:)?z-]*id="ContentRegion2"[^>]*>(.*?)z]*>.*?)flagsz]*>.*?z,]*href="([^"]+)"[^>]*>([^<]{0,120})z0\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z|et|dps)$z /downloadz/attachhttp///c3.K|] }|dk(ywr!N).0aabs_urls r zparse_detail..s9qqx7*9s)namer!c|jdj}|jds"|jds|jdr|jdS|jdr$|jdj|d|zS|jdj||jdr t|zStdz|zS) Nr@rC javascript#rrDzhttp:rE)groupr startswithreplaceBASE_URL)mhrefs rabs_linkzparse_detail..abs_linkswwqz! ??6 "dool&CtWZG[771:  ??4 771:%%dGdN; ;wwqz!!$4??3;O4kkU]`cUcfjUjkkrzhref="([^"]+)"z src="([^"]+)"z]*>z

z

\s*

zI]*class="[^"]*(share|ewm|qrcode|print|bshare)[^"]*"[^>]*>.*?z;]*class="[^"]*(article|content)[^"]*"[^>]*>\s*c3,K|] }|dv ywrGrH)rIrJ body_htmls rrLzparse_detail..s:ah)+:sc3`K|]&}d|ddtj|dd(yw)z

rNz

N)html_modescape)rIrJs rrLzparse_detail..s3mcdahZr(//!F):T9UU]^ms,. <[^>]+>zI'9=I{E95I}b)4Icegirz|z~z~BDBFBF{FGIUWY[dlnlplpsuswswlwxI :T::wwmhlmmH!((*T1H'Y*>I!|GId28TVWWyhW[ \\_  C5qc*+sAT T?$T::T?c0tjt}|jd|j }|jdd}|D]} |jd|j dd|j dd|j dd|j d d|j d d|j d d|j d d|j d d|j dd|j ddf ||j z }|j|jtd|dt||z |S#t$r,}td|j ddd|Yd}~@d}~wwxYw)NPRAGMA busy_timeout=300000aCREATE TABLE IF NOT EXISTS crawl_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_id INTEGER DEFAULT 0, title TEXT, url TEXT, content TEXT, publish_date TEXT, summary TEXT, domain TEXT, category TEXT DEFAULT '', content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, industry TEXT DEFAULT '', group_name TEXT DEFAULT '', script_name TEXT DEFAULT '', attachments TEXT DEFAULT '[]', UNIQUE(domain, url) )rzINSERT OR IGNORE INTO crawl_results (domain, title, url, content, publish_date, summary, industry, group_name, script_name, attachments) VALUES (?,?,?,?,?,?,?,?,?,?)domainr r%r!rcr&summaryindustry group_name script_namerd[]z [push] ERR : z[push_local] new=z skip=) sqlite3connect QUALITY_DBexecutecursorr2rowcountr6r3commitcloserq)recordsrconncurn_newr7r=s r push_localrsp ??: &DLL-. ++-CKK    E  : : KKxx$aeeGR&8!%%r:Jy"%quuZ'Rz2&lB(?}b)155+EG  S\\ !E : KKMJJL eWF3w<+=*> ?@ L  : M!%%b/!2"QC8 9 9 :sC E  F)!FFc tjt}|jd|jdtfj }|j |s tdytjtd}|jd|jddx}} |D]}|\ }}} } } } } }}|jd |f}|j} |rF|jd || xsd dd | xsd | xsd dd| xsd | xsd |xsd |xsd |df |dz }nH|jd||| xsd dd | xsd | xsd dd| xsd | xsd |xsd |xsd t|f |dz }|j|jdtf|j|jdtfjd}|j td|d|d|dy#t$r}td|Yd}~yd}~wwxYw#t$r}td|d|Yd}~d}~wwxYw#|j wxYw)Nr{zSELECT title, url, content, publish_date, summary, industry, group_name, script_name, attachments FROM crawl_results WHERE domain=? ORDER BY idz[sync] local read ERR: u[sync] 本地没有数据i,)rzPRAGMA journal_mode=WALrz'SELECT id FROM gov_raw WHERE page_url=?zUPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, industry=?, group_name=?, script_name=?, attachments=? WHERE id=?r i rr@zINSERT INTO gov_raw (title, page_url, content, publish_date, summary, industry, group_name, script_name, attachments, site_name, source_url) VALUES (?,?,?,?,?,?,?,?,?,?,?)z [sync] ERR rzINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z[sync] OK updated=z inserted=u DB共u条) rrrrDOMAINfetchallrr6r3rfetchone SITE_NAMEr)rrowsr=dstupdinsr7r%r!rcr&r}indgrpsnattrrowtotals rsync_to_serverrs z* 12|| ^ I  (*    )* //'3 /CKK)*KK,-MC# 2AHI EE37Cb#++G#PC,,.C 2KK^B 8(.b7=VXZ^[^J_CI2rxRc!fNO1HCKKFgmWf%=x~2PWP][]_c`cOdCI2rxRiQTVW1HC! 2&   QL    Lyl[ddfghi  se:cU&s CDQ  's+,8 2 cU"QC011 2 sPAG=14I &BH!6A!I = HHH! I*I;I II Icdtjv}d}dtjv}tjddD]O}|jdr t|j ddd}4|j sEt|}Qtd|d||r tyg}d}tdd D]W}|r||kDrnNt|\}}|r|}td |d t|d |d |sn|j|||k\sWntdt|d|d g} tt5} |D cic]} | jt | d| } } d} t#| D]>}| |} |j%}| dz } | dzdk(rtd| dt|dd|!td| ddd| dd| dgd}|dr'|dt&krtd|dd|dddt(t*|dxs| d| d|d |dxs| dt-j.d!d|d dd"dt0t2j4j7tjd|d#rt9j:|d#d$%nd&d' }| j=|A dddtd(t| | rHt?| t2j4j7tjd|r tyyy#t $rYwxYwcc} w#1swYxYw))Nz --sync-onlyz--syncr@z--pages==z[ChangyiYW] max_pages=z sync_only=ri'z[list] Pz: items=z (totalPages=)u[total] 列表共 u 条 (totalPages=) max_workersr!2u 详情 rEz...T)flushz [fail] r%(r r&rbz [cutoff] r rcr_rdF) ensure_asciir) r| site_namer%r!rcr&r}r~rrrdz[total] valid=) sysargvrSr4rn ValueErrorisdigitr3rranger>rqextendr MAX_WORKERSsubmitryrresultCUTOFFrrr rGROUPospathbasenamer-r.r5r) sync_only max_pagesdo_syncrJ all_itemsr:pgr;tpagesvalidexecutoritfut_mapdonefutdetrecs rmainrsa)II#(("G XXab\ << # Q 23 YY[AI "9+[ DEIKAuo  i "2 v  K HSZL fXQGH  <   s9~..? }A NO E  4JSTB8??<E;R?TT( CB**,C AIDbyA~ $qY(8 O=r*~66*b#i.A$3G#!ww// ryrrr__name__rHrrrs ? 0 & E D     **..$5 6 WW\\"''//"''//(*CDFZ [  D4  * >U]pD1EhCL zFr