ujs2dZddlZddlZddlZddlZddlZddlZddl Z ddl m Z dZ dZdZdZdZd Zd d d d ZdZeej,D]k\ZZej3dr eej7dddZ4edk(s:edzeej,ksS eej,edzZmedk\rendZdZdZ dZ!dZ"dZ#dZ$dZ%dZ&e'dk(re&yy#e $rdZ YwxYw#e$rYwxYw#e$rYwxYw)u crawl_xgsgkml_sp - 孝感市政府信息公开-行政许可审批 http://gkml.xiaogan.gov.cn/c/www/sp.jhtml 列表: li 块, href=/c/www/sp/ID.jhtml, 日期 YYYY-MM-DD 分页: sp_{N}.jhtml 详情: div.view N) BeautifulSoupzhttp://gkml.xiaogan.gov.cnz)http://gkml.xiaogan.gov.cn/c/www/sp.jhtmlu.孝感市政府信息公开-行政许可审批u湖北zcrawl_xgsgkml_sp.pyz/mnt/data/search.dbzkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagez--pages==z--pagesc|sytj|}|jddjddjddjdd}tjdd|}|j S)Nu​u‌u‍u\s+ )html_libunescapereplaceresubstrip)ts %/root/gov_crawler/crawl_xgsgkml_sp.py clean_titler5so !A (B''"5==hKSST\^`aA vsAA 779c|dk(rt}n-tjddd}|dzt|zdz}|j|td}d|_|j S) Nr.r_z.jhtmlheaderstimeoututf-8)LIST_URLrsplitstrgetHEADERSencodingtext)sessionpageurlbasers r fetch_listr)>s` qysA&q)Sj3t9$x/ C" 5AAJ 66Mrc*g}tjd|tjtjzD]V}|j d}tj d|tjtjz}|sN|j d}tj dd|j d}tj|j}tj dd|}|rt|d krd |vrtj d |}|r|j d nd}tjjt|} | jd rd| z} |j!| t#||fY|S)Nz]*>(.*?)rz+]+href="([^"]+\.jhtml)"[^>]*>(.*?)<[^>]+>rr r z/sp/z(20\d{2})-(\d{2})-(\d{2})r//http:)rfinditerSIgroupsearchrr r rlenurllibparseurljoinr startswithappendr) html_textitemsmliamhreftxtdmdateabs_urls r parse_listrEJsA E [[/BDD244K H8 WWQZ YYEr244RTRVRV; W xx{ffZRXXa[1$**,ffVS#&c#hl     YY3R 8 rxx{b,,&&x6   d #'G g{3/67'8( Lrcd}d}d}trt|d}|jddtjdtji}|r|j drt |d}|jddtjdtji}|rM|j dr<|dj}tjd |} | r| jd }|jd tjd tj } | s5|jd tjdtj } | s5|jd tjdtj} | s5|jd tjdtj } | s|jd tjdtj D]} | j dg} d| vsddj| vr-| jd} tjdd| j}t|jdkDst| dk\s| } n| s5|jd tjdtj} | r t| }|sAtj d|tj}|rt |jd}|sytj d|tj"tjz}|rAt |jd}tjdd|j}|}|sItj d|tj"tjz}|r|jd }|sItj d |tj"tjz}|r|jd }|s)tj d |}|r|jd }|||fS)!Nr html.parsermetaname ArticleTitle)attrscontentPubDatez(\d{4})-(\d{2})-(\d{2})rdivzTRS_Editor|trs_editor_view)class_zarticle-detailzjs_contentBox|contentBox)idz ^news-box$viewclasscmmaincontr pr+2rz zoom|Zoomz5\s*(.*?)\s*z\s*[|_-]\s*[^-|_]{2,30}$z?]*class="[^"]*TRS_Editor[^"]*"[^>]*>(.*?)\s*z-]*id="[^"]*zoom[^"]*"[^>]*>(.*?))rfindrcompiler2r rrmatchr3find_alljoinrget_textr5rr4r1)r;page_urltitlerC content_htmlsoupmtmddrBdcvclpsrAr=rs r parse_detailrhds[ E DLY 6 YYvfbjj"$$.O%PY Q "&&#9 .E YYvfbjjRTT.J%KY L "&&#9 ##%A4a8Bxx{ YYuRZZ0Mrtt%TY U54Ertt)LMB5RZZ0KRTT%RSB5M244)HIB]]5GRTT1J]K UU7B'2:sxx|!;ZZ_ffZQZZ\:syy{#b(SW\B 5RZZ bdd%CDB r7L  IINPY[][_[_ `  +E  II3Yrtt L AGGAJ'A2B:@@BAE  IIXZcegeieilnlplpep q 771:L  IIF SUSWSWZ\Z^Z^S^ _ 771:L  II0) < 771:D $ $$rc  |sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}trQ t |d}|j d D]%}d D]}|j j|d 't|}|}tjd d|tj}gifd }tjd ||tjtjz}gfd}tjd||tjtjz}tjd|tjrdnd}tjdd|tj}tjdd|tj}tjdd|tj}g} |jdD]} | j} | stjdd| } tj| } | jddjdd} tjdd| } | s~j!D]\} }| j| |} t#D]\}}| jd|d|} | j%| jdj'| }j!D]\} }|j| |}t#D]\}}|jd|d|}tjdd|}tjd d|}tj|}tjd!d|}tjd"d|}tjd#d|}t)}g}|jdD]X}tjd$dtjdd|}|s2||vr7|j+||j%|Zdj'|j}||fS#t$rYwxYw)%Nrrz )flagsz ]*>z]*>.*?z]*>.*?rGT) stylerRlangdiralignvalignwidthheightborder cellpadding cellspacingz;]*src="[^"]*fileTypeImages/icon_[a-z]+\.gif"[^>]*/?>c@|jd}|jd}tjdd|}tj|j }tjdd|j }|s5t jj|jddxsd}tjj|}|jd rd |z}j||fd td }d |d|d|<|S)Nrr,r+ru^附件[:：]\s*?ru附件r.r/__LINK____z z)r3rrr r rospathbasenamesplitr6r7r8r9r:r5) r=r@innerrArDkey attachments link_protectr]s r _link_replz html_to_text.._link_replswwqz ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6   d #'GGS>*\*+2.'y0B3%tL S rz&]*href="([^"]+)"[^>]*>(.*?)chj|jddtdz dS)Nr__TBL__rrx)r:r3r5)r= table_protects r _tbl_replzhtml_to_text.._tbl_repls3QWWQZ(]+A-.b11rz]*>.*?z ]*>rz

z

z  z(]*>z r+ r z z[\s\u200b\u200c\u200d\ufeff]+rrxz __LINK__\d+__z __TBL__\d+__z[ \t]+\nz\n{3,}z [ \t]{2,}r )rrr1r2rrZrKpopr Exceptionr4r|rr r r r< enumerater:r[setadd)rLr]_souptagattrcontent2rr has_tablepartsblockbplainkreitbloutseenfinalrUr~rrrs ` @@@r html_to_textrsC 1byff]BrttAIvvgz8244@HvvlD("$$?HvvA2xWYW[W[\H E'  KKM z2q)!!%( fc*228SA7UC  &&( DAq !QA  . 0FAs GA3b/3/A 0 QWWY ++e C""$ 1kk!Q M*03kkGA3b/3/0 &&!2s +C &&"c *C   C C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&c   sAS77 TTc |jd|tf}|jry d} ddlm}||}|jd|||||tt t||f }|jdkDrytjdd|} tj| } tjd d | jdd } |jd |j|t| fy y#t $rYwxYw#tj $r=} dt#| vr*t%j&dt)||||||cYd} ~ Sd} ~ wwxYw)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?Fotherr)classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)r+rr r zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)Tlocked)execute SITE_NAMEfetchone crawler_librr GROUP_NAME SCRIPT_NAMErowcountrrr r r lastrowidsqlite3OperationalErrorrtimesleep store_record) connr&r^rLrCrcurindustryrsummaryes rrrsF ,,PSVXaRb cC ||~  5(/Hll +#ugtY KQZ\d e   <d!d y#t$r}td|d|d Yd}~xd}~wwxYw#t$r#}td| d|d |dz }Yd}~d}~wwxYw)"Nr<)rzPRAGMA busy_timeout=60000zPRAGMA journal_mode=WALrzPage z fetch error: T)flushz: found z itemszEmpty page, stop paginationrr )errorsz detail error z: r+r z skip empty: rz + (z ()g?g?u新增: u跳过: z=== z done ===) requestsSessionrupdater!rconnectDB_PATHrrrange_PAGESr)rprintrEr5rrrLdecoderhrrrrrrrcommitcloser)rsr new_count skip_count seen_urlsr%r;rr<rDr^rCr(dhtmldtitleddater_rLrr plain_lenoks rmainr0sAIIW ??7B /DLL,-LL*+IJIa!$( "1d+I9% dV8CJ   y1#6dCa  s0 I>)I? I<I77I<? J+J&&J+__main__)(__doc__sysryrrhtmlr urllib.parser6rbs4r ImportErrorBASE_URLrrrrrr! _MAX_PAGESrargvrar9intr| ValueErrorr5rrr)rErhrrrr__name__rrrsm ! ( 6 <  # @O'   chh  DAq||J QWWS!_Q/0J iAECM1 SXXa!e_-J "QA 4=%@_'DD0:3z zFs M,       s5C)$C6 D)C32C36C>=C>D D