\8}j[< dZddlZddlZddlZddlZddlZddlZddlZ ddl m Z ejjddZejjddZdZd d d d d ZdZdZdZdZdZdZdZdddddddddZdZdZeed<d Zeej>D]k\Z Z!e!jEd!r e#e!jId"d d Z4e!d#k(s:e d ze&ej>ksS e#ej>e d zZmed k\rend Z'd$Z(d,d%Z)d&Z*d'Z+d(Z,d)Z-d*Z.e/d+k(re+yy#e $rdZ YwxYw#e%$rYwxYw#e%$rYwxYw)-u crawl_hekou_gsgg.py - 东营市河口区-公示公告 http://www.hekou.gov.cn/col/col101094/index.html CMS: Hanweb jpage dataproxy POST 列表: POST /module/web/jpage/dataproxy.jsp?startrecord=N&endrecord=M&perpage=15 body: col=1&webid=263&path=http://www.hekou.gov.cn/&columnid=101094&unitid=436047&webname=河口区人民政府&permissiontype=0&sourceContentType=1 响应 XML ...]]> 详情: /art/2026/7/31/art_192987_xxx.html, meta ArticleTitle, 正文 div.main-txt N) BeautifulSoupDB_PATHz/mnt/data/search.db JSONL_PATHutf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9XMLHttpRequest)z User-AgentAcceptzAccept-LanguagezX-Requested-Withu河口区-公示公告u山东zcrawl_hekou_gsgg.pyu 公示公告zhttp://www.hekou.gov.cnz0http://www.hekou.gov.cn/col/col101094/index.htmlz6http://www.hekou.gov.cn/module/web/jpage/dataproxy.jsp1263zhttp://www.hekou.gov.cn/101094436047u河口区人民政府0)colwebidpathcolumnidunitidwebnamepermissiontypesourceContentTypei*Refererz--pages==z--pagesc2|sytj|}|jddjddjddjdd}tjdd|}tjdd|}|j S) Nru​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts %/root/gov_crawler/crawl_hekou_gsgg.py clean_titler&Cs !A (B''"5==hKSST\^`aA Er1MA vsAA 779ctt}|r||d<d}tdD]} ddl}ddl}|j |j j|j|||d}|jdk(rt|_ |jcSd|jz}tj d t#d |dd d |d y#t$r} t| dd}Yd} ~ Jd} ~ wwxYw)NrrF)headerstimeoutverifyzHTTP %sPz [fetch fail] Zz -> Tflush)dictHEADERSrangerequestsurllib3disable_warnings exceptionsInsecureRequestWarningget status_codeENCencodingtext Exceptionstrtimesleepprint) urlrefererr+hlast_err_r6r7res r%fetchrLMs W A) H 1X  # $  $ $W%7%7%N%N O S!WU KA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #sA'CC C0C++C0c  |sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}gitjdd|}tjd d|}tr@ t |d }|j d d D]>}|j dsG|j ds6tjd|j ddtjs\|d}|j dxs|}|jd xsW|j dxsD|j dxs1tjj|jdd}tjj|}j!||fdt#z}d|d|d|<|j%|A|j dD]_} | j dxs| j dxsd} d| j'vsd| j'vsP| j)a|j dD]@} | j d} | s| jd r+g} | D]} | j dxs| j dxsd} | s.tjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}| j!d|d|d| sd"t#z}d#d$j-| zd%z|<| j%|C|j dD]} | j dxs| j dxsd} | rd| j'vr@tjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}d"t#z}d&|d|d'|<| j%|t/|}fd(}tjd)||tjtjz}trQ t |d }|j d D]%}d*D]}|j2j5|d 't/|}g fd+}tjd,|tjrd-nd}tjd.||tjtjz}tjd%d/|tj}tjd0d1|tj}tjd2d|tj}g}|jd3D]V}|j+}|stjd4d|}tjd5d|}|j7d6dj7d7d8}tjd9d8|}tjd:d|}t9j:|}|j7d;d8j7d|z|}|j!|j+Yd3j-|}j=D]\}}|j7||}t? D]\}}|j7d>|z|}tjd?d|}tjd@d|}tjdAd|}tjdBd|}t9j:|}tjdCd1|}tjdDd3|}tjd9d8|}tA}g}|jd3D]X} tjdEdtjd:d| }|s2||vr7|jC||j!| Zd3j-|j+}||fS#t0$rYwxYw#t0$rYwxYw)FNrrz )flagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】u'发布(?:时间|日期)[:：][^<]{0,30} html.parseraT)hrefappendixz data-appendixz&\.(pdf|docx?|xlsx?|zip|rar|wps|et|ofd)rQoldsrcr#_titletitle?z__ATTACH__%d__ imgsrczfileTypeImages/icon_filetypeimagespaltu图片z __IMG__%d__z

z
z

z

._link_replswwqz ??= ) ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*s<00ELcR S r'z&]*href="([^"]+)"[^>]*>(.*?)) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingcfj|jddtdz zS)Nr __TBL__%d__r)rjrbrk)rl table_protects r% _tbl_replzhtml_to_text.._tbl_repls.QWWQZ(M 2Q 677r'z ]*>rz]*>.*?z

z  z(]*>z z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  rz [ \t]{2,}ra z z[\s\u200b\u200c\u200d\ufeff]+rz __LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r)"r!r"SIrfind_allr;searchget_textrdrrerfrgrhrirjrk replace_withlower decomposer#joinrAr@attrspopr rritems enumeratesetadd)!contentrs_souprPrQ real_hrefrnrorpr[r\r^imgssrcsrttagattrr has_tablepartsblockbplainkvitbloutseenfinalrqrrrs! ` @@@r% html_to_textrbs 1byff]Brtt"))Luwxw|w|~DFHxIKMKOKOCPV9D !h 74I**4*0CAEE(OCquuW~CY[Y`Y`YiYijsjyjyz}j~@AkBZCC$ll228YGG&&~6*S->>CQXZ](^L%NN3' (~~e, $wwu~@):@b)SYY[8#KSWWW5E#K"R"R"T#HXZX_X_XhXhilirirsviwxyizX{#H@HC KKQXZ](^_ ` +c,.??,1GLL4F,F,O S)s+ ,~~e, &ggen?(9?R4 C ,,..x=wwu~?)9?RFFH{BGGL\L\]`]f]fgj]klm]nLo{s{#c,&77T[]`$a S!  % &%jGff> G[][_[_bdbfbf[fgG !'=9E~~d+ .L.DIIMM$-. .%jGM8YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  &&( DAq !QA  . 2FAs -!+S1A 2 QWWY% ( ++e C""$ 1kk!Q M*23kk-!+S12 &&!2s +C &&"c *C &&"c *C &&#R -C   C C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&u   6   sMA:g?Eg:gg1gB)g1Eg Ag( g%$g%( g54g5cddl}tjjdtjj tjj tddlm }|j}|jjtd}d}t}g}g}tdt dzD]&} t#||} t%| } | st)d|zdnt)d|t+| fzd| D]\} } }| |vr |j-| t/| }|s|dz }1t1|| \}}}|s| }|s|}t3|| \}}}t+t5j6d d |j9}t5j:d |t4j< xr+t5j:d |t4j< xr| }|d ks|rt)d|ddzd|dz }t>| | ||d |t@d tB|rdjEd|Dnd d }tFr/|jI|t)d|ddd|dd|dz }c|jI||dz }t)d|ddd|ddtKjLdtKjLdtFr|s||tNg})tFrltQtFdd5}|D]+}|jStUjV|ddz- dddt)d tFt+|fzdn|r||tNt)d!|zdt)d"|zdt)d#t>zdy#t&$r} t)d|| fzdYd} ~ d} ~ wwxYw#1swYxYw)$Nr)push_to_searchdbrzPage %d fetch error: %sTr1zPage %d: empty, stopzPage %d: found %d itemsrarz]z] z skip empty/placeholder: %s(z; c30K|]\}}|d|yw)|N).0ur$s r% zmain..'s(KdaAq)9(Ks) site_name source_urlrErVpub_datesummaryrcategorytags group_namerqz JSONL + z ()z + g?g?) batch_labelwr)r>F) ensure_asciirzJSONL written: %s (%d rows)u 新增: %du 跳过: %dz=== %s done ===),r6sysrinsertrddirnameabspath__file__ crawler_librSessionr*updater4rr5_PAGES fetch_list parse_listr@rDrkrrL parse_detailrr!r"r#rr SITE_NAMECATEGORY GROUP_NAMErrrjrBrC SCRIPT_NAMEopenwritejsondumps)r6rs new_count skip_count seen_urls jsonl_rowsbatchpagexml_textrrKrorVdatedhtmldtitleddate content_htmlrratts plain_lenno_paraitemfrows r%mainrs{HHOOArwwrwwx'@AB,AIIWIJIJ Ea!$2 !!T*Hx(E (4/t <  '4U*<r?)sessionrstartendparamsrJs r%rrEsz AX 1 $E .C )_FJF=c(F;G F9 YVWbQV WAAJ 66Mr'c g}tjd|tjD]}|jd}tjd|tjtjzD]d}t j |jdj}|jd}ttjdd|}|rt|dkrxtjd|s|jd r t|z}n8|jd s'tjj!td z|}tjd |}d} |rU|jdd |jdj#dd |jd j#d} |j%||| fg|S)Nzrz$]*href="([^"]+)"[^>]*>(.*?)r/rarz//art/\d{4}/\d+/\d+/art_\d+_\d+\.html|/art/\d{4}/http'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2})-r))r!finditerrrbrrrr#r&r"rkrrcBASE_URLrgrhrizfillrj) rrrlfraglmrQrmrndmrs r%rrQss E [[18RTT B,wwqz++EtRTTTVTXTX[Y ,B$$RXXa[1779DHHQKEbffZU;.s"?a3q6"?sr)rfindr!compilerr;r&rrrbr select_onercontents) html_textrsrVrrsoupmtrmdrels r%rrks E DLY 6 YYvfbjj"$$.O%PY Q "&&#9 .E4B#BKKdK$;< YYvfbjjRTT.J%KY L "&&#92i=IB%'XXa["((1+2C2CA2FQR HYHYZ[H\]oon-EAQ1RE0E48OOO4TEf%E)-)D  77"?2;;"??L  YYA9 M !#!bhhqk.?.?.BBHHQKDUDUVWDXYD $ $$r'__main__)Nr)0__doc__rrdr!rBrhtmlr urllib.parsergbs4r ImportErrorenvironr;rrr=r4rrrrrLIST_URLrrr TOTAL_RECORDS _MAX_PAGESrargvrrPrcintrf ValueErrorrkrr&rLrrrrr__name__rr'r%r#s?>>>! **..$9 : ZZ^^L" -  @O'(   %  #  $ = D   &&       chh  DAq||J QWWS!_Q/0J iAECM1 SXXa!e_-J "QA*Q'hL5^ 4%> zFu MP       s5D85E1E8EEE EEE