.rjF3dZddlZddlZddlZddlZddlZddlZddl Z ddl Z dZ dZ dZ dZdZdd d Zd d d Zd Zeej(D]y\ZZej/dr$ej1dddj3Z;edk(sAedzeej(ksZej(edzj3Z{eevredezej8de dzeezZ ddlmZej@jCddZ"ej@jCddZ#dZ$ddddZ%dZ&eej(D]k\ZZej/dr e'ej1dddZ&4edk(s:edzeej(ksS e'ej(edzZ&me&dk\re&ndZ)d Z*d(d!Z+d"Z,d#Z-d$Z.d%Z/d&Z0e1d'k(re-yy#e$rdZYwxYw#e($rYwxYw#e($rYwxYw))u\ crawl_htb_tzgg.py - 呼图壁县 住建局 ×2 https://www.htb.gov.cn/p222/{szfwzjj,tzgg}.html 列表: JEECMS API GET /content/page?channelIds={id}&page=N&size=15&orderBy=27 (JSON: data.content[].title/urlWhole/date) 分页: ?page=N (JSON) 详情: urlWhole 字段, 正文 div.article-content 用法: python3 crawl_htb_tzgg.py --pages=5 --col=tzgg Nzhttps://www.htb.gov.cnu呼图壁县住建局u新疆zcrawl_htb_tzgg.pyu 公示公告u 通知公告u 市政服务)tzggszfwzjj80908735rz--col==z--colz bad --col: %s-) BeautifulSoupDB_PATHz/mnt/data/search.db JSONL_PATHutf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagez--pages=z--pagesc2|sytj|}|jddjddjddjdd}tjdd|}tjdd|}|j S) Nr u​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts #/root/gov_crawler/crawl_htb_tzgg.py clean_titler@s !A (B''"5==hKSST\^`aA Er1MA vsAA 779ctt}|r||d<d}tdD]} ddl}ddl}|j |j j|j|||d}|jdk(rt|_ |jcSd|jz}tj d t#d |dd d |d y#t$r} t| dd}Yd} ~ Jd} ~ wwxYw)NRefererrFheaderstimeoutverifyzHTTP %sPz [fetch fail] Zz -> Tflush)dictHEADERSrangerequestsurllib3disable_warnings exceptionsInsecureRequestWarningget status_codeENCencodingtext Exceptionstrtimesleepprint) urlrefererr"hlast_err_r-r.res rfetchrCJs W A) H 1X  # $  $ $W%7%7%N%N O S!WU KA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #sA'CC C0C++C0c ~ |sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}gitjdd|}tjd d|}tr t |d }|j d d D]>}|j dsG|j ds6tjd|j ddtjs\|d}|j dxs|}|jd xsW|j dxsD|j dxs1tjj|jdd}tjj|}j!||fdt#z}d|d|d|<|j%|A|j dD]} | j d} | s| jd r+g} | D]} | j dxs| j dxsd} | s.tjj| }| j!d|dtjj| jddxsdd| sdt#z}d d!j'| zd"z|<| j%|t)|}fd#}tjd$||tjtjz}trQ t |d }|j d D]%}d%D]}|j,j/|d 't)|}g fd&}tjd'|tjrd(nd}tjd)||tjtjz}tjd"d*|tj}tjd+d,|tj}tjd-d|tj}g}|jd.D]V}|j1}|stjd/d|}tjd0d|}|j3d1dj3d2d3}tjd4d3|}tjd5d|}t5j6|}|j3d6d3j3d7d3}tjd8d|}|sj9D]\}}|j3||}t; D]\}}|j3d9|z|}|j!|j1Yd.j'|}j9D]\}}|j3||}t; D]\}}|j3d9|z|}tjd:d|}tjd;d|}tjdd,|}tjd?d.|}tjd4d3|}t=}g}|jd.D]X} tjd@dtjd5d| }|s2||vr7|j?||j!| Zd.j'|j1}||fS#t*$rYwxYw#t*$rYwxYw)ANr rz )flagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】u'发布(?:时间|日期)[:：][^<]{0,30} html.parseraT)hrefappendixz data-appendixz\.(pdf|docx?|xlsx?|zip|rar|wps)rHoldsrc)r_titletitle?z__ATTACH__%d__ pimgsrcu图片z __IMG__%d__z

z
z

c6|jd}|jdry|jd}tjdd|}t j |j }tjdd|j }|s5tjj|jddxsd }tjj|}j||fd tz}d |d |d |<|S)Nrz javascript:r r&<[^>]+>u^附件[:：]\s*rMru附件z __LINK__%d__rNrOrP)group startswithrrrrrospathbasenamespliturllibparseurljoinappendlen) mrHinnertxtabs_urlkey attachments link_protectpage_urls r _link_replz html_to_text.._link_replswwqz ??= ) ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*s<00ELcR S rz&]*href="([^"]+)"[^>]*>(.*?)) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingcfj|jddtdz zS)Nr __TBL__%d__r)r_rVr`)ra table_protects r _tbl_replzhtml_to_text.._tbl_repls.QWWQZ(M 2Q 677rz ]*>rz]*>.*?z

z  z(]*>z z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  rz [ \t]{2,}rU z z[\s\u200b\u200c\u200d\ufeff]+rvz __LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r) rrSIr find_allr2searchget_textrXrYrZr[r\r]r^r_r` replace_withjoinr8r7attrspoprrrritems enumeratesetadd)!contentrh_souprGrH real_hrefrcrdrerQimgssrcsrRrSritagattrrx has_tablepartsblockbplainkvitbloutseenfinalrfrgrws! ` @@@r html_to_textr_sK 1byff]Brtt"))Lnpqpupuv|AqBDFDHDHCIV9D !h 74I**4*0CAEE(OCquuW~CY[Y`Y`YiYijsjyjyz}j~@AkBZCC$ll228YGG&&~6*S->>CQXZ](^L%NN3' (^^C( ,zz%(  6D#L!ggenG0AGR&,ll&:&:8S&IG KKQXZ\ZaZaZjZjknktktuxkyz{k|Z}[JBJ[J)KL L +c,.??,1GLL4F,F,O S)s+ ,%jGff> G[][_[_bdbfbf[fgG !'=9E~~d+ .L.DIIMM$-. .%jGM8YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  &&( DAq !QA  . 2FAs -!+S1A 2 QWWY% ( ++e C""$ 1kk!Q M*23kk-!+S12 &&!2s +C &&"c *C &&"c *C &&#R -C   C C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&u   6   sGA:`?D``!1`A/`A`A`/ `,+`,/ `<;`<cVddl}tjjdtjj tjj tddlm }|j}|jjtd}d}t}g}g}tdt dzD]} t#||} t%| } | st)d|zdnt)d|t+| fzd| D]|\} } }| |vr |j-| t/| }|s|dz }1t1|| \}}}|s| }|s|}t3|| \}}}t+t5j6d d |j9}t5j:d |t4j< xr| }|d ks|rt)d |ddzd|dz }t>| | ||d |t@d tB|rdjEd|Dnd d }tFr/|jI|t)d|ddd|dd|dz };|jI||dz }t)d|ddd|ddtKjLdtKjLdtFr|s||tNg}tFrltQtFdd5}|D]+}|jStUjV|ddz- dddt)dtFt+|fzdn|r||tNt)d |zdt)d!|zdt)d"t>zdy#t&$r} t)d|| fzdYd} ~ d} ~ wwxYw#1swYxYw)#Nr)push_to_searchdbrzPage %d fetch error: %sTr(zPage %d: empty, stopzPage %d: found %d itemsrUr z] z skip empty/placeholder: %s(z; c30K|]\}}|d|yw)|N).0urs r zmain..s(KdaAq)9(Ks) site_name source_urlr<rLpub_datesummaryrcategorytags group_namerfz JSONL + z ()z + g?g?) batch_labelwr)r5F) ensure_asciiryzJSONL written: %s (%d rows)u 新增: %du 跳过: %dz=== %s done ===),r-sysrYinsertrXdirnameabspath__file__ crawler_librSessionr!updater+rr,_PAGES fetch_list parse_listr7r;r`rrC parse_detailrrrrrr~ SITE_NAMECATEGORY GROUP_NAMErr r_r9r: SCRIPT_NAMEopenwritejsondumps)r-rs new_count skip_count seen_urls jsonl_rowsbatchpage html_textrrBrdrLdatedhtmldtitleddate content_htmlrratts plain_lenno_paraitemfrows rmainrs[HHOOArwwrwwx'@AB,AIIWIJIJ Ea!$3 "1d+Iy)E (4/t <  '4U*< MQ\]aQbdhPii iC C"U CA 66Mrcg} tj|}|jdijdgD]x}t |jdd}|jdd}t |jddxsddd}|r|rt |dkre|j|||fz|S#t$r|cYSwxYw) NdatarrLr urlWholerr)rloadsr7r2rr8r`r_)rrritrLrHrs rrr;s Ezz)$hhvr"&&y"5*BFF7B/0vvj"%266&"%+,Sb1DCJN  dE4() * L  sB88 CCc jd}d}d}tjd|tjtjz}|r/t tj dd|j d}|sytjd|tjtjz}|rAt |j d}tj dd|}|j}tjd|}|rU|j dd|j d jd d|j d jd }trTt|d }|jd xs|jd }|r"djd|jD}|||fS)Nr z(
(.*?)
rUrz(.*?)u/[-\u2014|_]\s*(?:呼图壁县|网站首页).*?$u'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2})r r&rrFzdiv.article-contentz div.contentc32K|]}t|yw)N)r8)rcs rrzparse_detail.._s"?a3q6"?s) rrr}r~rrrVrzfillr select_onercontents) rrhrLrrrarsoupels rrrKsL E DL =y"$$QSQUQU+VABFF:r1771:>?  II-y"$$+ F AGGAJ'AI2qQAGGIE N>Nq>QRY 6 __2 3 Ut}7U 77"?2;;"??L $ $$r__main__)Nr)2__doc__rrXrr9htmlr urllib.parser\sqlite3rrrrrrCOL_MAPrrrargvrrGrWr[rr`r;exitbs4r ImportErrorenvironr2r r r4r+ _MAX_PAGESint ValueErrorrrrCrrrrr__name__rrrrsPHGGG # #  ! !n =&1   chh 'DAq||HwwsAq!'') g!a%#chh-/xxA$$& ' w /D ! CHHQK Ogdm + ! **..$9 : ZZ^^L" -  @O'   chh  DAq||J QWWS!_Q/0J iAECM1 SXXa!e_-J "QA*C'LM5`  %0 zFA M$       s64G)&G6"H)G32G36G>=G>H H