Aj(dZddlZddlZddlZddlZddlZddlmZddlZddlm Z ddl m Z ejjdejjejjeddlmZmZdZdZd Zd Zej0d ej2Zgd Zd ZeddZdZdZdZ dZ!dZ"dZ#dZ$e%dk(re$yy)u 仙桃文明网-公示公告 爬虫 (主站 xtwenming.com) 站点: http://www.xtwenming.com/zx/gsgg/ TRS 静态页: createPageHTML(6, 0, "index", "shtml") 列表:
  • 2026-07-15标题
    • 分页: index.shtml / index_1.shtml ~ index_5.shtml 详情容器: div.neroncont (TRS 生成) 标题: div.con > h2; 日期: 时间：YYYY年MM月DD日 注意: 与 cnxiantao.com/gg/ 为独立内容源, ID 0 重叠 N)datetime) BeautifulSoup)urljoin)push_to_searchdbnormalize_pub_dateu&仙桃文明网-公示公告-xtwenmingu 公示公告z!http://www.xtwenming.com/zx/gsgg/z/http://www.xtwenming.com/zx/gsgg/index_{}.shtmlzQ
  • \s*([^<]+)\s*]*>([^<]{4,120}))z div.neroncontz .TRS_UEDITORz div.contentzdiv.viewz div.articlezoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zzh-CN,zh;q=0.9)z User-AgentzAccept-Languagec:ddl}|j|xsd}tjdd|}tjdd|}tjdd|}tjdd|}tjdd |}|j S) u?标题清洗: 剥 ·/ /零宽/装饰符/栏目后缀rNz·| z[\u200b\u200c\u200d\ufeff]u^[•·●\-—\s]+u<[—\-–]\s*(公示公告|通知公告|仙桃文明网)\s*$\s+ )htmlunescaperesubstrip)titlehtml_mods )/root/gov_crawler/crawl_xtwenming_gsgg.py clean_titler%s   ekr *E FF%r5 1E FF0"e zparse_list..@sYaqEzYs)u首页u邮箱u搜索u 无障碍u地图zt20\d{6}_\d+\.s?html?$)rr date) LINK_REfinditergrouprlenanyrsearchrrappendr) r list_urlitemsmdate_strhrefr r.rs @r parse_listr;:s E   d # N ! 0 0 2AGGAJ4D4D4FPQ HXHXHZ$E Q  Y#XY Y yy2D9 h%!(+ {51#tLM N Lrc  t|d}d}|jd}|rt|jd}|s6|jdddi }|r|j d rt|d }d}t j d |}t j d |} d} | rK| jd dt| jdddt| jdd} | }|r| r|jd d|jdd|jd} tj| d} tj| d} t| j| jz dz| j| jz z}|dk\r| }|s,t j d|}|r|jd dd}d}tD][}|j!|}|st#|jdddkDs|jdsH|}|j%|n|||dgfSg}|j'ddD]}|dj)}t j d|t j*sd|j-vsNt/||}|jdxs|j1d d!}|j%||d"g}|j'gd#D]}|j2d$k(r|j5d%s"|jd%s|jd$rHt7d&|j'ddDrl|j'ddD]\}|dj)}t j d|t j*sd|j-vsNt/|||d<^|jdd}t j8d'd|}|s |j%d(|d)#|j2d%k(r|j%t;|N|j2dk(s_|j d*d}|sut/||}|j d+dxsd,}|j%d-|d.|d/|D] }|j%d-|d0d.|dd/"d1j=|}t j8d2d|}t j8d'd|}|rt#|d3kr,|jd%s|jdd}|rd(|d)}||||fS#t$rYwxYw)4uJ提取标题/日期/正文(保留 p HTML)/附件, 记录命中的容器名z html.parserr h2Trmetaname ArticleTitle)attrscontentzt(20\d{2})(\d{2})(\d{2})_u1时间[：:]\s*(\d{4})年(\d{1,2})月(\d{1,2})日r%-r&02dr'z%Y-%m-%d rz&.sa FfVXHY[][_[_`;vr!2!8!8!::;sA#A%r 

    z

    srcaltu 查看图片z

    z

    r  z<[^>]+>2)rfindrget_textrrr4r1intrstrptimeabsyearmonth ValueError CONTAINERS select_oner2r5find_allrrQrRrsplitr@ find_parentr3rstrjoin)r page_urlused_containersouprr=mtr.murlr8 page_dateurl_dated1d2 month_diffmt2body_elselel attachmentsrIr:fullr@partstxtrTrUattrCextracted_texts rextract_detailr{Ks } -D E 4B BKKdK34  YYvfn%=Y > "&&#9 .E D 9918 ~~g&"'''*:bggclSt4 [[4[0 8y(99I4QSQUQUVZdhlhrhrhtZt '$ 7AfI 8++c+.C&&c*C s3%t_- WW  LLR ! WW &&#Cx-ffUB'9> |D6C5AB7C:I |CJ*R/g9Ns$/ C5oG $ ,,O   s0A2U U%$U%ctjddD]O}|jdrt|j ddcS|j sDt|cSy)Nr%z--pages==)sysargv startswithrZrcisdigit)rIs rparse_pages_argrsU XXab\ << #qwws|A' ' 99;q6M  rct}|t}tdtd|t j }|j jtgt}}td|dzD]}|dk(rt}ntj|dz }t||}|rt|dkrtd|dnt!|t}|Dcgc] }|d|vs |} }| D]}|j#|d|j%| td|dt|d t| d t|| snt'j(d td t|d g} g} t+|D]\} } t| d|}|s/| j-t| d| dd| d| dt.ddFt1|| d| \}}}}|s| d}|s| d}| j-t|||| d| dt.|rt3j4|dndd| dzdzdk(r#td| dzdt|d|ddt'j(dt7| tt9d| D}t9d| D}t9d| D}| Dcgc] }|ds |d}}td t| d!|d"|d#|d$ td%| rt;t| nd&td'|r t=|nd(d)|r t?|nd(ycc}wcc}w)*N[z ] max_pages=r%rz Page u: 空/失败, 停止r z: u条(新u) 累计g?u 共 u 条, 抓详情...rr.r ) site_namerpub_daterC source_urlr categoryruF) ensure_asciirGrz [rLz] (g333333?) batch_labelc3HK|]}|djrdyw)rCr%Nr>r+r"s rr-zmain..s?a!I,*<*<*>?s""c30K|]}d|dvs dyw)rSrCr%Nr*rs rr-zmain..s=qu) '<=s c3DK|]}|jdsdyw)rur%N)rrs rr-zmain..s 9AAEE-$8a 9s  ru 统计: 共u条, 空正文u, 含

    正文u , 带附件u条u详情容器命中: u 未命中u日期范围: rDz ~ ) r TOTAL_PAGESprint SITE_NAMErequestsSessionheadersupdateHEADERSsetrangeLIST_URLPAGE_URLformatr#r2r;addextendtimesleep enumerater5CATEGORYr{jsondumpsrsumsortedminmax) max_pagesr! all_itemsseenpgr r r7r, new_itemsused_containersresultsiitemrr.rCruemptywith_pryr"datess rmainrsq!I  Ai[ YK 01 G OO7##%tIAy1}% 7C//"q&)CS'"s4y3 GB434 5 4* %>15)=Q> > A HHQuX  # t2c%j\Y0@YHXYZ  3%( F3y>""4 56OGY'4T%['* NNT']X\]cXd')eTRW[(0E F ,:4eo,^)tWkMEs M+M MM__main__)&__doc__osr~rrrrrbs4r urllib.parserpathinsertdirnameabspath__file__ crawler_librrrrrrcompileSr/r`UArrrr#r;r{rr__name__r*rrrs  277??277??8#<=>< 4  . < "**0DD  Y v0@ A  "i-X<\~ zFr