zyj#dZddlZddlZddlZddlZddlZddlZddlmZddl m Z ejjdejjejjeddlmZmZdZdZdZd Zej.d ej0Zgd Zd Zed dZdZdZdZdZ dZ!dZ"e#dk(re"yy)u 仙桃文明网-公示公告 爬虫 站点: http://www.cnxiantao.com/gg/ TRS 静态页: createPageHTML(19, 0, "index", "html") 列表:
  • 2026-07-21标题
    • 分页: index.html / index_1.html ~ index_18.html 详情容器: div.neroncont (TRS 生成, 含 p.Custom_UnionStyle 嵌套 span) 标题: div.con > div.p1 > h2; 日期: 发布时间：YYYY年MM月DD日 N) BeautifulSoup)urljoin)push_to_searchdbnormalize_pub_dateu仙桃文明网-公示公告u 公示公告zhttp://www.cnxiantao.com/gg/z)http://www.cnxiantao.com/gg/index_{}.htmlzQ
  • \s*([^<]+)\s*]*>([^<]{4,120}))z div.neroncontz .TRS_UEDITORz div.contentzdiv.viewz div.articlezoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zzh-CN,zh;q=0.9)z User-AgentzAccept-Languagec:ddl}|j|xsd}tjdd|}tjdd|}tjdd|}tjdd|}tjdd |}|j S) u?标题清洗: 剥 ·/ /零宽/装饰符/栏目后缀rNz·| z[\u200b\u200c\u200d\ufeff]u^[•·●\-—\s]+u<[—\-–]\s*(公示公告|通知公告|仙桃文明网)\s*$\s+ )htmlunescaperesubstrip)titlehtml_mods )/root/gov_crawler/crawl_cnxiantao_gsgg.py clean_titler"s   ekr *E FF%r5 1E FF0"e zparse_list..=sYaqEzYs)u首页u邮箱u搜索u 无障碍u地图zt20\d{6}_\d+\.html?$)rrdate) LINK_REfinditergrouprlenanyr searchrrappendr) r list_urlitemsmdate_strhrefrr,rs @r parse_listr97s E   d # N ! 0 0 2AGGAJ4D4D4FPQ HXHXHZ$E Q  Y#XY Y yy0$7 h%!(+ {51#tLM N Lrc x t|d}d}|jdxs|jd}|rt|j d}|sAt j d|t j}|rt|jd}|s@|jd }|r-t|j djd d }d}t j d |}|rK|jdd t|jddd t|jdd}|sAt j d|t j}|rt|jd}d} tD]=} |j| } | s| j ds*| } |j| n| ||dgfSg} | jddD]} | dj}t j d|t jsd|j!vsNt#||}| j dxs|jdd}| j||dg}| jgdD]} | j$dk(r| j'ds"| jds| jdrG| jddD]\} | dj}t j d|t jsd|j!vsNt#||| d<^| j dd}t j(dd|}|s|jd |d!| j$dk(r|jt+| '| j$d"k(s8| j-d#d}|sNt#||}| j-d$d}|jd%|d&|d'| D] }|jd(|d)d*|d+d,"d-j/|}t j(d.d|}t j(dd|}|rt1|d/kr,| jds| j dd}|rd |d!}|||| fS)0uJ提取标题/日期/正文(保留 p HTML)/附件, 记录命中的容器名z html.parserrzdiv.con div.p1 h2h2TrzE]*name=["\']ArticleTitle["\'][^>]*content=["\']([^"\']*)["\']r#ru——ru@发布时间[：:]\s*([0-9]{4})年([0-9]{1,2})月([0-9]{1,2})日-r$02dr%z@]*name=["\']PubDate["\'][^>]*content=["\']([^"\']*)["\']Na)r8r8z)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$download/)namer)ptableimgrDrEr r 

    z

    rFsrcaltz

    z

    z

    rCz

     z<[^>]+>2)r select_onefindrget_textr r2Ir/splitintr CONTAINERSr3find_allrlowerrrC find_parentrstrrjoinr0)r page_urlused_containersouprr;r6tr,body_elselel attachmentsr?r8fullrCpartstxtrHrIattcontentextracted_texts rextract_detailrfHs` } -D E , - @4B BKKdK34  II^`dfhfjfj k  +E  IIg    6 < ~~g&"'''*:bggcl[[4[0 8y(99I4QSQUQUVZdhlhrhrhtZt '$ 7AfI 8++c+.C&&c*C s3%t_- WW  LLR ! WW &&#Cx-ffUB' }TF'#fEF)G,I |CJ*R/g9Ns$/ C5oG $ ,,rctjddD]O}|jdrt|j ddcS|j sDt|cSy)Nr#z--pages==)sysargv startswithrQrPisdigit)r?s rparse_pages_argrmsU XXab\ << #qwws|A' ' 99;q6M  rct}|d}tdtd|tj}|j j tgt}}td|dzD]}|dk(rtntj|}t||}|rt|dkrtd|dnt|t}|Dcgc] }|d|vs |} }| D]}|j!|d|j#| td|d t|d t| d t|| snt%j&d td t|dg} g} t)|D]\} } t| d|}|s/| j+t| d| dd| d| dt,ddFt/|| d| \}}}}|s| d}|s| d}| j+t|||| d| dt,|rt1j2|dndd| dzdzdk(r#td| dzdt|d|ddt%j&dt5| tt7d| D}t7d| D}t7d| D}| Dcgc] }|d s |d }}td!t| d"|d#|d$|d% td&| rt9t| nd'td(|r t;|nd)d*|r t=|nd)ycc}wcc}w)+N[z ] max_pages=r#rz Page u: 空/失败, 停止rz: u条(新u) 累计g?u 共 u 条, 抓详情...rr,r) site_namerpub_daterd source_urlrcategoryr_F) ensure_ascii rz [rAz] (g333333?) batch_labelc3HK|]}|djrdyw)rdr#Nr<r)r s rr+zmain..s?a!I,*<*<*>?s""c30K|]}d|dvs dyw)rGrdr#Nr(rzs rr+zmain..s=qu) '<=s c3DK|]}|jdsdyw)r_r#N)rrzs rr+zmain..s 9AAEE-$8a 9s  rru 统计: 共u条, 空正文u, 含

    正文u , 带附件u条u详情容器命中: u 未命中u日期范围: r=z ~ )rmprint SITE_NAMErequestsSessionheadersupdateHEADERSsetrangeLIST_URLPAGE_URLformatr!r0r9addextendtimesleep enumerater3CATEGORYrfjsondumpsrsumsortedminmax) max_pagesr all_itemsseenpgrr r5r* new_itemsused_containersresultsiitemrr,rdr_emptywith_prcr datess rmainrsi!I  Ai[ YK 01 G OO7##%tIAy1}%'hxr':S'"s4y3 GB434 5 4* %>15)=Q> > A HHQuX  # t2c%j\Y0@YHXYZ  3  F3y>""4 56OGY'4T%['* NNT']X\]cXd')eTRW[(0E F ,:4eo,^)tWkMEs M#M M M __main__)$__doc__osrir rrrbs4r urllib.parserpathinsertdirnameabspath__file__ crawler_librrr~rrrcompileSr-rRUArrr!r9rfrmr__name__r(rrrs 277??277??8#<=>< *  ) 6 "**0DD  Y v0@ A "U-p8\v zFr