ijQdZddlZddlZddlZddlZddlZddlZddlZ ddl m Z m Z  ddl m Z ejj!ddZdZdZd Zd Zd Zd Zd ZdZe j4e dz j7dZddddZdZeej@D]k\Z!Z"e"jGdr e$e"jKdddZ4e"dk(s:e!dze'ej@ksS e$ej@e!dzZmedk\rendZ(dZ)d#dZ*dZ+dZ,dZ-d Z.d!Z/e0d"k(re/yy#e$rdZ YwxYw#e&$rYwxYw#e&$rYwxYw)$u crawl_jcx_tzgg.py - 江城哈尼族彝族自治县人民政府 通知公告 列表: https://www.jcx.gov.cn/xwzx/tzgg.htm CMS: TRS vsb 静态列表（同族：望谟 line_u6_N；本站是 line_u9_N） 分页: ⚠️ **倒序静态链接** —— 页脚 `` 第 1 页 = /xwzx/tzgg.htm ；第 N 页(N≥2) = /xwzx/tzgg/{33-N+1}.htm（编号递减） 实测：tzgg/1.htm=最老(2024-02, 第33页) / tzgg/32.htm=第2页 / tzgg/28.htm=第6页 页脚另有「尾页」也带 class="Next" → **必须按链接文字「下页」取**，不能按 class 取第一个 本脚本**跟随「下页」链接**逐页（不按数字猜，符合该 CMS 既有教训） 页脚「共492条 1/33」可校验总页数 条目:
  • … 2026-09-24
    • 15 条/页 ⚠️ 链接文字可能被站点截断（带 ...），**优先取 title 属性** 详情: meta ArticleTitle / PubDate / ColumnName / ContentSource 正文: div.article_detail（页面唯一） 附件: 正文**内**的 ../../virtual_attach_file.vsb?afc=... （vsb 虚拟附件，无扩展名） ⚠️ 链接文字里夹 CMS 的「已下载 次」计数器 → 需清洗 噪声: div.nextList（上一条/下一条）在正文容器外，天然不混入 N)datetime timedelta) BeautifulSoup JSONL_PATHutf-8u"江城县人民政府-通知公告u云南zcrawl_jcx_tzgg.pyu 通知公告zhttps://www.jcx.gov.cnz$https://www.jcx.gov.cn/xwzx/tzgg.htmz/info/iG)daysz%Y-%m-%dzkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagez--pages==z--pagesc|sytj|}|jddjddjddjdd}tjdd|}tjdd|}tjdd|}tjd d |}|jd d jd d }|j S) Nru​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*u ^[•·]\s*<[^>]+>\s+ u“u”)html_libunescapereplaceresubstrip)ts #/root/gov_crawler/crawl_jcx_tzgg.py clean_titlerAs !A (B''"5==hKSST\^`aA Er1MA ~r1%A z2q!A vsAA (E"**8U;A 779ctt}d}tdD]} ddl}ddl}|j |j j|j|||d}|jdk(rt|_ |jcSd|jz}tj dt#d |dd d |d y#t$r}t|dd}Yd}~Jd}~wwxYw)NrF)headerstimeoutverifyzHTTP %sPz [fetch fail] Zz -> Tflush)dictHEADERSrangerequestsurllib3disable_warnings exceptionsInsecureRequestWarningget status_codeENCencodingtext Exceptionstrtimesleepprint) urlrhlast_errattemptr*r+res rfetchr?Ns W AH8  # $  $ $W%7%7%N%N O S!WU KA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #sA'C C C)C$$C)cg}tjd|tjtjzD]}|j d}tj d|tjtjz}|sNt j|j dj}t|vs|jjdstj d|j dtj}t|r|j dn|j d}tj d|tj}|r|rt|dkrB|j dd |j djdd |j d jd} |jt j"j%t&||| f|S) uYli#line_u9_N → a[title] + b.date；优先 title 属性（链接文字可能被截断）z(]*id="line_u9_\d+"[^>]*>(.*?)r z&]*href="([^"]+)"[^>]*>(.*?)z.htmztitle="([^"]*)"rr#z.\s*(\d{4})-(\d{1,2})-(\d{1,2})-r)rfinditerSIgroupsearchrrrCOL_PATHlowerendswithrlenzfillappendurllibparseurljoinBASE_URL) html_textitemsmliamhreftmtitledmdates r parse_listr[asv E [[DiQSQUQUXZX\X\Q\ ]J WWQZ YY@"bddRTTk R   !-335 4 tzz|'<' @Q@QRS@TU fll**8T:E4HIJ Lrc tjd|tjD]S}tjj |t j|jdjcSy)u_跟随「下页」链接（⚠️「尾页」也是 class=Next，必须按链接文字判断）u-]*href="([^"]+)"[^>]*>\s*下页\s*r N) rrBrDrMrNrOrrrEr)rQcur_urlrSs r next_page_urlr^vs^ [[I9VXVZVZ [T||##GX->->qwwqz-J-P-P-RSST rc '()|sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}g'i(tjdd|}tr t |d }|j d d D]}|d }|j ds?|j ds.tjd|tjs d|vsd|vsY|xsdj}|j dxs|j dxsdj}|r(tjd|tjs|}n|xs|}|jd xs[|j dxsH|j dxs5tjj|jddxsd}tj j#|} 'j%| |fdt'(z} d| d|d(| <|j)| |j dD]`} | j d xs| j dxsd} tjd!| tjsQ| j+b|j d"D]@} | j d}|s| jd r+g}|D]} | j d xs| j dxsd} | s.tj j#| } | j d#xs| j dxsdjxs5tjj| jddxsd$}|j%d| d|d|sd%t'(z} d&d'j-|zd(z(| <| j)| C|j dD]} | j d xs| j dxsd} | s.tj j#| } | j d#xs| j dxsdjxs5tjj| jddxsd$}d%t'(z} d)| d|d*(| <| j)| t/|}'(fd+}tjd,||tjtjz}tr< t |d }|j d D]%}d-D]}|j2j5|d. '|j d/D]}|j7d/^|j7d0M|jd s3|j7ds"|j7d s|j+jd"|_r|jd sD|j7d0s3|j7ds"|j7d s|j+|j;t/|}g))fd1}tjd2|tjrd3nd}d.}||k7r@|}tjd4||tjtjz}||k7r@tjd5d&|tj}tjd6d7|tj}tjd8d9|tj}t=d:D]P}tjd;d&|tj}tjd|tj}tjd?d|tj}tjd(d7|tj}tjd@dA|tj}tjdBd|tj}g}|jd9D]}|j}|stjdCd|}tjdDd|}|j?dEdj?dFdG}tjdHdG|}tjdId|}tAjB|}|j?dJdGj?dKdG}tjdLd|}|stjdM|r(jED]\}}|j?||}tG)D]\}}|j?dN|z|}tjdO|tjs*tjdP|tjsdQ|z}|j%|jd9j-|} (jED]\}}| j?||} tG)D]\}}| j?dN|z|} tjdRd| } tAjB| } tjdSdA| } tjdTd9| } tjdHdG| } tjdUd| } tjdVdW| tjtjz} tjdXd| } tjdYd| } tjdHdG| } tjdTd9| } t=d:D]P}tjd;d&| tj} tjd/，附件转

    绝对URL

    段落rrz )flagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】 html.parseraT)rVrVappendixz data-appendixz&\.(pdf|docx?|xlsx?|zip|rar|wps|et|ofd)virtual_attach_filez/module/download/oldsrcOLDSRCz^(?:javascript:|mailto:|tel:|#)r_titlerX?附件z__ATTACH__%d__ imgsrcz(icons?/|fileTypeImages/icon_)paltu图片z __IMG__%d__z

    z
    z

    z

    *s<00ELcR S rz&]*href="([^"]+)"[^>]*>(.*?)) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingNdivtablecfj|jddtdz zS)Nr __TBL__%d__r )rLrErJ)rS table_protects r _tbl_replzhtml_to_text.._tbl_repls.QWWQZ(M 2Q 677rz ]*>r zz ]*>zz

    z]*>z rz]*>\s*]*>z\s*z]*>\s*()z\1z]*>\s*$z  z8]*>z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  rz [ \t]{2,}r z z[\s\u200b\u200c\u200d\ufeff]+u^[【\[（(]?\s*(?:返回顶部|回到顶部|返回列表|打印(?:本页)?|关闭(?:窗口)?|分享|扫一扫|上一篇|下一篇|顶部)\s*[】\]）)]?$r]z<(?:table|ul|ol|h[1-6])\bz

    %s

    z __(?:LINK|TBL|IMG|ATTACH)__\d+__z[ \t]+\nz\n{3,}u已下载\s*\d*\s*次u附件【\s*()\s*】z

    \1

    u附件【\s*】z

    \s*

    z]*>\s*z

    \s*\nr)&rrrCrDrfind_allr/rFrrsget_textrurvrwrxrMrNrOrLrJ replace_with decomposejoinr5r4attrspopfindnameunwrapr)rrrrR enumeratesetadd)*contentr_souprbrV_href_oldsrc real_hrefrzr{r|rnrorpimgssrcsrtagattrdr has_tableprev_partsblockbplainkvitbloutseenfinaldedupseen_uurr}r~rs* ` @@@r html_to_textr}s 1byff]Brtt>CQXZ](^L%NN3'# ($~~e, $wwu~@):@b99>RTTJMMO $^^C( ,zz%(  6D#\!ggenG0AGR"$"(,,"6"6x"E"wwu~G1AGRNNPO!ww// #q0ABOFN WVY$Z[\+c,.??,1GLL4F,F,O S)s+ , ~~e, &ggen?(9?R ,,..x=wwu~?)9?RFFHGww'' #q(9:G>F#c,&77T[]`$a S!  % &%jGff> BDD244K1G !'=9E~~d+ .O.DIIMM$-. .^^E* 66%=(QVVG_-D::D:1!&&-PQPVPVWZP[ !$$/w !u affSkKKMHHJ %jGM8YY>AI D '/&&;Y rtt 5 '/ff^UG244@Gff[*gRTTBGff,fgRTTJG 1XL&&3UG244P&&-vwbddKLff.wbddKGff&G244@GffWj'>Gff\4=GffPRTBDD*G Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  88rty z  &&( DAq !QA  . 2FAs -!+S1A 2 )Q- ">244HaA QWWY1 4 ++e C""$ 1kk!Q M*23kk-!+S12 &&4b# >C   C C &&dC (C &&FC (C &&sC (C &&)2s 3C &&3\3bddUWUYUYk ZC &&#R -C &&C (C &&sC (C &&FC (C1XDff/244Hff)63bddCD &&("c >C &&eS 5C1XDff/244Hff)63bddCD &&("c >C%%D YYv ffVR B!:;cTk    Q  ++e  " " $C6E1 ;  1  aV   5  q   T   sG-A7y(%F+y(:y( y(Cy(zparse_detail..us&C!s1v&Csu'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2}))rrrcompilerDr/rrFrErKrrr select_onercontentsr4) rQrrXrZ content_htmlsoupmtmdrYrttrels r parse_detailrZs0"$$E$D< D  M:D6&"**_bdd2S)TUBbffY'#ByM26&"**Z2N)OPBbffY'YY=r)}M)+!bhhqk6G6G6JBHHUVKL]L]^_L`aDYYt_' Ct (DEEYYw'#BKKdK$;dt@|r-tCjD|Dcgc] \}}||d c}}dndd }tFr|jI|n|jI||dz }td|dd|||rdnd|rdt+|zndfzdtKjLd tFs|r||t!g}tO| |}|r||vrtd"dn|}tKjLd#tFrltQtFd$d%&5}|D]+}|jStCjD|dd'z- dddtd(tFt+|fzdn|r||t!td)|||fzdtd*tzdycc}}w#1swYgxYw)+Nr)push_to_searchdbz"=== %s (%s) pages=%d cutoff=%s ===Tr%r zPage %d fetch error, stopzPage %d: empty, stopu共(\d+)条\s+(\d+)/(\d+)zPage %d: found %d items%s [%s]u 页码r#/ru 共u条rrrz]r z skip empty/placeholder: %s()rXr9F) ensure_ascii) site_name source_urlr9rXpub_datesummaryrcategorytags group_namer}u + %s (%s, %d字%s%s)$u, 表格u , 附件%dg?) batch_labelu → 无下页，停止g333333?wr)r2rzJSONL written: %s (%d rows)u*新增: %d | 跳过: %d | CUTOFF截断: %dz=== %s done ===)*sysrvinsertrudirnameabspath__file__ crawler_librr8 SITE_NAME SCRIPT_NAME_PAGESCUTOFFrLIST_URLr)r?r[rrFrJrErrPrrrrrrDCATEGORY GROUP_NAMEjsondumpsrrLr6r7r^openwrite)r new_count skip_count cutoff_count seen_urls jsonl_rowsbatchr9pagerQrRnavr{rXrZdhtmldtitleddaterrratts plain_lenno_pararritemnxfrows rmainrsHHOOArwwrwwx'@AB, .)[&RX1Y Y-.-I- \IJ E Ca!$A#J  -4D A 9% (4/t < ii4i@ . #e*VY ! ciilCIIaL Q_a KK" %2''/3 4 %*(  GUD)# MM' "v ! 'NEa *6ug*F 'FE<'3L''J $GYBFF:r7;AACDI99Y>>U!yyGRTTBBUKT} 1}4vcr{B$Oa &ggUr""(VZ $zz8<=1q+=E S`b D!!$' T" NI *fSbk5);DZ"IM\CI-ESU.WW^b d JJsOQ( Re U <E 9c *R9_ ,D 9  3CAF *cG 4 D! D 3U;dBC D D +z3z?.KKSWX  U < 6)ZQ]9^ ^fjk i 't4A>. D Ds6Q"01QQ__main__))1__doc__rrurr6rhtmlr urllib.parserMrrbs4r ImportErrorenvironr/rr1rrrrrPrrGnowstrftimerr( _MAX_PAGESrargvrrbrtintrx ValueErrorrJrrr?r[r^rrr__name__rrrs(?>>>(!ZZ^^L" -   0  !  # 1  (,,.9'2 2 <