j.dZddlZddlZddlZddlZej j dej jej je ddl m Z dZ dZ e dZdZd dZd Zed k(rej&eyy) u crawl_rongcheng_gggs.py - 荣成市人民政府-公告公示 (col61226, number=RCA21) 站点: www.rongcheng.gov.cn (TRS 系, 无 WAF, curl 可直连列表页但 search.jsp 需浏览器上下文) 列表: 页面 div56364 由 loadDynamic('/module/xxgk/search.jsp?infotypeId=RCA21...') 渲染 翻页: 调用页面全局 funGoPage('/module/xxgk/search.jsp', N) (AJAX, 不刷新页面) curl 直连 search.jsp 全部被拒("请传入正确参数") → 必须 playwright 页面内操作 每页 20 条, 共 7727 条/387 页 详情: /art/YYYY/M/D/art_61226_XXXX.html meta ArticleTitle/PubDate, 正文 到 , 容器 div.xqwz N)push_to_searchdbzhttp://www.rongcheng.gov.cnu荣成市-公告公示z%/col/col61226/index.html?number=RCA21zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36cb Kddlm}|4d{}|jjddgdd{}|j t ddd d d d{}|j d{} |jtd dd{}|jdd{td|r |jndg}t} t!d|dzD]} | dkDr> |j#dt$d| dd{|jdd{|j#dd{} d} | D]]} | dj'dr| dn t$| dz}|| vs-| j)||j+|| d | d!d"| dz } _td#| d$t-| d%| d&t-|d t-| d'kstd(ng}t/|D]L\}}td)|dzd*t-|d|d dd+d, |j|d-d d.d{}|jd/d{|j1d{}|d }t3j4d1|}|r>|j7dj9r|j7dj9}|d!}t3j4d2|}|r7t3j4d3|j7d}|r|j7}d4}|j;d5}|j;d6}|dkDr$||kDr||t-d5z|j9}|sRt3j4d7|}|r:|j7d}|j;d8}|dkDr||dj9}|j9s td9t3j<d:d;t$zd*z|}t3j<d|d4d>O|jd{dddd{td?t-d@|r tA|dAt-|S77v7U7@7"7 #t$rN}tdt|dd|jd{7Yd}~dddd{7yd}~wwxYw77#t$r)}td| dt|ddYd}~Ud}~wwxYw7 7 77#t$r%}td0t|ddYd}~d}~wwxYw7*7#1d{7swY.xYww)BNr)async_playwrightTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNii)widthheightz Asia/Shanghai) user_agentlocaleviewport timezone_iddomcontentloadedi@) wait_untiltimeouti@z [LOAD] status=?z [LOAD ERR] xz funGoPage('z/module/xxgk/search.jsp', )ipz [PAGE ERR pz] da() => { const div = document.getElementById('div56364'); if (!div) return []; const out = []; const seenHref = new Set(); for (const tr of div.querySelectorAll('tr')) { const a = tr.querySelector('a[href*="/art/"]'); if (!a) continue; const href = a.getAttribute('href'); if (!href || seenHref.has(href)) continue; seenHref.add(href); const txt = tr.textContent.replace(/\s+/g, ' ').trim(); const dm = txt.match(/(\d{4})-(\d{2})-(\d{2})/); out.push({ href: href, title: a.textContent.trim(), date: dm ? dm[0] : '' }); } return out; }hrefhttptitledate)urlrru 第u 页: 读取 u 行, 新增 u (累计 u [END] 列表结束z [/&z...ri0ui z [DETAIL ERR] z/zz9
([\s\S]*?)(?:
)uu [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")rrpub_date site_namecontentsummaryu 共 u 条有效rongcheng_gggs)!playwright.async_apirchromiumlaunch new_contextUAnew_pagegotoLIST_URLwait_for_timeoutprintstatus ExceptionstrclosesetrangeevaluateBASE startswithaddappendlen enumerater"researchgroupstripfindsub SITE_NAMEr)pagesrpbrowserctxpagerespe items_allseenpgrowsnew_rowsrrrecordsidxithtmlrmr dmr"sieisegsi2s )/root/gov_crawler/crawl_rongcheng_gggs.py run_asyncr[s5!ssQ ))\n*   ''2g;?31O]l(nn\\^# 88JTYZZD''- - - $DT[[c$BC D u519%' BAv--+dV;UVXUYYZ([\\\//555(D*H "#$V9#7#7#?aiTAfIEUd?HHSM$$S1W:qQWy%YZMH  " E"\#d)M(9UXYbUcTddef g4y2~,-O' T +0 GC CAwaI/r"W+cr2B1C3G H !YYr%y=OY^Y__++D111!\\^+ wKE LdSAQWWQZ%%' ((*&zH GNAYY3QWWQZ@!xxzHG78B56BAv"r'rC(D$EEbIOOQIIZ\`a''!*C((#BCCQw"%cd)//"3==?,-ffZD3)>HGffY$(?L`1+ )#a&#,89 V gsssssV/R/V/$VR2#V R5!V8R89V>SR;S2R>3 S$V8T!TT!0T1T!5V U 4V?A+V+A V5U"UU"(U)U"UU"G$V)V*V. V/9V:6V/2V5V8V;S>S T -T7S:8T=V V/ T V/TVT!T!! U*UVUVU"U"U"" V+V V VVV/V,V" V,'V/c\ddl}|j}|jdtd|jdd|j }|j rdn |j }td td |tjt| }td |d y)Nrz--pages)typedefaultz--all store_true)action2z z pages=)rCzDone: z records) argparseArgumentParser add_argumentint parse_argsallrCr.rBasynciorunr[)rcparserrrCcnts rZmainrms  $ $ &F  Q7  5    D((B E Byk () ++ie, -C F3%x ! __main__)r])__doc__sysosr<ripathinsertdirnameabspath__file__ crawler_librr6rBr,r)r[rm__name__exitrnrZr|s 277??277??8#<=>($ $ V8 9vzz   z CHHTVrn