jdZddlZddlZddlZddlZddlZddlZejjdejjejje ddl m Z dZdZdedzdd d ed zd d d edzdd dZej"dZddZdZedk(rej,eyy)u crawl_cnqc_v2.py - 青川县人民政府多栏目 (cnqc.gov.cn) 站点: 四川广元青川县 (瑞数4代+帝联NWAF 双WAF, 与 ahshx/gdqy 同版本) 过盾: playwright-stealth, 全程 goto (fetch 被瑞数拒, goto 过盾后 ~7s/页) 详情: meta ArticleTitle/PubDate/ContentSource, 正文 div.msg-content (截断到 div_div 前) 限速: 详情间 3-8s 随机抖动 栏目 (--col): zdly : 重点领域信息公开 /New/List/20171128112728617.html?page=N (已爬75) gsgg : 公示公告 /new/list/20171130181953989.html?page=N (50页~750条) hjbh : 环境保护-政府信息公开 /gongkai/list/20220811145224789.html?page=N&t=31 (31页~465条) hjbh 详情: /gongkai/show/{catId}/{contentId}.html (其他栏目 /New/Detail/{id}.html) N)push_to_searchdbzhttp://cnqc.gov.cnzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36u"青川县-重点领域信息公开z,/New/List/20171128112728617.html?page={page} cnqc_zdly) site_namelist_urlbatchu青川县-公示公告z,/new/list/20171130181953989.html?page={page} cnqc_gsggu(青川县-环境保护政府信息公开z5/gongkai/list/20220811145224789.html?page={page}&t=31 cnqc_hjbh)zdlygsgghjbhz~]*href="([^"]*(?:Detail|show)[^"]*\.html)"[^>]*title="([^"]*)"[^>]*>[\s\S]{0,200}?\s*(\d{4}-\d{2}-\d{2})cKt|}|d}ddlm}ddlm}|4d{}|j j ddgdd{}|jtd d d d d d{}|} | j|d{|jd{} | jtdzddd{} | jdd{| jd{} tj d| } t#d| r | j$ndd| r| j'dddnddt)| | rHd| j'dvr5t#d|j+d{ dddd{yg}t1}t3d|dzD]%} | j|dj5| dd!d{} | jd"d{| jd{} t6j9| }d}|D]`\}}}|j;d&r|nt|z}||vs(|j=||j?||jA|d'|dz }bt#d(|d)t)|d*|d+t)|d, t)|d-kst#d.nt#d/t)|d0g}tC|D]\}}t#d1|dzdt)|d$|d2dd3d4 | j|d5dd!d{} | jd6d{| jd{}d|dd7vr t#d8 |d2}tj d:|} | r>| j'djAr| j'djA}|d;}tj d<|} | r7tj d=| j'd}|r|j'}d>}|d?k(rqtj d@|} | r| j'djA}|stj dA|} | r| j'djA}nptj dB|} | r| j'djA}|s7tj dC|} | r| j'djA}|jAs t#dDGtjDdEdFtzdz|}tjDdGdHtzdz|}|j?||d5|||d>dI|t)|dz kstGjHtKjLdJdKd{|j+d{dddd{t#dLt)dM|rtO||dNt)|S7/7 777777k77#t,$rN}t#dt/|dd|j+d{7Yd}~dddd{7yd}~wwxYw777#t,$r)}t#d#|d$t/|dd%Yd}~d}~wwxYw777#t,$r%}t#d9t/|dd%Yd}~/d}~wwxYw7T7<7/#1d{7swY@xYww)ONrr)async_playwright)StealthTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNii)widthheightz Asia/Shanghai) user_agentlocaleviewport timezone_id/domcontentloadediȯ) wait_untiltimeouti.z([^<]*)z [LOAD] status=?z title=(z len=u 请稍候u [WAF] 挑战未过z [LOAD ERR] xr)pagei@ipz [LIST ERR pz] dhttp)urltitledateu 第u 页: 解析 u 条, 新增 u (累计 ) u [END] 列表结束u [LIST] 去重后 u 条z [r%$z...r$iiu [WAF] 详情被拦z [DETAIL ERR] z/]*>([\s\S]*?)\s*]*class="[^"]*(?:foot|qr|ewm|div_div)z*class="_srcoll[^"]*"[^>]*>([\s\S]*?)z=
([\s\S]*?)
\s*
([\s\S]*?)
u [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")r%r$pub_datercontentsummaryg@g @u 共 u 条有效r)(COL_MAPplaywright.async_apirplaywright_stealthrchromiumlaunch new_contextUAapply_stealth_asyncnew_pagegotoBASEwait_for_timeoutr,researchprintstatusgrouplenclose ExceptionstrsetrangeformatLIST_REfindall startswithaddappendstrip enumeratesubasynciosleeprandomuniformr) col_keypagescolrrrpbrowserctxstealthr!resphtmlme items_allseenpgrowsnew_rowsutdr$recordsidxitdhr%r+dmr,s "/root/gov_crawler/crawl_cnqc_v2.py run_asyncrk-s ' CK I5*!ooQ ))\n*   ''2g;?31O]l(nn)))#...\\^# 4#::LV[\\D''. . .'D 3T:A $DT[[c$B']^!''RS*UXVX/dgIhhmnqrvnwmxy z[AGGAJ.,-mmo%%-ooo: u519% B !YYs:'='=2'='FSeotYuu++D111!\\^+??4(DH "1a<</aTAXd?HHSM$$S1779a%PQMH  " E"\#d)M(9UXYbUcTddef g4y2~,-' * #C N#3489 +6 >GC CAwaI/r"W+cr2B1C3G H !YYr%y=OY^Y__++D111<<>)"Ud)+23, wKE LbQAQWWQZ%%' ((*&zH GLAYY3QWWQZ@!xxzHG& IIz|~ggaj..0G "OQSTA"#''!*"2"2"4II^`bcggaj..0G "NPRSA"#''!*"2"2"4==?,-ffZD3)>HGffY$(nmmo_oob F3w<. +,#g,/ w<io n /#] .' &+o.  M#a&#,0 1--/ ! !5ooo. v1+  bTCF4CL>:; .`1) )#a&#,89 Z>_oooos)&]&Y]&$]Y#]4Y5]Y ],Y#-]2Y5Y&Y5)Y)*Y5Y,BY5Y/Y5 ]&!Y2"]&']*[1[2[ [ ["[#['A](A3]A!]?\\ \2\3\ \ \!H$],]3]4]] ] ]&] 9]&]] ]#]&Y5)Y5,Y5/Y52]&5 [ >-[+Z.,[1]5 ]&[]&[  ][[[ \ !\>]\  ] \\\ ]\?9]?]] ] ]&]#] ]#]&c ddl}|j}|jddddjtj z|jdt d |jd d |j}|jtvr=td |jddjtj y|jrdn |j}tdt|jdd|tjt|j|}td|dy)Nrz--colr u 栏目键: ,)defaulthelpz--pages)typernz--all store_true)actionu 未知栏目 u , 可选: r2z rz pages=)rSzDone: z records)argparseArgumentParser add_argumentjoinr.keysint parse_argsrTr<allrSrNrunrk)ruparserrrScnts rjmainrs   $ $ &F mchhw||~F^6^_  Q7  5    D xxw dhhZz#((7<<>2J1KLM((B E Bwtxx -.geW => ++i6 7C F3%x ! __main__)rp)__doc__sysosr:rNrPtimepathinsertdirnameabspath__file__ crawler_librr8r4r.compilerFrkr__name__exitrrjrs *))277??277??8#<=>(v:II  .II  @RR  $ "**F  zz " z CHHTVr