Hj_*hdZddlZddlZddlZddlZddlZddlZddlZejjdejjejje ddl mZdZdZdZddd d d d d d dZdZdZddZddZdZedk(rej2eyy)u  crawl_tongcheng_hpgs.py - 桐城市生态环境分局 (www.tongcheng.gov.cn) 站点: CDN 521 状态但内容正常; playwright 过盾 列表: GET /site/label/8888 (注意: 无 huangshanzwgk 前缀, 与 ahshx 不同) labelName=publicInfoList&siteId=2000004241&organId={organ}&pageSize=10&pageIndex=N &isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&isJson=true&catId={cat} 详情: /public/{organ}/{contentId}.html, meta ArticleTitle/PubDate, 正文 div.wzcon.j-fontContent 栏目 (--col): hpgs : 桐城市生态环境分局-环评公示/意见征集 organ=2000002271 cat=6725591 (total=75) luting_yjzj : 桐城市吕亭镇-意见征集 organ=2000002421 cat=6736061 (total=14, 含规划公示/环评公示/征求意见) 进程锁: flock 站点级串行化 N)push_to_searchdbzhttps://www.tongcheng.gov.cn 2000004241z/tmp/tongcheng_hpgs.lock 20000022716725591u(桐城市生态环境分局-环评公示)organcatname 20000024216736061u桐城市吕亭镇-意见征集)hpgs luting_yjzjzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36czddlm}||d}|gdD]}|j|jdD]}|j dxs|j dxsd}|rH|j d r|nt |z}|jd | }d |_|j|u|j|jgd D]}|j|jd D]h}|j dd}|r0|j ds|j d r|nt |z|d<t|jD] } | d vs|| = j|jdD]f} t| jD]} | | =| jdr3| jdrE| jdrW| jh|jgdD]$}t|jD] } | dvs|| = &t|jD]_} | j!} | sd| vs d| vsd| vs#| j"} | r| j$dvr| jP| j'a|jd D]7}d|jvsd|jvs(|j9t|jD]9} | j!} | st)| dks%| dvs*| j';t+|S)u]白名单清洗: 去 inline style, unwrap span/font 装饰, 保留 p/table/a, 链接绝对化r) BeautifulSoupz html.parser)scriptstyleiframeobjectimgsrczdata-srchttpa)hrefu 查看图片) spanfontbstrongemiusdivr javascriptpT)striptable)r&trtdth)border cellpadding cellspacingu如果内容不能正常显示u下载本PDF文档updf浏览器插件)r$r"u链接另存为u鼠标右键点击) u]。u。]u】。】u。u）)]r.)bs4r decomposefind_allget startswithBASEnew_tagstring replace_withunwraplistattrsget_textfindstringsr%parentr extractlenstr)htmlrsouptrrabs_srcrrattrr$nodetxtr@s )/root/gov_crawler/crawl_tongcheng_hpgs.py clean_htmlrL s! } -D 9 : }}U#ggen9 39r  ^^F3cG Sw /A%AH   Q  MMO]]V W   ]]3 uuVR   5 $ 7TD[AfIM D9$dG  ]]3 M D$ zzz%affWoaffUm KKM  ]]6 7M DCCdG  T\\"jjl 4;?SWZ?Z^rvy^y[[F&++5  " ]]3   ,0D 0T KKMT\\"jjl 3s8q=S,l%l LLN t9c,ttd} tj|tjtj z|S#t $rtd|dtj|z}tj|krv tj|tjtj z|cYS#t $rtjdYnwxYwtj|krvtdYywxYw)u3站点级互斥锁, 阻塞等待最多 max_wait 秒wu< [LOCK] 另一 ahshx 实例运行中, 等待释放 (最多 zs)... u [LOCK] 等待超时, 放弃N) open LOCK_FILEfcntlflockLOCK_EXLOCK_NBBlockingIOErrorprinttimesleep)max_waitfddeadlines rK acquire_lockr^Us i B  B  56   LXJV[\]99;)iikH$  B  => "  2  iikH$ -. s<6A AD6C D C*'D)C**D DDc: Kt|}|d}|d}|d}tdtd|d|}td|d|d }d d lm}d d lm} |4d{} | jjd dgdd{} | jtdddddd{} | } | j| d{| jd{} |j|ddd{}|jdd{|jd{}t!j"d|}t%d|r |j&ndd|r|j)dndd t+||r[d!|j)dvsd"|j)dvr5t%d#| j-d{ dddd{y g}t3}d }t5d|dzD]]}|j7t1t9t;d&|d'|d(d|)}|j=d*|d+d{}d,|vrt%d-|d,n|j?d.d }|j?d/gxsg}|snt%d0|d1t+|d2|d3|D]}|j?d4d5}|s||vs|jA||jC|jEd6r|nt|z|j?d7d5jG|j?d8xsd5dd9d:t+||k\s^nt%d;|d|dzd?t+|d@|d7ddAdB |j|dCddDd{}|jdEd{|jd{}|d7}t!j"dH|}|r>|j)djGr|j)djG}|dI}t!j"dJ|}|r7t!j"dK|j)d} | r| j)}d5}! |j=dLd{}!|!jGs7t!j"dM|}|r|j)djG}!|!jGs t%dNtK|!}!|jC||dC|||!d5dO| j-d{dddd{t%dPt+dQ|rtM|dR|t+|S7a7<77777777#t.$rN}t%d$t1|dd%| j-d{7Yd}~dddd{7y d}~wwxYw77e7O7:#t.$r%}t%dFt1|ddGYd}~d}~wwxYw7#t.$rYwxYw77#1d{7swYxYww)SNrrr z/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNii)widthheightz Asia/Shanghai) user_agentlocaleviewport timezone_iddomcontentloadedi) wait_untiltimeouti.z([^<]*)z [LOAD] status=?z title=z len=NWAF Environmentu [WAF] 仍被拦截z [LOAD ERR] x tongcheng__i)randpagezOasync () => { try { const r = await fetch('zv'); return await r.json(); } catch(e) { return {error: e.message}; } }errorz [API ERR] totaldatau 第u页: u 条 (total=r/linkrrtitle publishDaterP)urlr{datez [LIST] total=u , 去重后 u 条z [/z] (z...r}i0ui z [DETAIL ERR] dz/]*>([\s\S]*?)\s*u [SKIP] 正文空)r{r}pub_date site_namecontentsummaryu 共 u 条有效ahshx_)'COL_MAPr6SITE_IDplaywright.async_apir`playwright_stealthrachromiumlaunch new_contextUAapply_stealth_asyncnew_pagegotowait_for_timeoutrresearchrXstatusgrouprBclose ExceptionrCsetrangeformatabshashevaluater4addappendr5r% enumeraterLr)"col_keypagescolorgan_idcat_idrAPI_TPLLIST_URLr`rar$browserctxstealthrvresprDme items_allseenrxpgapi_urlryitemsitrzrecordsidxr{rdmrs" rK run_asyncrhs ' C7|H ZFF I33:)9XJbbhaikGxjvhFXYH5*!kkQ ))\n*   ''2g;?31O]l(nn)))#...\\^# 88JTYZZD''. . .'D 3T:A $DT[[c$B'XY!''RS*_bIcchilmqirhst uf *mqwwqz.I,-mmo%%-kkk: u519% Bnn#c$G9AbT7R2S.T*UVYVZ*[bdneG,,,395(D $ T']O45HHWa(EHHVR(.BE E"U3u:,l5'C D vvfb)D,HHTN$$'+v'>tD4K!#!4!:!:!2B$GH +* GC CAwaI/r"W+cr2B1C3G H !YYr%y=OY^Y__++D111!\\^+ wKE LdSAQWWQZ%%' ((*&zH GNAYY3QWWQZ@!xxzHG  $ /S!TT==?IIbdhiggaj..0G==?,- )G NN%y$&"  G* VmmoWkkZ F3w<. +,F7)"45 w<ak n /#[ .' &+k.  M#a&#,0 1--/ ! !5kkk. B`1+ )#a&#,89 "T  & WkkkksAZV7Z$Z>V:?#Z"V=#ZWZWZ W7W8WW W(W )BWWW ZWZA0ZX/A>ZZBZAZ3X; X2X;&X5'X;>X8?X;B1Z5Y/ Y, Y/B Z.Y?/Z3 Z>Z?9Z:Z=ZZZW W WWZ X,-X' X X'Z Z X#!Z'X,,Z2X;5X;8X;; Y)Y$Z$Y))Z,Y// Y<8Z;Y<<ZZZ Z ZZc ddl}|j}|jddddjtj z|jdt d |jd d |j}|jtvr7td |jdttj yt}|y t|j}td|dd|dd|dd|jtjt|j|j s |jnd}td|d t#j$|t"j&|j)y#t#j$|t"j&|j)wxYw)Nrz--colxitou_zbu 栏目键: ,)defaulthelpz--pagesro)typerz--all store_true)actionu[ERR] 未知栏目 u , 可选: z r z (organ=rz cat=rz) pages=2)rzDone: z records)argparseArgumentParser add_argumentjoinrkeysint parse_argsrrXr;r^rasynciorunrallrSrTLOCK_UNr)rparserrdlock_fdrcnts rKmainrsr  $ $ &F -#((SZS_S_SaJb:bc  Q7  5    D xxw #DHH:ZW\\^8L7MNOnGdhh 3v;-xG ~U3u:,htzzl[\kk)DHHdhhDJJTVWX se8$% GU]]+   GU]]+ s BF##6G__main__)i)ro)__doc__sysosrrjsonrSrYpathinsertdirnameabspath__file__ crawler_librr6rrRrrrLr^rr__name__exitrMrKrs /...277??277??8#<=>(%  & #9>h i))Efg  w2j&~B2 z CHHTVrM