Xj(dZddlZddlZddlZddlZddlZddlZddlZddlZ ddl m Z m Z ddl m Z dZdZdZdZe j$e d z j'd Zd Zd ZdZdZdZdZdZdZdZdZdZe dk(reyy)ucrawl_wenjiang_sthj.py - 成都市温江区-生态环境 站点: https://www.wenjiang.gov.cn (瑞数4代: 412 + $_ts; 证书过期须 ignore_https_errors) 列表: /gkml/sthj/detail-list/column-index-1.shtml?page=N (静态 ul.list>li.myflex) onclick="window.open('/gkml/sthj/{id}.shtml')" 提取详情链接 字段: .cont 标题 / .unit 部门 / .date 日期 分页: column-index-{N}.shtml (路径式, 同 longquanyi) N)datetime timedelta)sync_playwrightu成都市温江区-生态环境zhttps://www.wenjiang.gov.cnzIhttps://www.wenjiang.gov.cn/gkml/sthj/detail-list/column-index-{pn}.shtmliG)daysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36z/tmp/wenjiang_sthj.jsonlz/root/search.dbz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chromezcrawl_wenjiang_sthj.pyctjddD]O}|jdrt|j ddcS|j sDt|cSt S)Nz--pages==)sysargv startswithintsplitisdigitMAX_PAGES_DEFAULT)as (/root/gov_crawler/crawl_wenjiang_sthj.pyparse_pages_argrsX XXab\ << #qwws|A' ' 99;q6M  ctjdd|xsdjjddjddS)Nu ^[•··\s]+u​u)resubstripreplace)ts r clean_titler$s? 66"BR 0 6 6 8 @ @2 N V VW_ac ddrcjg}|jd}|D]}|jdxsd}tjd|}|stjd|}|sJ|j d}|j dr t |z}n|j ds t dz|z}|jd }|jd }|r|jjnd} |r|jjnd} | s|jt| || d |S) u'从 li.myflex 提取 (title, url, date)zul.list > li.myflex, li.myflexonclickrzwindow\.open\('([^']+)'zwindow\.open\(\"([^\"]+)\"r /httpz.contz.date)titleurldate) query_selector_all get_attributersearchgroupr BASEquery_selector inner_textrappendr) pageitemslislirmhrefcontdate_elr"r$s r extract_itemsr5(s E  ! !"B CCS""9-3 II0' : 7AA wwqz ??3 $;D(#:$D  )##G,-1!'')r/6w!!#))+B  LL;u#5dDQ R#S$ LrcP|jdk(r&t|}tjdd|}|gS|jdvro|j drgS|j dd}|rtj d|rgStjd dt|jgS|jd k(rdg}|jgd d D]}|jt||s*|j dd}|r|jd|d|SgS)uV递归提取 HTML 块 (保持顺序): p/pre 保留内部HTML, table 保留, div 递归tablezhref=["\'](/[^"\']+)["\']c8dt|jddS)Nzhref="r ")r)r()mms rzextract_block..EstUWU]U]^_U`>ar)ppre Trud^(责任编辑|初审|复审|终审|\[纠错\]|浏览次数|字号|分享到|扫一扫|相关附件)z\s+divr<r=r7r@F recursivez

z

) namestrrr find_parentget_textmatchrfind_allextend extract_blockr,)eltbltxtblockscs rrKrKAs ww'"gff13acfgu  ww, >>' "Ikk#Tk*bhh GILMIvsCG,22455 ww%9UK ,A MM-* + ,++c+.C C5o. Irc|j}d}tjd|tj}|r4tjdd|j dj }|sFtjd|tj}|r|j dj }d}tjd|}|r|j d}|s)tjd|}|r|j d}d} dd lm}||d }|jd tjd tj }|s5|jd tjdtj }|s5|jd tjdtj}|r|jtjdtj D]} | jg} |jgddD]} | jt| g} | D]!} | s| r | d| k7s| j!| #dj#| }g} |jdtjdtjD]}|j%dd}tjdd|j'dj)dxs|j+dd}|j-dr t.|z}n|j-d s t.dz|z}| j!||d!| D]}d"|d#d$|d%d&}||vs|d'|zz } t1|j d(krd"|j2d$|d&}|||fS#t4$rYwxYw))uA详情页: title/date/content (附件型: 正文=附件PDF链接)rz]*>(.*?)z<[^>]+>r um(.*?)(?:\s*-\s*(?:重点项目清单|信息公开目录|成都市龙泉驿区人民政府).*)?z1rDz

 )contentrr'Srr(rbs4rRfindcompileIrI decomposerJrKr,joingetrGlstriprr r)lenr# ImportError)r-htmlr"r1 date_textrXrRsoupwzrLpartsdedupr< attachmentsrr2rDlinks r parse_detailrlZsv <<>D E ($5Az2qwwqz288:  IIGIMOQOSOS T GGAJ$$&EI FMAGGAJ  IIA4 H  IG$ %T=1 YYuRZZ0QSUSWSW%XY Y54SUWUYUY)Z[B5RZZ0NPRPTPT%UVB kk4Mrtt)TkU   Ekk">%kP 0 ]2./ 0E $euRyA~LLO $kk%(G s4qsuswsw)xy |d#rt]|d#j_d?d=nd8t`f |jbs|d z }|je|jCtd@|d|jC dddycc} w#t6$rYwxYw#t6$r9}td&|d zd't'|d,t=|dd-dYd}~Hd}~wwxYw#1swYxYw#|jCwxYw#1swYyxYw)ANz--dryrunz--jsonlu&[longquanyi] 重点项目清单 pages=z cutoff=T)flush)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)executable_pathheadlessargsiVi)widthheight) user_agentviewportignore_https_errorsr )pnu [列表] 第u页i`domcontentloaded)timeout wait_untilu 第u页 0 条, 停止u页 u 条u [列表] 共 r#iȯz/div.content, div.article, div.detail, div.wzconi:)ryr$r")r#r"r$rXz [r z] rU(z | u ] 失败: <u [完成] F) ensure_asciirzutf-8)encodingrVz[jsonl] u 条 -> zPRAGMA busy_timeout=300000rz'SELECT id FROM gov_raw WHERE page_url=?zINSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank, script_name) VALUES (?,?,?,?,?,?,?,?,?,?)u四川rXrz\d{4}-\d{2}-\d{2}-u[入库] 新增 )3rr r printCUTOFFrchromiumlaunchCHROMIUM new_contextUAnew_pagerangeLIST_TPLformatgototimesleepr5rbrJrandomuniformsetadd enumeratewait_for_selector Exceptionrlr,rEjsondumpscloseopenOUTwritesqlite3connectDB_PATHexecutecursorfetchone SITE_NAMErrHrr SCRIPT_NAMErowcountcommit) max_pagesdryrun jsonl_moder<browserctxr-r.rwr# page_itemsseenxresultsidxitr"rerXerfconnrP new_counts rmainrsL!I 388 #Fchh&J 29+XfX NVZ[  Qa**##$tn$p!!RDTW:Xnr!s||~K EAy1}- 1ooo,rd#.d; #u9K L a1Q/*40 !GB4'89Ft4J'8=TJ Z( 6>>!Q/0 1 M#e*T2$ ?5D %V1ah$.>$((1U8BTQVEVG$U+ YR 6>>!Q/0YIIbiCUIV../`jo.pJJqM0T?S-*S=8S-5T?6AS-AT? U#T?71T2)(T?U+CT?7T?>UT? S*&S-)S**S-- T/6.T*$T?*T//T?2T< 7T??UUU__main__)!__doc__r osrrrrrrdhtml_librrplaywright.sync_apirrr)rrnowstrftimerrrrrrrrr5rKrlr__name__rrrsBAAA(/ - $ V (,,.9%0 0 : :: Fv   K& e229%xWt zFr