4jo% N d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl
m Z ddl m
Z
ddlmZ dZdZd Z ej& e d
z
j) d Zej, j/ d
d Zdddddddddddd Zdd iZd Zd Z d Z d Z d/d!Z d0d#Z e!d$k( rejD d%d Z#d"Z$dZ%dZ&d&d'jO e# v r7 e( ejR d(d'jO e# jU d% Z$d)d'jO e# v r] ejR d*d'jO e# jU d% Z+e+d+k7 re+jY d, n e-ej] Z&d-e#v re#e#j_ d- d%z Z% e e&e$e%. yy)1um
仪征市乡镇人民政府 - 通知公告 ×11 (yizheng.gov.cn/zfxxgk/yqxzbmxxgkml/yqxz/<镇>/fdzdgknr/tzgg/)
CMS: 翰Web/unitbuild JS 渲染 (Playwright 必须)
列表: Playwright 渲染 + 点击翻页 (共~22页/镇, 新规只抓前5页)
列表项: a href=".../tzgg/art/2026/art_{md5}.html"
详情: requests + bt-content/article-content/content
用法:
python3 crawl_yizheng_xz_tzgg.py # 全部11乡镇
python3 crawl_yizheng_xz_tzgg.py --col=xjz # 单乡镇
python3 crawl_yizheng_xz_tzgg.py --col=all --pages=5
python3 crawl_yizheng_xz_tzgg.py --json out.jsonl # JSONL模式(不写DB)
N)datetime timedelta)
BeautifulSoup)urljoin)sync_playwrightzhttps://www.yizheng.gov.cnzwww.yizheng.gov.cnu 江苏iG )daysz%Y-%m-%dDB_PATHz/root/search.dbu 新集镇u 马集镇u 陈集镇u 刘集镇u 大仪镇u 月塘镇u 新城镇u 青山镇u 枣林湾旅游度假区u 经济开发区u 真州镇) xjzmjzcjzljzdyzytzxczqszzlwlydjqjjkfqzzz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c t | d y )NT)flush)print)msgs */root/gov_crawler/crawl_yizheng_xz_tzgg.pylogr # s #T c t j dd| xs d } t j dd| } t j dd| j } | S )Nz | |\u200b|\ufeff z\s+ u \.\.\.|…$)resubstrip)ts r clean_titler$ & sK
-r17;A
vsA A
~r1%++-A Hr c < t d| d}g }t }t 5 }|j j ddg }|j t d }|j } |j |dd
|j d d
t d|dz D ] } |j d }
d} |
D ]c } | d j d r| d n t | d z }
|
|vs-|j |
|j t! | d |
dd | dz
} e t# d| dt% |
d| dt% | | |k s|j d }|s n) |j' d |j) d d d |j+ d d d |S # t $ r Y !w xY w# t $ r Y 8w xY w# t $ r Y Fw xY w# t $ r}t# d | Y d }~fd }~ww xY w# |j+ w xY w# 1 sw Y |S xY w)!uA Playwright 渲染列表 + 点击翻页, 返回 [{title,url,date}]z/zfxxgk/yqxzbmxxgkml/yqxz/z/fdzdgknr/tzgg/index.htmlTz --no-sandbox)headlessargsr )
user_agent networkidlei@ )
wait_untiltimeoutz a[href*="/fdzdgknr/tzgg/art/"]i: )r+ a () => {
const out = [];
document.querySelectorAll('a').forEach(a => {
const href = a.getAttribute('href') || '';
if (!href.includes('/fdzdgknr/tzgg/art/')) return;
const t = (a.getAttribute('title') || a.textContent || '').trim();
if (t.length >= 6) out.push({t: t, h: href});
});
return out;
}r hhttpr# r )titleurldatez [Pz] z links, z new, total uP () => {
const links = Array.from(document.querySelectorAll('.page a, .pagination a, a'));
const next = links.find(a => (a.textContent||'').trim() === '下一页');
if (next) { next.click(); return true; }
return false;
}i a^ (cnt) => {
let n = 0;
document.querySelectorAll('a').forEach(a => {
if ((a.getAttribute('href')||'').includes('/fdzdgknr/tzgg/art/')) n++;
});
return n > 0;
}N@ )argr+ z Playwright ERR: )BASEsetr chromiumlaunch new_contextHEADERSnew_pagegotowait_for_selector Exceptionrangeevaluate
startswithaddappendr$ r lenwait_for_timeoutwait_for_functionclose) town_code max_pagestown_urlitemsseenpbrowserctxpagepgdatapg_newitr0 clickedes r fetch_list_playwrightrV , sW 1) "mm - G # --d3 ! 2 2 4! '+D
3 :C+
^
MMOu: v Le
T ) ! !$ * $QC( ) ) *
MMOu: v Ls AH,GF)AG(A'GG&G8F9 GH) F62G5F66G9 GGGG GGGG G9!G4/G<4G99G<<HHHc z t j | t d }d|_ |j }t
|d }dddd}|j d }|r t |j d |d< t j d
|d d }|r|j d |d
< d }dD ] }|j d| }|s n |s(|j dd xs |j dd }|r7 |ddg D ] } | j t t | | |d< |S # t
$ r ddddcY S w xY w)N )headersr+ utf-8r )r/ publish_datecontentlxmlr/ Tr" z(\d{4}-\d{2}-\d{2})r2 r, r[ )z
bt-contentz
bt-articlezarticle-contentr\ z page-contentztheme-showcasediv)class_zoom)idz UCAP-CONTENTscriptstyler\ )requestsgetr9 encodingtextr= r findr$ get_textr searchgroup decomposeextract_clean_textstr)
r0 rhtmlsoupresultttagm content_divclstags
r parse_detailry n sO @ LLgr :
vv v &D2"
=F 99W D %dmm$m&?@w
($u+6A!"~ Kk iici2 ii&i1XTYYu Y5X ' 23 C MMO .s;/? Ey M+ @RB??@s /D( (D:9D:c | syt | d }|j dd D ]V }|j dd }|j d xs d}t |xs t
| }|j
d | d
| d X |j g d D ] }|j g }|j d
dg D ]y } | j d
k( r|j t | -| j dk( s=| j d
rO| j dd }
|
se|j d|
d { |rdj | S | j S )Nr z html.parseraT)hrefr| r^ u 附件z z
)spanbstrongfontemiustablerL ) separatorr" zz
z
)r find_allrf rj r r4 replace_withunwrapnamerB ro find_parentjoinr" ) content_htmlpage_urlrr a_tagr| rh full_urlrx partselr# s r rn rn sF } 5D s . Fyy$~~D~)5X8+tT2
\(2dV8DE F
}} QR
EmmWcN+ ,
77g LLR !
WW^BNN7$; b 5A s1#T]+
, "'6;;u @L,>,>,@@r c
( | t t j } g }| D ] }t j || }d| d}t d| d t || }t dt
| d t |d D ] \ }} t | d }
|
d r
|
d t k r&|j |
d
xs | d
| d |
d xs | d |
d |d
d |dz dk( rt d| dt
| t j d t dt
| d |rct |dd 5 } |D ]+ } | j t j | d dz - d d d t d t
| d | y t# j$ t& }
|
j) d |
j+ }dx}}|D ]W } |j) d | d | d! | d! | d
| d | d | d xs d"d d# | d$ d%f |j, dkD r|dz
}n|dz
}Y |
j1 |
j3 t d'| d(| d) y # 1 sw Y xY w# t. $ r }t d&| |dz
}Y d }~d }~ww xY w)*Nu 仪征市u
-通知公告z
===== z =====u
列表共 u 条, 开始抓详情...r, r0 r[ r/ r1 r\ u 通知公告)r/ r r[ r\ site_namecolumn
r z /g?u
共 u 条wrZ )rg F) ensure_ascii
zJSONL: z
items -> zPRAGMA busy_timeout=10000zINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name) VALUES (?,?,?,?,?,?,?,?,?)r r r i r zcrawl_yizheng_xz_tzgg.pyz
DB error: zResult: z new, z skipped)listTOWNSkeysrf r rV rC enumeratery CUTOFFrB timesleepopenwritejsondumpssqlite3connectr executecursorrowcountr= commitrF )townspagesjson_out all_enrichedtctnamer rJ idxrS dfitemconncstoredskippedrU s r crawlr s }UZZ\"L "b! wm4 hyk ()%b%0 jU $< => * GC RY'A Q~%6%? 72r'{ uI !. 1 ?RZ Y< &(
!
Rx1}bQs5zl+, JJsO
* &\"#4()
(C'
2 Ea$
E
4e ??7 #DLL,-
AFW
II. k"D$4d:6Fg ^ 4d9oy/'R#.X@Z\
] zzA~! 1 KKMJJL(6(& 239 E E. *QC ! qLG s% =1I AI+ I(+ J4J J__main__r, z--pages=r z
--pages=(\d+)z--col=z--col=([\w,]+)all,z--json)r r r )N)Nr N)0__doc__r sysosr r r re r r bs4r urllib.parser playwright.sync_apir r4 DOMAINGROUPnowstrftimer environrf r r r9 r r$ rV ry rn r __name__argvr' r r r r intrk rl colsplitr r index r r r s 2 1 1 1 ( /#
(,,.9%0
0 : :: F
**..$5
6 {;{ {;{*5F{ K L
@D8A*84t z 88AB!bii)388D>:@@C"%, #D 4F4
8,q01 UX6 r