j$
d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl m
Z
dZ dZ dZ
ej ed z
j d
Zd Zd Zd
ZdZd Zd Zd Zd Zd Zd ZddZe dk( r e yy)u 简阳市人民政府 - 公示公告栏目爬虫
站点: https://www.scjy.gov.cn (四川成都, /jianyang/ 站群)
栏目: /jianyang/c123218/list_more.shtml (公示公告, 当前 5 条静态)
WAF: 瑞数4代 JS 挑战 (412, $_ts) → Playwright 过挑战
列表: list_more.shtml 静态条目 标题 (无日期, 标题可能截断带...)
详情: fetch → meta ArticleTitle(完整标题)/PubDate + div.main_bd 正文 (图片型公告 img 相对路径绝对化)
入库: 自动检测 /root/search.db 存在则 INSERT OR IGNORE 直插 gov_raw (FTS 触发器自动同步 gov_search)
运行: python3 crawl_jianyang_gsgg.py [--no-db] [--sync-only]
N)datetime timedelta)sync_playwrightu 简阳市-公示公告zhttps://www.scjy.gov.cnz8https://www.scjy.gov.cn/jianyang/c123218/list_more.shtmliG )daysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z/tmp/jianyang_gsgg.jsonlz/root/search.dbu 公示公告c t j dd| xs d j } | j dd j dd } t j dd| j } | S )Nu ^[•··.\s]+ u u z\.{3,}$)resubstripreplacets (/root/gov_crawler/crawl_jianyang_gsgg.py clean_titler s_
"ag2.446A (B ''"5A
z2q!'')A H c ddl m} || d }|j d D ] }|j |j ddg D ] }|j |j d D cg c] }|j d s| }}|sn|d j G|j d D ]? }|j
d s|j
d rd |j
d ini |_ A t | S c c}w )
u@ 正文清洗: span/font unwrap + 嵌套p扁平化 + 保留表格r )
BeautifulSoupz html.parserc : | j xr d| j v S )N:)namer
s r z clean_content..# s qvv'?#- r spanfontpstylealign) bs4r find_all decomposeunwrapfindgetattrsstr)htmlr souptagr nesteds r
clean_contentr) s ! } -D}} ?@
}}ff -.
!]]3 /?166#;!?? q
]]3
J 55>3455>wg/rAGJ t9 @s 3D
Dc
g }t j d| t j t j z D ] }|j d j |j d }}|r|j
d rHt j d|t j snt j dd| j dd }|j
d
r
t |z }n,|j
d s|j d
d d d
z |z }|j |xs |j d
d
|d t }g }|D ]/ } | d |vs |j | d |j | 1 |S )u> 正文内附件: 相对路径绝对化z$]+href="([^"]+)"[^>]*>(.*?)
javascriptz#\.(pdf|docx?|xlsx?|zip|rar|7z|doc)$<[^>]+>r Nd /httpr )r urlr3 )r finditerISgroupr
startswithsearchr
BASErsplitappendsplitsetadd)
html_frag
detail_urlattsmhref name_htmlr seenuniqas
r extract_attachmentsrI 3 sO
D
[[@)RTTTVTXTX[
Y
L''!***,aggajit|4
99;T244 H66*b)4:: { const r = await fetch(u); return await r.text(); }N)rangeevaluate Exceptiontimesleep)pager3 _s r
fetch_htmlrR I sN
1X ==!hjmn n
JJqM s & AAc | syt j j t st dt dd yt j t d }|j d d x}}|j }| D ] } |j d
t |d |d |d |d
|d |d xs dd d t d|d |j dd |j dd f |j d kD r|dz
}n|dz
}|j |j ||fS # t
j $ r }t d| d |dz
}Y d }~Sd }~ww xY w)N)r r u 未发现 u , 跳过入库 (仅写 JSONL)Tflushi" )timeoutzPRAGMA busy_timeout=290000r zINSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments, group_name, industry)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)r3 titlepub_datecontentr i zcrawl_jianyang_gsgg.py attachments
group_name 四川industryotherr+ z
DB err: )ospathexistsDB_PATHprintsqlite3connectexecutecursor SITE_NAMECATEGORYr" rowcountErrorcommitclose)resultsconnaddedskippedcurres r
push_to_dbru R s[
77>>' "
WI%BC4P ??7C 0DLL-.EG
++-C
KK7 AeHah'
AjM1Y<I,$"ds+X7OQRS`Qa|X.j'0JL
||a
1
#$ JJL '> }} Jqc"$ / qLG s A]+href="(/jianyang/c123218/\d{4}-\d{2}/\d{2}/content_[^"]+)"[^>]*class="special_content_list"[^>]*>(.*?)zW]+href="(/jianyang/c123218/[^"]+)"[^>]*class="special_content_list"[^>]*>(.*?)u 列表条目: rw r+ r. r r1 u 详情失败: rx z3]+name="ArticleTitle"[^>]+content="([^"]+)"z.]+name="PubDate"[^>]+content="([^"]+)"
rz zA]*class="main_bd"[^>]*>(.*?)\s*]*class="xgfj"z]+src=")([^"]+)(")c r | j d | j d j d r| j d na| j d j d rt | j d z n)j dd d dz | j d z z | j d z S )Nr+ r, r1 r0 r r )r7 r8 r: r; )mmrA s r r zmain.. s rxx{RXXa[=S=STZ=[bhhqkxz yA yA BC yD yO yO PS yTbfikiqiqrsitbt Zd Zk Zk lo qr Zs tu Zv y| Z| A G G HI J ZJ L OQ OW OW XY OZ Z r r\ r^ ensure_ascii) site_namerW rX rY
source_urlr3 r[ r] rZ u === 入库: 新增u 跳过z ===u === 完成: 列表u 详情OKu 空u
CUTOFF截)%sysargvrc rh CUTOFFr chromiumlaunch new_contextUAnew_pagegotoLIST_1rM rK wait_for_timeoutrY lenrm r findallr6 r r
r8 r: rR r9 r7 r) r rI jsondumpsr< save_resultsru )no_dbrn statsr browserctxrP passedir% itemsrD t_html
list_title detail_htmlrC rW rX content_htmlrY textrB recrp rq rA s @r mainr q s !E
B$O HYKy
1>GQa @E Qa**##m $ o !!RTXdgJh C! D||~ IIfeI A r A ! !$ '
||~ T!c$i$&6+AaC52$? / . MMO 5Q Q:
P RV XZ X\ X\ ]JJ y{ BD BF BF GE
s5zl+48 ! - LD& &MQ M$RVVJF%CDJ!% !8dTkJ$T:6K *:cr?*;dK uV}oYu[7I6J$uU\~N^^hinowixhyy}
~ GK L[
#Q Qsh "A Q/P(!Q)P89Q>Q
9QI:Q( P51Q4P55Q8 QQQQQc t t dd 5 }| D ]+ }|j t j |d dz - d d d t dt
| dt d
y # 1 sw Y *xY w) Nwzutf-8)encodingFr
u 已写 u 条到 TrT )openOUTwriter r rc r )rn r frs s r r r sm
c3 ) >Q >A
GGDJJqu5< = >>
GCL>#
/t<> >s 1A..A7__main__)N) __doc__r r_ r r rN rd r r playwright.sync_apir rh r: r nowstrftimer r r rb ri r r) rI rR ru r r __name__ r r r s ( ' ' ( / $ C
(,,.9%0
0 : :: Fv
(, >`LD= zF r