+'j' d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
ej j dej j ej j e ddlmZmZmZmZmZmZ dZdZdZd ZdZ ej8 d
d Z d Z d
Z d Z d Z!d Z"d Z#d)dZ$e%dk( r/ddl&Z& e&jN de Z(e(jS de*d e(jS ddd e(jW Z,e,jZ r e Z.e.j_ d ja d Z1e.j_ d ef ja d Z2 e3d e1 d e d!e2 d" e.ji ejj d e3d#e d$ ej Z6 e$e,jn % e3d& ej e6z
d'd( yy)*u/
crawl_lc_sthj_v2.py — 陵川县人民政府-部门工作（环境保护相关）
==========================================================
列表页: http://xxgk.lczf.gov.cn/lczxdw/lcsthjj/fdzdgknr/gzdt/
结构: ul.list-items-box-inner > li (20条，唯一页，无分页)
详情页: div#Zoom 正文
存储: /root/search.db (gov_raw + gov_search_v3)
用法:
python3 crawl_lc_sthj_v2.py # 全量爬
python3 crawl_lc_sthj_v2.py --limit=5 # 测试只爬5条
python3 crawl_lc_sthj_v2.py --stats # 查看 search.db 统计
N)urljoin)
BeautifulSoup)
fetch_page
parse_date
clean_htmlextract_date_from_htmlextract_content_by_selector
save_jsonlu 陵川县-部门工作zhttp://xxgk.lczf.gov.cnz5http://xxgk.lczf.gov.cn/lczxdw/lcsthjj/fdzdgknr/gzdt/z#Zoom SEARCH_DBz/root/search.dbc t j j t } | rt j | d t j t d }|j d |j d |j d |S )u' 获取 search.db 连接（WAL 模式）T)exist_ok )timeoutzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALzPRAGMA busy_timeout=30000)ospathdirnameDB_PATHmakedirssqlite3connectexecute)db_dirconns %/root/gov_crawler/crawl_lc_sthj_v2.pyget_dbr 1 sa
WW__W
%F
FT* ??7B /DLL*+LL,-LL,- K c | j d | j d | j d | j d | j t d y# t $ r Y Iw xY w)u= 创建 gov_raw + gov_search_v3（FTS5）表（如不存在）a9 CREATE TABLE IF NOT EXISTS gov_raw (
id INTEGER PRIMARY KEY AUTOINCREMENT,
site_name TEXT,
source_url TEXT,
page_url TEXT,
title TEXT,
publish_date TEXT,
summary TEXT,
content TEXT,
status TEXT,
category TEXT,
tags TEXT
)zOCREATE UNIQUE INDEX IF NOT EXISTS idx_gov_raw_source_url ON gov_raw(source_url)zACREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)zDCREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date)u [DB] 表就绪: gov_rawN)r Exceptioncommitprint)r s r
ensure_tablesr! = si LL
fg LLTULLWXKKM %'
s A" " A.-A.c L | j d|f }|j duS )u 检查 page_url 是否已存在z&SELECT 1 FROM gov_raw WHERE page_url=?N)r fetchone)r page_urlcurs r
row_existsr& X s&
,,?(
MC <<> %%r c T |j d xs ddd }|j dd }|j d xs ddd }|j dd }|j d d }|sA|r?t j d
d | }t j d d | j dd }|j d
d } |}| j dt
|||||||f | j d j
d } | dkD ryy# t $ r}
t d|
Y d}
~
yd}
~
ww xY w)uY 插入一条记录到 gov_raw + gov_search_v3
返回: True=新增, False=跳过
title Ni urlpub_date
contentsummaryz<[^>]+> z\s+categoryzINSERT OR IGNORE INTO gov_raw
(site_name, title, source_url, page_url, publish_date, summary, content, category)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)zSELECT changes()r TFz
[DB ERROR] ) getresubstripr SITE_NAMEr# r r ) r itemr( r$ publish_dater- r. r0
source_urlaffectedes r
insert_rowr; ^ s4
XXg
$"ds +Exxr"HHHZ(.B4Lhhy"%G hhy"%G w&&S'2&&g.446t<xx
B'H
/ z8\7GU] ^ ` << 23<<>qA a<
aS!"s =AD D'D""D'c g }t | d }|j dd }|s
t d |S |j d D ] }|j d }|s|j dd j }|j
d
}|s|j d d }d }|j d
} | r| j
d
}|r!|j d st t | }|s|s|j |||d |S )u 从列表页 HTML 提取所有条目
ul.list-items-box-inner > li
每个 li 包含: 标题 2026-04-29
返回: [{"title": "...", "url": "...", "date": "..."}, ...]
html.parserulzlist-items-box-inner)class_z* [WARN] ul.list-items-box-inner not foundliahrefr) T)r4 r( spanhttp)r( r* date) r findr find_allr1 r4 get_text
startswithr LIST_URLappend)
htmlitemssoupr> r@ rA rB r( r+ rC s
r get_list_itemsrO s
E } -D
4 6 7B
:; kk$ GGCL uuVR &&(
&EE'2&E wwv }}4}0H /8T*D T LL
)4 Lr c t | }|sdddS t |t }|rt | dk rKt |d }|j dd }|r t
| }n |j d }|r t
| }t | }||dS ) ur 访问详情页，返回正文 HTML 和发布时间
返回: {"content": "...", "pub_date": "YYYY-MM-DD"}
r) )r- r+ 2 r= divZoom)idbody)r r DETAIL_SELECTORlenr rF strr )r* rL r- rN zoomrU r+ s r
get_detailrZ s
c?D 2.. *$@G c'lR' T=1yy6y* $iG99V$D d) &d+H H 55r c
t dt t t }|s t d yt | }t dt | d | r|d| }t }t
| d\ }}t |d D ]' \ }}t ||d r|dz
}d }nnt |d } | d
xs |j d d }
|d
|d | d |
d} t | t } t | t || r|dz
}d}n|dz
}d}|j |j d d xs t! j d
xs d }
|dz dk( s|t | k( r+t d| dt | d|
d|d
dd d|
t d|
d|d
dd d| * |j# t d | d | ||fS ) u$ 全量爬取（单页，无分页）u
列表页: z# [ERROR] Failed to fetch list page)r r u 列表页: 提取 条N r* u
⏭ 已存在r+ rE r) r( r- )r( r* r- r+ ) site_nameu
✅ 新增u ⏭ 写入失败 r z [/z] r/ ( z... z u ✅ 完成: 新增u , 跳过)r rJ r rO rW r r! enumerater& rZ r1 dictr5 r
r; r localsclose) max_itemsrL rM r okfailir6 statusdetailr+ entryentry_with_site display_dates r crawlro s M(
$% h D
34 4 E U D
12jy! 8D$HB UA& $H4 dDK ( AID$F U ,F j)ATXXfb-AH g E{!), $ E #5I >O ' $&a% +
xxr*Nvx||J/G/M2 q5A:c%j Cs!CJr s ( ' ' 277??277??8#<= >
% % C "))K!2
3 (6& "R)`6B<~ z
$X
$
$8I;1G
HF
2IJ
,=VW D zzx ;<EEGJ\\