ujg: d Z ddlZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl m Z dZdZdZdZdZd Zej( j+ d
d Zej( j+ d d
ZdZdZdZ ej6 dej8 ej: z Z ddddZ dZ e!ejD D ]k \ Z#Z$e$jK d r e&e$jO dd d Z 4e$dk( s:e#dz e)ejD k sS e&ejD e#dz Z m e dk\ re ndZ*d Z+d Z,d Z-d Z.d Z/d Z0d Z1d! Z2e3d"k( r e2 yy# e
$ r dZ Y 5w xY w# e($ r Y w xY w# e($ r Y w xY w)#u7
crawl_huaxi_tzgg.py - 花溪区人民政府-通知公告
https://www.huaxi.gov.cn/xwzx/tzgg/
政府站通用模板: 列表 {year}{month}/t{date}_{id}.html, 分页 index_{N}.html
列表: div.right-list-box ul > li
标题
2026-08-04 15:52
分页: createPageHTML(56, 0, "index", "html", 839) → index_{N}.html, 共56页 839条
详情: meta ArticleTitle + meta PubDate (YYYY-MM-DD HH:MM)
标题: div.Article_bt / meta ArticleTitle
日期: meta PubDate 取 YYYY-MM-DD
正文: div.Article_zw (内含 div#Zoom > div.zw-con > div.trs_editor_view TRS_UEDITOR)
附件: 附件名 (oldsrc 才是真实绝对路径)
N)
BeautifulSoupzhttps://www.huaxi.gov.cnz#https://www.huaxi.gov.cn/xwzx/tzgg/z4https://www.huaxi.gov.cn/xwzx/tzgg/index_{page}.htmlu" 花溪区人民政府-通知公告u 贵州zcrawl_huaxi_tzgg.pyDB_PATHz/mnt/data/search.db
JSONL_PATH utf-8zmeta:ArticleTitlezdiv.Article_zwz}\s*]*href="([^"]+/xwzx/tzgg/\d{6}/t\d+_\d+\.html)"[^>]*title="([^"]*)"[^>]*>(.*?)\s*\s*(\d{4}-\d{2}-\d{2})zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-Language z--pages==z--pagesc 2 | syt j | } | j dd j dd j dd j dd } t j dd| } t j dd| } | j S ) Nr u u u u u3 ^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts %/root/gov_crawler/crawl_huaxi_tzgg.py clean_titler F s ! A (B ''"5==hKSST\^`aA
Er1MA
vsA A 779 c |dk( rt nt j | }| j |t dd }d|_ |j S )Nr )page Fheaderstimeoutverifyr ) LIST_URL_P1 LIST_URL_PNformatgetHEADERSencodingtext)sessionr urlrs r
fetch_listr) P sC + (:(:(:(EC C"U CAAJ 66Mr c H g }t j | D ] }|j d }t |j d }|j d }|rt | dk rPt
j j t | }|j |||f |S )Nr )
LIST_ITEM_REfinditergroupr lenurllibparseurljoinr append) html_textitemsmhreftitledateabs_urls r
parse_listr< W s E
"
"9
- -wwqzAGGAJ'wwqzE
Q ,,&&{D9
gud+,- Lr c d}d}d}t rt | d }t j d rqt j dd d }|j ddt j |t
j i }|r |j d rt |d }|s/|j d
}|r t |j d d
}|j ddt j dt
j i } | rK| j d r:t j d| d j }
|
r|
j d }t r9|j t } | r"dj! d | j" D }|sct j$ d| }
|
rK|
j d dt' |
j d ddt' |
j d d}|||fS )Nr html.parserzmeta::r metaname)attrscontentzdiv.Article_btr
Tr zPubDate|publishdatez(\d{4}-\d{2}-\d{2})c 3 2 K | ] }t | y w)N)str).0cs r z parse_detail..{ s &C!s1v&Cs u' (\d{4})[-年/](\d{1,2})[-月/](\d{1,2})-r+ 02d )r DTITLE_MODE
startswithsplitfindr compileIr" r
select_oneget_textmatchr r/ DCONTENT_SELjoincontentssearchint) r5 page_urlr9 r: content_htmlsouprA mtbtmddmels r parse_detailrc d s E
DL Y
6 ! !' * $$S!,Q/D6&"**T2442H)IJBbffY'#ByM2 !12B#BKK4K$@A
YYvfbjj9OQSQUQU.V%WY
X
"&�"Y-2E2E2GHBxx{ .B !ww&Cr{{&C C
YYA9
M
hhqk]!C $4S#93rxx{;KC:PQD $ $$r c | sddg fS t j dd| t j } t j dd| t j } t j dd| t j t j z } t j dd| t j t j z } g i t j dd| } t j d d| } t r t | d
}|j d d
D ]? }|j
d sG|j
d s6t j d|j
dd t j s\|d }|j
d xs |}|j d xsW |j
d xsD |j
d xs1 t j j |j d d }t j j | } j! ||f dt# d}d| d| d |< |j% | B |j d D ] } | j d }
|
s| j d r+g } |
D ] } | j
d xs | j
d xs d}
|
s.t j j |
}| j! d| dt j j |
j d d xs d d | sd t# d}d!d"j' | z d#z |< | j% | t) | } fd$}t j d%|| t j t j z } t rQ t | d
}|j d D ]% }d&D ] }|j, j/ |d ' t) | } g fd'}t j d(| t j rd)nd}t j d*|| t j t j z } t j d#d+| t j } t j d,d-| t j } t j d.d| t j } g }| j d/ D ]W }|j1 }|st j d0d| }t j d1d| }|j3 d2d j3 d3d4 }t j d5d4| }t j d6d| }t5 j6 | }|j3 d7d4 j3 d8d4 }t j d9d| }|s j9 D ] \ }}|j3 || } t; D ] \ }}|j3 d:| d| } |j! |j1 Z d/j' | } j9 D ] \ }}|j3 || } t; D ] \ }}|j3 d:| d| } t j d;d| }t j dd| }t5 j6 | }t j d?d-| }t j d@d/| }t j d5d4| }t= } g } |j d/ D ]X } t j dAdt j d6d| }|s2|| v r7| j? | | j! | Z d/j' | j1 }|| fS # t* $ r Y w xY w# t* $ r Y w xY w)BNr r z
)flagsz ]*>z]*>.*?z]*>.*?u 【字号：[^】]*】u' 发布(?:时间|日期)[:：][^<]{0,30}r> aT)r8 appendixz
data-appendixz \.(pdf|docx?|xlsx?|zip|rar|wps)r8 oldsrcrD _titler9 ?
__ATTACH____ pimgsrcu 图片__IMG__zz
z
c 8 | j d }|j d ry| j d }t j dd| }t j
| j
}t j dd| j
}|s5t j j |j d d xs d }t j j | }j ||f d
t d }d | d
| d|< |S )Nr z javascript:r r+ <[^>]+>u ^附件[:：]\s*rj r u 附件__LINK__rl rm rn ro )r/ rN r r r r r ospathbasenamerO r1 r2 r3 r4 r0 ) r7 r8 innertxtr; key attachments link_protectr[ s r
_link_replz html_to_text.._link_repl s wwqz ??= )
ffZU+ $**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*\*+2. 'y0B3%t L S
r z&]*href="([^"]+)"[^>]*>(.*?)) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingc h j | j d dt dz
dS )Nr __TBL__r rl )r4 r/ r0 )r7
table_protects r _tbl_replz html_to_text.._tbl_repl s3 QWWQZ(]+A-.b11r z ]*>r z]*>.*?z
z
z(]*>z
z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)
r
z [ \t]{2,}ru z z [\s\u200b\u200c\u200d\ufeff]+r z
__LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r ) r r SrR r find_allr" rY rT rw rx ry rO r1 r2 r3 r4 r0 replace_withrW rF ExceptionrB popr r r r r6 enumeratesetadd)!rC r[ _souprf r8 real_hrefr{ r; r| rp imgssrcsrq rr r tagattrr has_tablepartsblockbplainkvitbloutseenfinalr} r~ r s! ` @@@r html_to_textr sd 1byff]Brtt"))Lnpqpupuv| A qB DF DH DH CI V9D !h 74I**4*0 CAEE(O CquuW~ CY[Y`Y`YiYijsjyjyz}j~ @A kB ZCC $ll 2 28Y GG &&~6 &s<'8&9
G[][_[_bdbfbf[fgG !'=9E ~~d+
. L .DIIMM$-.
. %jG M2 YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG E v& KKM FF+R 3 FF*B 2
IIdB ' 'c 2 FF<a (z2q)!!%(
fc*228SA7UC &&( DAq !QA
. 0FAs GA3b/3/A 0
QWWY ' * ++e
C ""$ 1kk!Q M* 03kkGA3b/3/0
&&!2s
+C
&& "c
*C
&& "c
*C
&&#R
-C
C
C
&&dC
(C
&&FC
(C
&& sC
(C 5D E
YYv
ffVR
B!:; $;
Q ++e
"
"
$C ; &&y 6 sG A:`# ?D`# `# "1`# A/`# A`# A`3 # `0/`03 a ?a c
| j d|t f }|j ry d} ddlm} || }| j d|||||t t t ||f
}|j dkD ryt j dd| } t j | } t j d d
| j d d } | j d |j |t | f y
y# t
$ r Y w xY w# t j $ r=}
dt# |
v r*t% j& d t) | ||||| cY d }
~
S d }
~
ww xY w)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?Fotherr )classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)ru r r r
zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)TlockedrL )execute SITE_NAMEfetchone crawler_libr r
GROUP_NAME SCRIPT_NAMErowcountr r r r r lastrowidsqlite3OperationalErrorrF timesleep store_record) connr' r9 rC r: r curindustryr summaryes r r r
sF
,,PSVXaRb
cC
||~ 5(/H ll
+ #ugtY
KQZ\d e
<]
z skip empty/placeholder: ( )r[ r9 rC publish_dater site_name
group_name script_namez
JSONL + z ()z + g ?g ?wr )r$ F) ensure_asciir zJSONL written: z rows)u 新增: u 跳过: z=== z done ===)(requestsSessionr updater# r r connectr r r range_PAGESr) r printr< r0 r r rc r r r r rY rR r4 r r r r r r openwritejsondumpscommitclose)r sr new_count
skip_count seen_urls
jsonl_rowsr r5 r r6 r; r9 r: dhtmldtitleddater\ rC r atts plain_lenno_paraokfrows r mainr 7 s AIIW D w3 01 ./IJIJa!$ 2 "1d+I 9%
dV8CJfETR
HYK
- HZL
!. D 9
%T2y E$~aS1 >
y1#6dCa
H D Ds< ! L. M;1N. M7MM N!M??NN__main__)4__doc__sysrw r r htmlr urllib.parser1 r r bs4r ImportErrorBASE_URLr r r r r environr" r r ENCrM rV rQ r rR r- r#
_MAX_PAGESr argvr rf rN rZ rO
ValueErrorr0 r r r) r< rc r r r r __name__ r r r s
!
&3 D 0
#
**..$9
:
ZZ^^L"
-
! rzz EDD244K @O'
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA
%>G'T D0N3b zF Y M>
s5 E E# E.E E #E+*E+.E65E6