uj(9 d Z ddlZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl m Z dZdZdZdZdZd Zej( j+ d
d Zej( j+ d d
ZdZ ej2 dej4 ej6 z Z ddddZ dZ e ej@ D ]k \ Z!Z"e"jG d r e$e"jK dd d Z 4e"dk( s:e!dz e'ej@ k sS e$ej@ e!dz Z m e dk\ re ndZ(d Z)d Z*d Z+d Z,d Z-d Z.d Z/d Z0e1d k( r e0 yy# e
$ r dZ Y 1w xY w# e&$ r Y w xY w# e&$ r Y w xY w)!uQ
crawl_jzzq_tzgg.py - 左权县生态环境局-通知公告
http://www.jzzq.gov.cn/zwgkzy/bmdwgkml/10zqxhjbhj/xxgkml10zqxhjbhj/zcwjsthjj/tzgg10zqxhjbhj
CMS: 开普云 kaipuyun (power.core.js, pucha.kaipuyun.cn)
列表: div.boxGkxxList ul.infoList > li
2026-06-30
标题
分页: URL 尾缀 _2/_3/... (tzgg10zqxhjbhj_{pageid}), 共16页(quickJump上限), 每页约18条
详情: 列表URL + "?LMCL=HcSri1" (无此参数返回 BBI JS混淆跳转页)
标题: articleCon h2.title
日期: div.property "发布时间：YYYY-MM-DD"
正文: div.conTxt (含表格; 需剔除元信息表 table.govDetailTable)
附件: conTxt 内 (附件后缀 pdf/doc/xls/zip等)
N)
BeautifulSoupzhttp://www.jzzq.gov.cnzhhttp://www.jzzq.gov.cn/zwgkzy/bmdwgkml/10zqxhjbhj/xxgkml10zqxhjbhj/zcwjsthjj/tzgg10zqxhjbhj{page_suffix}z ?LMCL=HcSri1u% 左权县生态环境局-通知公告u 山西zcrawl_jzzq_tzgg.pyDB_PATHz/mnt/data/search.db
JSONL_PATH utf-8z~]*>\s*(\d{4}-\d{2}-\d{2})\s*]*href="(/[^"]*content_\d+)"[^>]*title="[^"]*"[^>]*>(.*?)zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-Language z--pages==z--pagesc 2 | syt j | } | j dd j dd j dd j dd } t j dd| } t j dd| } | j S ) Nr u u u u u3 ^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts $/root/gov_crawler/crawl_jzzq_tzgg.py clean_titler E s ! A (B ''"5==hKSST\^`aA
Er1MA
vsA A 779 c |dk( rdnd| }t j | }| j |t dd }d|_ |j
S ) Nr r _) page_suffix Fheaderstimeoutverifyr )LIST_TPLformatgetHEADERSencodingtext)sessionpagesuffixurlrs r
fetch_listr+ O sK 19RAdV*F
//f/
-C C"U CAAJ 66Mr c
g }t j | D ] }|j d }|j d }t t j
dd|j d }|rt
| dk ret j j t | t z }|j |||f |S )Nr <[^>]+>r )
LIST_ITEM_REfinditergroupr r r lenurllibparseurljoinBASE_URLLMCLappend) html_textitemsmdatehreftitleabs_urls r
parse_listrB W s E
"
"9
- -wwqzwwqzBFF:r1771: >?E
Q ,,&&x6=
gud+,- Lr c : d}d}d}t rt | d }|j d }|r t |j dd }|j d }|r7t j
d|j }|r|j
d }|j d
} | rH| j d D ] }
|
j dj d | j D }|s)t j
d| }|r|j
d }|||fS )
Nr html.parserzarticle.articleCon h2.titler
Tr z article.articleCon div.propertyu( 发布时间[:：]\s*(\d{4}-\d{2}-\d{2})r z article.articleCon div.conTxtztable.govDetailTablec 3 2 K | ] }t | y w)N)str).0cs r z parse_detail..y s "@a3q6"@s ) r
select_oner get_textr searchr3 select decomposejoincontents) r; page_urlr@ r> content_htmlsouph2propdmcontbls r parse_detailrZ d s E
DL Y
6
__ :
;
Ct <=E @A F
XBxx{oo => zz"89
77"@3<<"@@L
YYBI
N
88A;D $ $$r c 6 | sddg fS t j dd| t j } t j dd| t j } t j dd| t j t j z } t j dd| t j t j z } g i t j dd| } t j d d| } t ra t | d
}|j d d
D ] }|j
d sG|j
d s6t j d|j
dd t j s\|d }|j d xsD |j
d xs1 t j j |j d d }t j j | } j! ||f dt# d}d| d| d |< |j% | |j d D ] }|j d } | s|j d r+g }
| D ] } | j
d xs | j
d xs d} | s.t j j | }|
j! d| dt j j | j d d xs d d |
sd t# d}d d!j' |
z d"z |< |j% | t) | } fd#}
t j d$|
| t j t j z } t rQ t | d
}|j d D ]% }d%D ] }|j, j/ |d ' t) | } g fd&}t j d'| t j rd(nd}t j d)|| t j t j z } t j d"d*| t j } t j d+d,| t j } t j d-d| t j } g }| j d. D ]W }|j1 }|st j d/d| }t j d0d| }|j3 d1d j3 d2d3 }t j d4d3| }t j d5d| }t5 j6 | }|j3 d6d3 j3 d7d3 }t j d8d| }|s j9 D ] \ }}|j3 || } t; D ] \ }}|j3 d9| d| } |j! |j1 Z d.j' | } j9 D ] \ }}|j3 || } t; D ] \ }}|j3 d9| d| } t j d:d| }t j d;d| }t j dd,| }t j d?d.| }t j d4d3| }t= }g } |j d. D ]X }t j d@dt j d5d| }|s2||v r7|j? | | j! | Z d.j' | j1 }|| fS # t* $ r Y w xY w# t* $ r Y w xY w)ANr r z
)flagsz ]*>z]*>.*?z]*>.*?u 【字号：[^】]*】u' 发布(?:时间|日期)[:：][^<]{0,30}rD aT)r? appendixz
data-appendixz \.(pdf|docx?|xlsx?|zip|rar|wps)r? rE _title?
__ATTACH____ pimgsrcoldsrcu 图片__IMG__zz
z
c 8 | j d }|j d ry| j d }t j dd| }t j
| j
}t j dd| j
}|s5t j j |j d d xs d }t j j | }j ||f d
t d }d | d
| d|< |S )Nr z javascript:r r- r. u ^附件[:：]\s*r` r u 附件__LINK__rb rc rd re )r3
startswithr r r r r ospathbasenamesplitr5 r6 r7 r: r4 ) r= r? innertxtrA key attachments link_protectrR s r
_link_replz html_to_text.._link_repl s wwqz ??= )
ffZU+ $**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*\*+2. 'y0B3%t L S
r z&]*href="([^"]+)"[^>]*>(.*?)) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingc h j | j d dt dz
dS )Nr __TBL__r rb )r: r3 r4 )r=
table_protects r _tbl_replz html_to_text.._tbl_repl s3 QWWQZ(]+A-.b11r z ]*>r z]*>.*?z
z
z(]*>z
z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)
r
z [ \t]{2,}r. z z [\s\u200b\u200c\u200d\ufeff]+r z
__LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r ) r r SIr find_allr" rM rL rn ro rp rq r5 r6 r7 r: r4 replace_withrP rG Exceptionattrspopr r r r r< enumeratesetadd) contentrR _soupr] r? rs rA rt rf imgssrcsrg rh rw tagattrr has_tablepartsblockbplainkvirY outseenfinalru rv r s ` @@@r html_to_textr s- 1byff]Brtt"))Lnpqpupuv| A qB DF DH DH CI V9D**4*0kAEE(OkrwwGWGWX\XbXbcfXghiXjGkC $ll 2 28T BG &&~6 &s<'8&9
G[][_[_bdbfbf[fgG !'=9E ~~d+
. L .DIIMM$-.
. %jG M2 YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG E v& KKM FF+R 3 FF*B 2
IIdB ' 'c 2 FF<a (z2q)!!%(
fc*228SA7UC &&( DAq !QA
. 0FAs GA3b/3/A 0
QWWY ' * ++e
C ""$ 1kk!Q M* 03kkGA3b/3/0
&&!2s
+C
&& "c
*C
&& "c
*C
&&#R
-C
C
C
&&dC
(C
&&FC
(C
&& sC
(C 5D E
YYv
ffVR
B!:; $;
Q ++e
"
"
$C ; &&y 6 sG A:_; ?C'_; '_; :1_; ,A/_; A_; *A` ; `` ``c
| j d|t f }|j ry d} ddlm} || }| j d|||||t t t ||f
}|j dkD ryt j dd| } t j | } t j d d
| j d d } | j d |j |t | f y
y# t
$ r Y w xY w# t j $ r=}
dt# |
v r*t% j& d t) | ||||| cY d }
~
S d }
~
ww xY w)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?Fotherr )classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)r. r r r
zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)Tlockedr/ )execute SITE_NAMEfetchone crawler_libr r
GROUP_NAME SCRIPT_NAMErowcountr r r r r lastrowidsqlite3OperationalErrorrG timesleep store_record) connr) r@ r r> r curindustryr summaryes r r r sF
,,PSVXaRb
cC
||~ 5(/H ll
+ #ugtY
KQZ\d e
<]
z skip empty/placeholder: ( )rR r@ r publish_dater site_name
group_name script_namez
JSONL + z ()z + g ?g ?wr )r$ F) ensure_asciir zJSONL written: z rows)u 新增: u 跳过: z=== z done ===)(requestsSessionr updater# r r connectr r r range_PAGESr+ r printrB r4 r r rZ r r r r rM r r: r r r r r r openwritejsondumpscommitclose)r sr new_count
skip_count seen_urls
jsonl_rowsr' r; r r< rA r@ r> dhtmldtitleddaterS r r atts plain_lenno_paraokfrows r mainr 3 s AIIW D w3 01 ./IJIJa!$ 2 "1d+I 9%
dV8CJfETR
HYK
- HZL
!. D 9
%T2y E$~aS1 >
y1#6dCa
H D Ds< ! L. M;1N. M7MM N!M??NN__main__)2__doc__sysrn r r htmlr urllib.parser5 r r bs4r ImportErrorr8 r r9 r r r environr" r r ENCcompiler r r1 r#
_MAX_PAGESr argvr r] rm intrq
ValueErrorr4 r r r+ rB rZ r r r r __name__ r r r s
!
$ u 3
"
**..$9
:
ZZ^^L"
-
rzzJDD244K @O'
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA
%:E'P D0N3b zF Q M<
s5 E E E*E E E'&E'*E21E2