uj: d Z ddlZddlZddlZddlZddlZddlZddl Z ddl
m Z dZ
dZe
ez ZdZdZdZd Zd
d d d
ZdZ eej. D ]k \ ZZej5 d r eej9 dd d Z4edk( s:edz e ej. k sS eej. edz Zm edk\ rendZ d Z ddZ!d Z"d Z#d Z$d Z%d Z&e'dk( r e& yy# e $ r dZ Y w xY w# e $ r Y w xY w# e $ r Y w xY w)u
江西景瑞祥环保科技有限公司 - 项目公告 (www.jxjrxhb.com/index.php/news/hjkx/1/cn/15410/15410.html)
CMS: zhasp 双阳网络老式 PHP 建站系统 (index.php/news/{user}/{page}/cn/{columnId}/{columnId}.html)
列表: 单页 15 条固定, 无分页控件 (所有分页参数变体返回相同 15 条 = 栏目全量)
条目 ![]() 标题 2026-07-22 |
详情: /userlist/hjkx/newshow-{ID}.html
标题区 标题 (完整)
正文 ...Word转换逐段 ... |
无 meta ArticleTitle/PubDate — 标题取 前缀或 strong 区, 日期取列表页 font 日期
无独立附件区 (环评公示纯文本, 无 <a href> 下载链接)
N)
BeautifulSoupzhttp://www.jxjrxhb.comz*/index.php/news/hjkx/1/cn/15410/15410.htmlu" 江西景瑞祥环保-项目公告u 企业zcrawl_jxjrxhb_tzgg.pyz/mnt/data/search.dbzkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-Language z--pages==z--pagesc | syt j | } | j dd j dd j dd j dd } t j dd| } | j S )z;unescape entities + strip zero-width + collapse whitespace. u u u u \s+ )html_libunescapereplaceresubstrip)ts '/root/gov_crawler/crawl_jxjrxhb_tzgg.py clean_titler < so ! A (B ''"5==hKSST\^`aA
vsA A 779 c J dd l }d }t | D ]d } |xs |j | t | }|j dk( r|j
xs d|_ |c S |}t j d|dz z f t |t r||S # t $ r }|}Y d }~Ad }~ww xY w)Nr )headerstimeout utf-8g ?r ) requestsrangegetHEADERS status_codeapparent_encodingencoding Exceptiontimesleep
isinstance) urlsessionr retriesr lastires r fetchr, F s D
7^ " $H))#w)PA}}# 0 0 ;G
D
3!a%=! " $ "
K D s AB
B
B"B B"c
| sddg fS t j dd| t j } t j dd| t j t j z } t j dd| t j t j z } t rQ t | d }|j d D ]% }d D ] }|j j |d
' t | } | }t j d d|t j }g i fd }t j d
||t j t j z }g fd}t j d||t j t j z }t j d| t j rdnd}t j dd|t j }t j dd|t j }t j dd|t j }g } |j d D ] }
|
j } | st j dd| } t j | } | j dd j dd } t j d d| } | s~j! D ] \ }
}| j |
| } t# D ] \ }}| j d | d | } | j% | j dj' | }j! D ] \ }
}|j |
| } t# D ] \ }}|j d | d | } t j d d| }t j d d| }t j | }t j d!d| }t j d"d| }t j d#d| }t) }g }|j d D ]X }t j d$dt j dd| }|s2||v r7|j+ | |j% | Z dj' | j }||fS # t $ r Y w xY w)%zConvert article-body inner HTML to stored format:
- keep <table> HTML intact
-
between <p> blocks
- attachment <a> links embedded with absolute URL
- strip <script>/<style>/noise + style attrs via BS4
r r z
<!--.*?-->)flagsz<script[^>]*>.*?</script>z<style[^>]*>.*?</style> html.parserT) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingNz;<img[^>]*src="[^"]*fileTypeImages/icon_[a-z]+\.gif"[^>]*/?>c | j d }| j d }t j dd| }t j | j }t j dd| j }|s5t j j |j d d xs d}t j j | }j ||f d t d
}d | d | d
|< |S )Nr <[^>]+>r u ^附件[:：]\s*?r u 附件__LINK____z <a href="z" target="_blank">z</a>)groupr r r r r ospathbasenamespliturllibparseurljoinappendlen) mhrefinnertxtabs_urlkey attachments link_protectpage_urls r
_link_replz html_to_text.<locals>._link_reply s wwqz
ffZU+ $**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*\*+2. 'y0B3%t L S
r z&<a\s[^>]*href="([^"]+)"[^>]*>(.*?)</a>c h j | j d dt dz
dS )Nr __TBL__r r@ )rI rA rJ )rK
table_protects r _tbl_replz html_to_text.<locals>._tbl_repl s3 QWWQZ(]+A-.b11r z<table[^>]*>.*?</table>z <table[^>]*>r z</p>z</p>
z <br\s*/?>
z(</?(?:span|font|o:p|st1?:[a-z]+)\b[^>]*>z
r= r
z z [\s\u200b\u200c\u200d\ufeff]+rV r@ z
__LINK__\d+__z __TBL__\d+__z[ \t]+\nz\n{3,}z [ \t]{2,}r )r r SIr find_allattrspopstrr! searchrE r r r r
items enumeraterI joinsetadd)contentrS _souptagattrcontent2rT rX has_tablepartsblockbplainkvr) tbloutseenfinalprP rQ rR rW s ` @@@r html_to_textrx X s1 1byff]Brtt<Gff12wbddRTTkRGff/WBDD244KPG !'=9E ~~d+
. L .DIIMM$-.
. %jG HvvFHddH KL vv?X]_]a]adfdhdh]hiH M2 vv0)XRTTTVTXTX[YHYY>AIvvgz8244@HvvlD("$$?HvvA2xWYW[W[\H E ' KKM z2q)!!%(
fc*228SA7UC &&( DAq !QA
. 0FAs GA3b/3/A 0
QWWY ++e
C ""$ 1kk!Q M* 03kkGA3b/3/0
&&!2s
+C
&& "c
*C
C
C
&&dC
(C
&&FC
(C
&& sC
(C 5D E
YYv
ffVR
B!:; $;
Q ++e
"
"
$C ; &&i s *AS S S c g }t rSt | d }|j dd D ]0 }|j dd j }t j
d| s;t j j t | }t |j dd }d}|j } | +| j d
k( r t | j dd }t j d | sM|j d }
|
r:t j
d
|
j dd } | r| j! d }|j# |||f 3 |st j$ d| t j& D ]~ } t j j t | j! d }|j# |t | j! d | j! d j f |S )uB Extract (url, title, date) from list page.
zhasp list: <td style="border-bottom:#D7E5F2 solid 1px"><img src="/tp/newslist0021_box/alloon2.gif" width="11" height="16" align="absmiddle" /> <a href="/userlist/hjkx/newshow-18202.html" target="_blank">标题</a> <font color="#89CAFE">2026-07-22</font></td>
r/ aT)rL rL r z!/userlist/hjkx/newshow-\d+\.html$r
r fontz^20\d{2}-\d{2}-\d{2}$tdz(20\d{2}-\d{2}-\d{2})r zm<a[^>]+href="(/userlist/hjkx/newshow-\d+\.html)"[^>]*>(.*?)</a>\s* \s* \s*<font[^>]*>([^<]*)</font>r< )r r] r r r ra rF rG rH LIST_URLr get_textfind_next_siblingnamematch find_parentrA rI finditerr[ )
html_textrS rb souprz rL rO titledatenxtparentfmrK s
r
parse_listr s
E Y
6 s . 1A55$**,D99 A4H ll**8T:G
3d
;<ED%%'C388v#5"3<<4<#@A88 4d; t,#;V__SX\_=]^B !xx{ LL'5$/ 0% 1& | rtt
QA ll**8QWWQZ@G LL';qwwqz#:AGGAJ<L<L<NO P Q Lr c d}d}t r
t | d }|j d }|r6|j dd }d|v r|j d d }t | }|sF|j d d
d i }|r/|j d
}|r t |j dd }d} |j d D ]5 }
|
j
dd } |
j
dd } | dk( s-| dk( s3|
} n | r#| j d }
|
r|
j }|r|st j d| t j }|rt |j d }|sAt j d| t j }|rt |j d }|s;t j d| t j }|r|j d dz }||fS )uM Extract title, date, content from detail page.
zhasp detail: 标题区 <font color="#1B8CD1"><strong>标题</strong></font>
正文: <table width="99%" height="25"...><tr><td><P align=center><B><FONT face=宋体>...</FONT></B></P>...</span></td>
无 meta — title 从 <title>前缀 或 strong 区, date 从列表页.
r r/ r r
Tr{ z - r r| colorz#1B8CD1)r^ strongNtabler6 r7 z99%25r} z3<font color="#1B8CD1"><strong>(.*?)</strong></font>r u" <title>([^<]*?)\s*-\s*项目公告zF<table width="99%" height="25"[^>]*>\s*<tr>\s*<td>(.*?)</span>\s*</td>z</span>) r findr rE r r] r decode_contentsr ra r[ rA )r rS r content_htmlr mtr str rs tbwhr} rK s r parse_detailr s
EL Y
6
YYw
Ct ,AzGGEN1% NE6'9)=>B* '4(H IE--( Bw#Ax$AEza4i $ B ! 1 1 3 IILiY[Y]Y] ^
+E ?BDDQA#AGGAJ/ cenprptptuA wwqzI 5 , r c
| j d|f }|j ry d} ddlm} || }| j d|||||t
t t ||f
}|j dkD ryt j dd| } t j | } t j d d
| j d d } | j d |j |t
| f y
y# t $ r Y w xY w# t j $ r=}
dt# |
v r*t% j& d t) | ||||| cY d }
~
S d }
~
ww xY w)Nz'SELECT id FROM gov_raw WHERE page_url=?Fotherr )classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)r= r r r
r zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)Tlockedr~ )executefetchone crawler_libr r! SITE_NAME
GROUP_NAME SCRIPT_NAMErowcountr r r r r lastrowidsqlite3OperationalErrorr` r" r# store_record) connr% r rg r rl curindustryr summaryr+ s r r r sF
,,1 C ||~ 5(/H ll
+ #ugtY
KQZ\d e
<<! ffZW5G ''0GffVS'288:4C@G LLf y':
# $ # # s1v JJqM c5'4K K
sF C7 C( B/C7 ( C41C7 3C44C7 7E
1E;EEEc
dd l } | j }|j j t t j t d }|j d |j d d}d}t }t dt dz D ] }t } t || }|j j dd
} t% | | } t# d | dt' | dd
| s|dkD rt# dd
n | D ] \ } }
}| |v r
|j) | t | | }|j j dd
}t+ || \ }}|s|
}t- || \ }}}t' t/ j0 dd| j3 }|dk rt# d|d d d
|dz
}t5 || |||| }|r |dz
}t# d|d d d| d d
n|dz
}t7 j8 d t7 j8 d |j; |j= t# d | d
t# d | d
t# d!t> d"d
y # t $ r }
t# d | d |
d
Y d }
~
xd }
~
ww xY w# t $ r#}
t# d| d|
d
|dz
}Y d }
~
d }
~
ww xY w)#Nr < )r zPRAGMA busy_timeout=60000zPRAGMA journal_mode=WALr )r& r r
)errorszPage z fetch error: T)flushz: found z itemszEmpty page, stop paginationz detail error z: r= r
z skip empty: z + ( z ()g333333?g ?u 新增: u 跳过: z=== z done ===) r Sessionr updater r connectDB_PATHr re r _PAGESr r, rg decoder! printr rJ rf r rx r r r r r" r# commitcloser )r sr new_count
skip_count seen_urlspager% r* r r+ rb rO r r dhtmldtitler rg rl _ plain_lenoks r mainr B s AIIW ??7B /DLL,-LL*+IJI a!$ ( c1%A (((CI 9c*
dV8CJ<v6dC /t < $) GUD)# MM' "
'1- (((C
$0w#? FL $0 w$G !GYBFF:r7 ; A A CDI2~vcr{m4DAa
dGVWdINBQ VCR[MD634@a
JJsO5 6
3Q(T KKMJJL HYK
- HZL
!. D 9
%T2S E$~aS1 >
y1#6dCa
s0 *I;(*J%; J"J J"% K.K K__main__)Nr r~ )(__doc__sysrB r r" htmlr urllib.parserF r bs4r ImportErrorBASE_URL LIST_PATHr r r r r r
_MAX_PAGESrc argvr) rz
startswithintrE
ValueErrorrJ r r r, rx r r r r __name__ r r <module>r ss
!
$ 8 i 0
%
@O'
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA$h'V"J2 j"J<3~ zF S M0
s5 C, )C9%D,C65C69D DD D