wj F d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ej j dej j ej j e
ddlmZ ej ed z
j# d ZdZd Zd
d d Zd
ZdZeez dz Zd Zd Zd Zd Z d dZ e dk( rW ej> Z e jC de"d e jG Z$ e e$jJ Z& e'de de& d yy) u crawl_qzftz_tzgg.py - 钦州港片区-通知公告 (中国（广西）自由贸易试验区钦州港片区管理委员会)
站点: http://qzftz.gxzf.gov.cn/zwdt/tzgg/
CMS: TRS (UTF-8, 服务端渲染)
列表: 标题2026-07-10
⚠️ title="" 为空 → 取 span 文本; 标题可能含 零宽空格 → strip
分页: createPageHTML(50,0,"index","shtml") → index_N.shtml 49 页 (P50 404), 每页 20 条
⚠️ CUTOFF 3 年截断 (2023-08 之前自动停)
详情: meta ArticleTitle + meta PubDate + div.content (内含 TRS_Editor) 平衡 div
N)datetime timedelta)push_to_searchdbiG )daysz%Y-%m-%du 钦州港片区-通知公告u 广西zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zhttp://qzftz.gxzf.gov.cn/)z
User-AgentRefererzhttp://qzftz.gxzf.gov.cnz
/zwdt/tzggz /index.shtmlc t j | dt d }d|_ |j S # t
$ r+}t
d| d| t j Y d }~yd }~ww xY w) N) F)timeoutheadersverifyzutf-8z[WARN] fetch failed: z - )file ) requestsgetHEADERSencodingtext Exceptionprintsysstderr)urlres %/root/gov_crawler/crawl_qzftz_tzgg.pyfetchr sZ LLgwu M
vv
%cU#aS1
Cs /2 A&!A!!A&c g }t j d| D ] }|j d |j d |j d j }}}|j dd j }t j
dd| }t
| dk rt t z d z |z }t j d
| }|r|j d nd}|j |||d |S ) u= 列表条目: ./t{sid}.shtml + span 标题 + span.time 日期zk\s*]*>\s*([^<]+)\s*\s*([^<]+) r ^\s*(?:·| |·|\s)+ /(\d{4}-\d{2}-\d{2}))r pub_datetitle) refinditergroupstripreplacesublenBASELIST_DIRsearchappend) htmlitemsmhrefr( date_textr mdpds r
extract_itemsr; ( s E
[[ H JN O
C!"QWWQZ9I9I9KYe
h+1138"eD u:> Xo#d*
YY-y
9 RXXa[B
Sb5AB
C L c t j || t j }|sy| j d|j }|dk( ry|dz }d}t j
d| |d t j D ]j }|j
d j d s |j
d j d r|dz
}I|dz }|dk( sT| |||j z c S | |d S )
u& 平衡 div 匹配 (大小写不敏感)r >r z]|Nr z .
88A; ! !& )RXXa[-C-CF-K QJE QJEzAa"((*n--
. 8Or< c \ t }|syd}t j d| }|r |j d j }|st j d|t j
}|r[t j dd|j d j }t j d| d j }|j d d j }t j d
d| }d}t j d | }|r8t j d |j d }|r|j d }d}t |d
}|j r|j }|j s,t |d }|j r|j }t j dd|t j
t j z }t j dd|t j
t j z }ddl m t j d fd| }t j d fd| }t j dd| }t j dd| }|||j fS )N)r r r r z/\s*([^<]+?)\s*z\s+z\s*-\s*r r" r# z*]*class="[^"]*content[^"]*"[^>]*>z+]*class="[^"]*TRS_Editor[^"]*"[^>]*>z)flagsz)urljoinz,href="(?!https?://|//|#|javascript:)([^"]+)"c < d | j d dS )Nzhref="r "r+ rH r rL s r z fetch_detail..r s &bhhqk)B(C1 E r< z src="(?!https?://|//)([^"]+)"c < d | j d dS )Nzsrc="r rN rO rP s r rQ z fetch_detail..t s %RXXa[(A'B! D r< z%^(]*>(\s*()?\s*)?)+z%(]*>(\s*()?\s*)?)+$)
r r) r2 r+ r, DOTALLr. splitr- rI r@ urllib.parserL ) r r4 r( r6 r8 r9 contentcrL s ` @r fetch_detailrX M s
:D E
DdKA
" II6bii H FF62qwwqz288:EHHZ/288:E MM(B ' - - /E FF4b% @EI
?FA
YY-qwwqz
:
IG T#NOAwwy'') ==? 'U V 779ggiGff+R BDD@PQGff)2wbii"$$>NOG$ffDEwPGff5DgOG ff=r7KGff=r7KG )W]]_ ,,r< r c g }t }t | D ] }|dk( rt nt t z d|dz dz }t | }|st
d|dz d nt | }|s6t | dk rt
d|dz d nt
d|dz d nt
d|dz d
t | d |D ]j }|d r
|d t k r|d
|v r |j |d
t |d
\ }} }
|s|d }|
st
d|d
_| xs |d } g } t j d|
t j D ] }
|
j d t j dd|
j d j# }}t j$ d|t j& sd|j) v s| j+ |xs d|d |j+ ||d
|d
| t, |
d| rt/ j0 | d ndt2 d m t
d t | d |D cg c] }|d j# s| }}|r t5 |d t | S c c}w ) Nr z/index_r z.shtmlz Page z: fetch fail (stop)i z: 404/empty (stop)z: 0 items (stop): itemsr' r r( z SKIP empty content: z$]*href="([^"]+)"[^>]*>(.*?)z<[^>]+>r r z*\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps)$downloadu 附件)r( r F) ensure_asciiz[]) r( r
source_urlr' site_namerV summary attachments
group_namez Total: rV
qzftz_tzgg)setrangeLIST_URLr0 r1 r r r; r/ CUTOFFaddrX r) r* rS r+ r. r, r2 r@ lowerr3 SITE_NAMEjsondumps
GROUP_NAMEr ) max_pagesrecordsseenpgr r4 r5 itr( r8 rV r: attsamhtr valids r runrx { s G 5DI ' 'hth726(&9Q'QSz GBF8#67 8 d#4y4 Qx'9:;Qx'789
Qxr#e* V45 B*~"Z.6"9%yD HHRY (4RY(? %E9g7 0E <=,bnBDkk"I7TVT]T]^
Dxx{BFF:r288A;$G$M$M$O199 JArttTXbfgfmfmfoXoKK!-x BC
D
NN r%y5 Y7 VZdjjE.R`d (
' 'P
Ic'l^6
*+ 811Y<#5#5#7Q 8E 8 - u:
9s J)J__main__z--pages)typedefaultz [RESULT] rZ r[ )r )(__doc__r osr r) argparserk r r pathinsertdirnameabspath__file__ crawler_libr nowstrftimerg rj rm r r0 r1 rf r r; rI rX rx __name__ArgumentParserap add_argumentint
parse_argsargspagesnr r< r r s - , , ( 277??277??8#<= > (
(,,.9'2
2 <