yE}jL1dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZejj!ejj#eZejj)edZdZdZdZd Zd Zd Zd ZddlZddlZej@Z!d diZ"ddZ#ddZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,e-dk(re,yy)u 荆州生态环境局 — 环评公示 爬虫 ======================== API模式（POST JSON → 列表页）： python3 crawl_jz.py # 增量（默认：只爬第1页） python3 crawl_jz.py --full # 全量：223页 python3 crawl_jz.py --pages=5 # 指定页数 python3 crawl_jz.py --sync # 同步本地数据到服务器 数据流：本地 quality_results.db → SCP sql文件 → 服务器 search.db N)datetime)urljoinzquality_results.dbzroot@1.94.217.116/root/search.db荆州市生态环境局zjzssthjj.zwgk.jingzhou.gov.cnzNhttp://jzssthjj.zwgk.jingzhou.gov.cn/api/content_center/document/list-data-newz$http://jzssthjj.zwgk.jingzhou.gov.cnz User-Agentz>11#wPS1TYY[\ \4s ABBc ttdddd|ddtjd}|r|jdd k7ry |d jd g}|d jd d}||fS)u通过API获取一页列表ZiOFT)dept_id column_id has_relatedis_contain_sub page_index page_sizereturn_data_type randNumbercode)Nrrlisttotalr)r"API_URLtimeget)pageritemsr2s rfetch_list_pager8/sz 'E%ddiik$ D 488F#s* L  VR (E L  Wa (E %<cZt|}|syd}tjd|tj}|r4tjdd|j dj }|s[tjd|tj}|r4tjdd|j dj }d}tjd|tj}|r5|j dj }tjd d|tjtjz }tjd d|tjtjz }tjd d|}tjd d|}tjdd|tj }tjdd|tj }d}tjd||tj }||dS)uE获取详情页：正文HTML（↔）Nr&z2]*class=['"]zw-box-title['"][^>]*>(.*?)<[^>]+>z(.*?)z \s*[-_].*$u%(.*?)z]*>.*?)flagsz]*>.*?z

\s*\s*

u+\s*关闭\s*z#
.*?
uh
\s*

相关(?:链接|文档|图片|音频|视频)

.*?
c g}tjd|jdtjD]|}|jdj tj dd|jdj }}|sY|s\|j dj||~dj|S) Nz.]*href=[\'"]([^\'"]+)[\'"][^>]*>(.*?)rr<r;r&z

{}

 ) refinditergroupDOTALLstripsubappendformatjoin)mparasamhreftexts r att_cleanzget_detail..att_cleanasEkk"SUVU\U\]^U_acajajk RXXa[..0"&&RRS2U2[2[2]dDLL!=!D!DT4!PQ R99U# #r9uH
\s*

相关附件

.*?
)titlecontent)rrAsearchrDrFrCrEI)rhtmlrPhmtmrQcmrOs r get_detailrX>s C=D  E I4QSQZQZ [B z2rxx{399;  YY.bii @ FF="bhhqk:@@BEG ;T299 MB ((1+##%&&5r7"))TVTXTX.Y&&3R RTRVRVW&&2B@&&GWU&&?WTVT]T]^&& w ryy* $&& W wbii1 w //r9c||sytjdd|}tjdd|}tjdd|}tj|}tjdd|}tjd d|}|j d d }gd }g}|j dD]L}|j s|j d'tfd |Dr<|j Ndj|}tjd d|}|j S)u%从HTML提取纯文本，保留段落r&zz z r@r;z[ \t]+ z\n{3,}N,) u 扫一扫u打印|u关闭u 相关链接u 相关文档u 相关图片u 相关音频u 相关视频u 相关附件c3&K|]}|v yw)N).0nlss r z%extract_plain_text..s2Q"W2s) rArFhtml_modunescaperEsplitrGanyrI) html_contentrN noise_lines clean_linesliner`s @rextract_plain_textrjos  662FL ID 66,d +D 66*b$ 'D   T "D 66)S$ 'D 66)VT *D ::< DUKK 4 # ZZ\   r "2k22   r " # 99[ !D 66)VT *D ::<r9ctjt}|jd|j |j y)Na CREATE TABLE IF NOT EXISTS crawl_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_id INTEGER DEFAULT 0, title TEXT, url TEXT UNIQUE, content TEXT, publish_date TEXT, summary TEXT, domain TEXT, category TEXT DEFAULT '', content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) )sqlite3connect QUALITY_DBexecutecommitclose)conns r ensure_tablerss5 ??: &DLL  KKMJJLr9ctjt}|jd|f}|j }|j |duS)Nz(SELECT id FROM crawl_results WHERE url=?)rlrmrnrofetchonerq)rrrcurrs ris_url_crawledrxsA ??: &D ,,AC6 JC AJJL D=r9c|jddj}|rt|dksd|vsd|vr#td|jdddddy |jd djdd }|r&t j d |std |dy t jt} |jd||d|jdd||jddtdtj|jddxsdjjf|j|j dkD|j#S#|j#wxYw)NrPr&<>u ⚠️ 跳过无效标题: '('Fdate z^\d{4}-\d{2}-\d{2}$u ⚠️ 跳过无效日期: 'zINSERT OR IGNORE INTO crawl_results (title, url, content, publish_date, summary, domain, category, content_hash) VALUES (?,?,?,?,?,?,?,?)rrQsummaryu 环评阶段r)r5rElenprintrAmatchrlrmrnroDOMAINhashlibmd5r hexdigestrp total_changesrq)itemrPrrrs r store_itemrsR HHWb ! ' ' )E CJNcUlcUl .txx/CCR/H.IKL 88FB  % % ' ,D BHH3T: .tfA67 ??: &D   'DK)B!7 XXi #V^ [[$((9R06B>>@ A K K M O   !!A%  s B"E44Fc 6tdtjt}|j dt fj }|j|s tdytjd}|j dd}d}|D]}|\}}}} } |xsd j}|rt|d ksd |vsd |vr8| xsd jdd } | rtjd| sd } |j d||xsd | | xsd dd|d |f} | jdk(r$|j d|||xsd | | xsd dd|d f|jdkDr|dz }|dkDr3|j|j d|f|j|j d|fj!d} |jtd|dt|d| dy#t$r} td| Yd} ~ d} ~ wwxYw)u@将本地数据直接写入 search.db（服务器本地模式）u 📤 同步到 search.db...z_SELECT title, url, content, publish_date, summary FROM crawl_results WHERE domain=? ORDER BY idu 本地没有数据NrzPRAGMA journal_mode=WALrrr&rzr{r|rz^\d{4}(-\d{2}){2}$zfUPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, site_name=?, tags=? WHERE page_url=?r[zmINSERT INTO gov_raw (title, page_url, content, publish_date, summary, site_name, tags) VALUES (?,?,?,?,?,?,?)r<z Error: zINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z OK /u 条同步到 search.db (DB共u条))rrlrmrnrorfetchallrqrErrArrowcountr Exceptionrpru)rrrowsdst site_name new_countrwrPrrQpub_daterrver2s rsync_to_serverrs: )* ??: &D <<i   hj  JJL  $% //+ ,CKK)**II #12.sGXw"##%E Q#,#, N))+CR0 BHH%:HEH #++# 2x'-R#1F SUWZ[C ||q  -CB7=b$3:OQZ\^_   1$Q 3#:1}   QL   KK89, HHP ST VEIIK E)Ac$i[(FugT RS# # IaS/ " " #s>A+G66 H?HHctjdd}d|v}d|v}|r ty|rtnd}|Dcgc]}|j ds|}}|r t |dj dd}|rdnd|d }td td |d td td}dx}} td|dzD]h} t| \} } | (td| dd|dd| dkDrn;t<|dk(r| }dx} }| D]}|jdd}|st|r |dz }| dz } .t|}|jdd||jddxsdddddd}|rN|jdxs|d|d<|jdd|d<t|jdd|d<t!|| dz } |dz }td| dd|dt#| d| d|d dzd | d!zdk(r tt%j&d"ktd#|d$| d%||dkDr tytd&ycc}w)'Nr<z--fullz--syncz--pages=r=u全量u 增量（u页）u🔍 u 爬虫 — u模式 (u页)z [p3dru] ✗ API失败r&)endpub_urlrPrel_timer)rPrrrQrrQru] ✓ u 条 (新增u 跳过)z  r%g333333?u 📊 完成！新增: u | 跳过: u | 总数: u 无新数据，跳过同步)sysargvr TOTAL_PAGES startswithintrdr SITE_NAMErsranger8r5rxrXrjrrr4sleep)args full_mode sync_onlypages_to_crawla pages_matchmode total_site new_total skip_totalr6r7r2page_new page_skiprpage_urldetailentrys rmainrs 88AB1?K?[^11#6q9: 8 .1A&HD E)KvXk]$ GHNJI a!+,%&t, u = Db >"2/B Kax GX ?J  9 *Dxx 2.HXh'Q a  )F'2.*R06B< E!'G!4!Fgg#)::i#<i #5fjj26N#Oi u  MH9>9- *0 T"IQ~.fSZL H:U\]f\gghilrrx|} "9> G 3K%N &ykZL T^S_ `a1} -.o@s II__main__)r%).__doc__rosrArrlrr4base64 subprocessrTrbr urllib.parserpathdirnameabspath__file__BASE_DIRrIrn SERVER_SSH SERVER_DBrrr3BASE_URLrsslurllib.requestr_create_unverified_contextrrrr"r8rXrjrsrxrrr__name__r]r9rrs WVVVV  77??277??84 5 WW\\($8 9  & ( Z 1 $c$$& W X  /0b0(8?TF@/D zFr9