uj dZddlZddlZddlZddlZddlZddlZddlZ ddl Z dZ dZ dZ dZddd Zd Zd Zd Zd ZdZdZedk(reyy)u( crawl_jskydhb_xxgk.py — 江苏科易达环保科技股份有限公司「信息公开」栏目 http://www.jskydhb.com/index.php/List/index/cid/237.html PHP ThinkPHP 风格: - 列表: div.childcc > p > a[href=/Show/index/cid/237/id/{N}.html] + div.time(发布日期：[2026/06/15]) - 分页: /List/index/cid/237/p/{N}.html (5条/页, 共38页=189条) - 详情: /Show/index/cid/237/id/{N}.html 标题 div.news_content > h1, 日期 div.datetime, 正文 div.news_show - 内容: 土壤污染状况调查报告/环评公示, 纯文本正文(无附件) Nzhttp://www.jskydhb.comu江苏科易达环保zcrawl_jskydhb_xxgk.pyu江苏zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36zzh-CN,zh;q=0.9)z User-AgentzAccept-Languagec|sy|j}tjdd|tj}|j}tjdd|}|S)u=标题清洗: strip ·  实体前缀与空白z·| | | flagsz\s+ )stripresubI)ts '/root/gov_crawler/crawl_jskydhb_xxgk.py clean_titler"sN   A .ARTTBA  A vsAA Hc |}gfd}tjd||tjtjz}tjdd|tjtjz}tjdd|tj}tjdd |tj}tjd d|}t j |}t d D]&\}\}}}|jd |d d|d|d}(tjdd|}tjdd|}tjdd|}|jfS)uPHTML -> 文本, 保留 a 链接(内嵌URL)与表格, 去 script/style, 分段c|jd|jd}}tjdd|}tj|j }|r"|j ds|j dr|xsdStjjt|}j|||fdtdS) N<[^>]+>r# javascript__LINK____) groupr r html_libunescaper startswithurllibparseurljoinBASEappendlen)mhreftxtabs_urllinkss r _link_replz html_to_text.._link_repl2sGGAJ cffZS)$**,dooc*dool.K9" ,,&&tT2 dGS)*#e*R((rz&]*href="([^"]+)"[^>]*>(.*?)rz<(script|style)[^>]*>.*?rz z<(p|div|li|tr|h[1-6]|br)[^>]*> rrrrz zz\rz\n{3,}z [ \t\u00a0]+r) r r Sr rr enumeratereplacer )html_segsegr)ir%r'r&r(s @r html_to_textr2-sH C E) &&:JSUSWSWZ\Z^Z^S^ _C &&12s"$$+ NC &&.244 HC &&2D#RTT JC &&R %C   C C#,UA#6^ D'3kkHQCr*iy@RSVRWW[,\]^ &&C C &&FC (C &&#s +C 99; rc(tjd|tj}g}|D]e\}}}}}tjj t |}t|} d|t|t|fz} |j| || fg|S)u)列表页 -> [(title, art_url, pub_date)]u

]*>(.*?)

\s*
发布日期[：:]\s*\[?(\d{4})/(\d{1,2})/(\d{1,2})\]?
%s-%02d-%02d) r findallr,rrr r!rintr") html_srcitemsoutr%r ymodart_urltitlepubs r parse_listr@Ms JJ j"$$ E C"*aB,,&&tT2A3r7CF33 E7C() * Jrctjd|tj}|rt|j dnd}|sStjd|tj}|r,t|j dj dd}tjd|}d}|rGd|j dt |j d t |j d fz}tjd |tj}d}|r|j dj}|xsd |xsd |gfS) u4详情页 -> (title, publish_date, body_html, links)z+
\s*

(.*?)

rrz(.*?)-ru0发布日期[：:]\s*(\d{4})/(\d{1,2})/(\d{1,2})r4rz"
(.*?)
N)r searchr,rrsplitr6r )r7r=r$r>publish body_htmls r parse_detailrH[s @(BDDQA'(K #bE  II-x >  0 0 5a 89E ExPAG AGGAJAGGAJQWWQZ#QQ 7244HAIGGAJ$$& =D'/T9b 88rctj}|jdtdd|jddd|jd t |j }|j }|j}|jd stjj|}tj|}|j}d }d }d }tj } t#d|j$dzD]} | dk(r t&d } n t&d| d} | j)| t*t,} d| _t5| j6}t3d| dt9|d| |sx|D]V\}}} |j;d||f|j=d d kDr|dz }9| j)|t*t,}d|_t?|j6|\}}}}|jAst3d|dd|dz }|r|}|r|}tC|\}}|rdjEd|Dnd}|r|ddnd}|j;dtF||||d |ddd |dd|tHd |vrdnd |f|j;d!|jJ|tF|f|jM|dz }t3d"|d#|dd$YtOjPd&|jSt3d'|d(|d)|y#t0$r$} t3d| d| d| |dz }Yd} ~ 9d} ~ wwxYw#t0$r$} |dz }t3d%|ddd| Yd} ~ d} ~ wwxYw)*Nz--pagesru 抓取页数)typedefaulthelpz--dbz/mnt/data/search.dbuSQLite 数据库路径)rKrLz --script-name)rKz/mnt/rz"/index.php/List/index/cid/237.htmlz /index.php/List/index/cid/237/p/z.html)headerstimeoutzutf-8[u] 请求失败 z: u ] 列表 u 条 z?SELECT COUNT(*) FROM gov_raw WHERE page_url=? AND script_name=?u 跳过空正文: rr*c3&K|] }|d yw)rN).0as r zmain..s%>qad%>srizINSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, status, category, visits, content, tags, industry, attachments, group_name, has_table, script_name) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)otherzr=pub_datedrd_titled_date content_htmlr(content links_outatt_txtsummarys rmainrqs  "BOOICOHOOF$9@XOYOOO[O9 ==?DggG""K   g &''$$W- ??7 #D ++-CIII Gaa(4 19F<=CF:4&FC  C' BA AJ 166" $yU E#78 (-# 2 $E7H" 2 ]`git_uv<<>!$q(NI[['7[K% 7CBGGW7U4u#))+/cr |<=NI#E%H%1,%?"BK&++%>I%>>QS+2'$3- A%1grSUWXZacegzG8K1QRT_a  `]]E9g> Q XJacr |45A# 2H 3i4l JJL OI;j :i[ QR_  AdV?3%r!5 6 NI  R 2Q uSbzl"QC011 2s>)#L /M rs}    # %  '   @ 9,KS\ zFr