ujJdZddlZddlZddlZddlZddlZddlZddlZ ddl Z dZ dZ dZ dZddd Zd Zd Zd Zd ZdZdZedk(reyy)uv crawl_jsrthj_xmgs.py — 江苏润天环境科技有限公司「项目公示」栏目 http://www.jsrthj.com/article/29/1.aspx ASP.NET 站: - 列表: dl.news_dl > dt > a.dt_1[href=/article/show/{id}.aspx][title] + span.dt_2[YYYY-MM-DD] - 分页: /article/29/{N}.aspx (10条/页, 共65页≈650条) - 详情: /article/show/{id}.aspx 标题 div.agent_tit, 日期 div.agent_data(更新时间：2026/6/24 15:58:15), 正文 div.agent_con - 正文为纯文本公示(环评一次/报批前/验收), 无 PDF 附件; "附件"=正文内链接(百度网盘报告下载链接+提取码 / mee.gov.cn 公众意见表) → 内嵌URL保留 Nzhttp://www.jsrthj.comu江苏润天环境zcrawl_jsrthj_xmgs.pyu江苏zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36zzh-CN,zh;q=0.9)z User-AgentzAccept-Languagec|sy|j}tjdd|tj}|j}tjdd|}|S)u=标题清洗: strip ·  实体前缀与空白z·| | | flagsz\s+ )stripresubI)ts &/root/gov_crawler/crawl_jsrthj_xmgs.py clean_titler#sN   A .ARTTBA  A vsAA Hc |}gfd}tjd||tjtjz}tjdd|tjtjz}tjdd|tj}tjdd |tj}tjd d|}t j |}t d D]&\}\}}}|jd |d d|d|d}(tjdd|}tjdd|}tjdd|}|jfS)uPHTML -> 文本, 保留 a 链接(内嵌URL)与表格, 去 script/style, 分段c||jd|jd}}tjdd|}tj|j }|r|j dr|xsdStjjt|}j|||fdtdS)N<[^>]+>r#__LINK____) groupr r html_libunescaper startswithurllibparseurljoinBASEappendlen)mhreftxtabs_urllinkss r _link_replz html_to_text.._link_repl4sGGAJ cffZS)$**,dooc*9" ,,&&tT2 dGS)*#e*R((rz&]*href="([^"]+)"[^>]*>(.*?)rz<(script|style)[^>]*>.*?rz z<(p|div|li|tr|h[1-6]|br)[^>]*> rrrrz zz\rz\n{3,}z [ \t\u00a0]+r) r r Sr rr enumeratereplacer )html_segsegr(ir$r&r%r's @r html_to_textr1.sH C E) &&:JSUSWSWZ\Z^Z^S^ _C &&12s"$$+ NC &&.244 HC &&2D#RTT JC &&R %C   C C#,UA#6^ D'3kkHQCr*iy@RSVRWW[,\]^ &&C C &&FC (C &&#s +C 99; rctjd|tj}g}|D]G\}}}tjj t |}|jt|||fI|S)u)列表页 -> [(title, art_url, pub_date)]zt
\s*
]*>(.*?)\s*\[(\d{4}-\d{2}-\d{2})\]
) r findallr+rrrr r!r)html_srcitemsoutr$r dart_urls r parse_listr9Osl JJ"$$ E C1 a,,&&tT2 KNGQ/01 Jrctjd|tj}|rt|j dnd}|sStjd|tj}|r,t|j dj dd}tjd|}d}|rGd|j dt |j d t |j d fz}tjd |tj}d}|r|j dj}|xsd |xsd |gfS) u4详情页 -> (title, publish_date, body_html, links)z"
(.*?)
rrz(.*?)-ru0更新时间[：:]\s*(\d{4})/(\d{1,2})/(\d{1,2})z %s-%02d-%02drz"
(.*?)
N)r searchr+rrsplitintr )r4r8r#titlepublish body_htmls r parse_detailrC[s 7244HA'(K #bE  II-x >  0 0 5a 89E ExPAG AGGAJAGGAJQWWQZ#QQ 7244HAIGGAJ$$& =D'/T9b 88rctj}|jdtdd|jddd|jd t |j }|j }|j}|jd stjj|}tj|}|j}d }d }d }tj } t#d|j$dzD]} t&d | d} | j)| t*t,} d| _t5| j6}t3d| dt9|d| |si|D]V\}}} |j;d||f|j=d d kDr|dz }9| j)|t*t,}d|_t?|j6|\}}}}|jAst3d|dd|dz }|r|}|r|}tC|\}}|rdjEd|Dnd}|r|ddnd}|j;dtF||||d |ddd |dd|tHd|vrdnd |f|j;d |jJ|tF|f|jM|dz }t3d!|d"|dd#YtOjPd%|jSt3d&|d'|d(|y#t0$r$} t3d| d| d| |dz }Yd} ~ *d} ~ wwxYw#t0$r$} |dz }t3d$|ddd| Yd} ~ d} ~ wwxYw))Nz--pagesru 抓取页数)typedefaulthelpz--dbz/mnt/data/search.dbuSQLite 数据库路径)rFrGz --script-name)rFz/mnt/rz /article/29/z.aspx)headerstimeoutzutf-8[u] 请求失败 z: u ] 列表 u 条 z?SELECT COUNT(*) FROM gov_raw WHERE page_url=? AND script_name=?u 跳过空正文: r)c3&K|] }|d yw)rN).0as r zmain..s%6qad%6srizINSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, status, category, visits, content, tags, industry, attachments, group_name, has_table, script_name) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)otherz!$q(NI[['7[K% 7CBGGW7U4u#))+/cr |<=NI#E%H),7 :;&++%6A%66+2'$3- A%1grSUWXZacegzG8K1QRT_a  `]]E9g> Q XJacr |45A# 2H 3c1f JJL OI;j :i[ QR_  AdV?3%r!5 6 NI  R 2Q uSbzl"QC011 2s>#L =/L;-A$L;B9L; L8L33L8; M(M##M(__main__)__doc__rShtmlrrZr r]rr urllib.parserr`r rnrVrorerfrr1r9rCr__name__rMrrrs}    $  '   B 9,HSV zFr