zj3.dZddlZddlZddlZddlZddlZddlZejdddlmZejjejje Z ejjde ejjdejje ddddlmZdZd Zd Zej*d d Zd ZdZdZdZefdZdZdZdZddZe dk(rdZ!ejDddZ#e$e#D])\Z%Z&e&dk(s e%dze'e#ks e(e#e%dzZ!+e!?e#D]:Z&e&jUdjWs#e(e&jUdZ!nee!yy#e)$rYwxYw)u 海宁市人民政府信息公开平台 爬虫 (JCMS/JPAAS) 站点: www.haining.gov.cn 栏目: col1455897 - 重点领域信息公开 > 环境保护 (E1-2) API返回全量列表HTML，逐条抓取详情 Nignore)datetimez..crawler) fetch_pageu#海宁市-重点领域-环境保护zwww.haining.gov.cnzhttps://www.haining.gov.cn SEARCH_DBz/root/search.dbahttps://www.haining.gov.cn/api-gateway/jpaas-publish-server/front/page/build/unit?parseType=bulidstatic&webId=2780&tplSetId=oo54alAKysOEKkUzWqYCa&pageType=column&tagId=%E7%BB%84%E9%85%8D%E5%88%86%E7%B1%BBlist&editType=null&pageId=1455897¶mJson=%7B%22pageNo%22%3A1%2C%22pageSize%22%3A99999%2C%22search%22%3A%22%7B%5C%22xxgkId%5C%22%3A%5C%22E1-2%5C%22%2C%5C%22xxgkType%5C%22%3A%5C%22xxgk_combination%5C%22%2C%5C%22className%5C%22%3A%5C%22%5C%22%7D%22%7Dc |sytjt}|jd|jd|jd|jd|jd|j d}d}|D]} |j dxsd dd }|j d xs|dd }|j d xsd }|j d xsd dd}|j dxsd } |j dxs| } |j dxsd } |jdt | | ||||d| f |jdkDr|dz }n|dz }|j  |jdt f|j |jtd|d||S#t$r |dz }YGwxYw#t$r} td| Yd} ~ Td} ~ wwxYw)u;直接连接 /root/search.db，写入 gov_raw + 同步 FTS5NzPRAGMA journal_mode=WALzPRAGMA busy_timeout=8000zPRAGMA synchronous=NORMALa"CREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT, source_url TEXT UNIQUE, page_url TEXT, title TEXT, publish_date TEXT, summary TEXT, content TEXT, status TEXT, category TEXT, tags TEXT, attachments TEXT )zCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5( title, site_name, summary, content='gov_raw', content_rowid='id' )rtitleisummarycontent publish_date source_urlurl attachmentszINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, attachments) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)activezINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)z [FTS ERROR] u [DB] 新增: u , 跳过: ) sqlite3connectrexecutecommitget SITE_NAME total_changes Exceptionprintclose) itemsdbokskipitemr r r pub_daterpage_urlres "/root/gov_crawler/crawl_haining.py insert_dbr'%s    #BJJ()JJ)*JJ*+JJ  JJ  IIK B D XXg&,"ds3Exx *3eTc:Ghhy)/RG06B4'G>GG G4G//G4cV|sytjdd|tjtjz}tjdd|tjtjz}tjdd|tj}|j S)uD从 HTML 中提取纯文本，去掉 flagsz]*>.*?z )resubDOTALL IGNORECASEstrip)htmls r& clean_contentr1hsp  66,b$bii"-->W XD 66.D BMM@Y ZD 66-T ;D ::<c|sy|}tjdd|tj}tjdd|tj}tjdd|tj}tjdd|tj}tjdd|tj}tjd d|tj}tjd d|tj}tjd d|}tjd d |}tjdd|}tjdd|}tjdd|}tjdd|}tjdd |}|jS)uK将清洗后的 HTML 转为纯文本（但保留表格 HTML 标签占位）r z  r)z

zzzzzzT<(?!/?table|/?tr|/?td|/?th|/?thead|/?tbody|/?tfoot|/?caption|/?colgroup|/?col)[^>]*>z  z<z&&z\n{3,}z z[ \t]+)r+r,r.r/)r0texts r&extract_text_contentr:usB  D 66,d"-- @D 66'4R]] ;D 66)T4r}} =D 66(D$bmm ]*)?z]*href=["\']([^"\']*?(?:z))["\'][^>]*>([^<]*)http/u [附件: z]()r4) r+findallr. startswithrstriplstripappendr/join) r0base_url ext_patternlinkspartshref_r9full_urls r&extract_attachmentsrLs FK JJ(;69TT bmm E  E!>aD??6249MPS9SVZVaVabeVf9f yb !<=> 99U r2c g}|s|Stjdtjtjz}|j |D]}|j dj }|j dj }|j dj }|r|se|jds*tjddz|jdzn|}|}|j|||||d|S)u<从 API 返回的 json['data']['html'] 中提取文章列表z]*>.*?]*href=["\']([^"\']+)["\'][^>]*title=["\']([^"\']*)["\'][^>]*>.*?.*?]*>([^<]*).*?rr<r=)rIr$rr date) r+compiler-r.finditergroupr/r@BASE_URLrArBrC) r0rpatternmrIr date_strr$rs r&parse_list_htmlrXs E  jj  BMM! G  d # wwqz!   "771:##%5 HLX^H_8??3'#- C0@@ei   "     Lr2c|d}t|d}|std|yd}tjd|tj}|r |j dj }ntjd|tj}|r |j dj }nFtjd |tj}|r|j dj }tjd |tjtjz}|s6tjd |tjtjz}|std |y|j d}t|} t|} t| } | r| jd z| z} || | dS)u"获取详情页 div.article 内容r$timeoutu ⚠ 详情页获取失败: NzC]*content=["\']([^"\']+)["\']rz7]*class=["\']title["\'][^>]*>\s*([^<]+)\s*z([^<]+)z;]*class=["\']article["\'][^>]*>(.*?)\s*(?:<|$)zi]*class=["\'](?:content|main|text|TRS_Editor)["\'](?:[^>]*class=["\'][^>]*)?>(.*?)\s*(?:<|$)u ⚠ 未找到内容容器: u --- 附件 --- )r r r) rrr+searchr.rSr/r-r1rLr:rA) r"rr0 detail_titlemtdtttrVraw_htmlcleanedr text_contents r& fetch_detailres z C c2 &D  -cU34L Y[_acanan oB xx{((* YYQSWY[YbYb c 88A;,,.L4dBMMJB!xx{002  PRVXZXaXadfdqdqXqrA IICEIKMKTKTWYWdWdKd e  -cU34wwqzHH%G%h/K'0L#**,/EE S " r2c|sy|j}tjd|}|rt|j dt|j dt|j d}}}d|cxkrdkr,nn)d|cxkrdkrnnd|cxkrd krnn |d d |d d |d Stjd |}|rt|j dt|j dt|j d}}}d|cxkrdkr,nn)d|cxkrdkrnnd|cxkrd krnn |d d |d d |d Stjd|}|rt|j dt|j dt|j d}}}d|cxkrdkr,nn)d|cxkrdkrnnd|cxkrd krnn |d d |d d |d Stjd|}|rt|j dt|j dt|j d}}}d|cxkrdkr,nn)d|cxkrdkrnnd|cxkrd krnn |d d |d d |d S|ddS)u)将各种格式的日期转为 YYYY-MM-DDr z(\d{4})-(\d{1,2})-(\d{1,2})rrNrOii3 04d-02dz(\d{4})/(\d{1,2})/(\d{1,2})u"(\d{4})年(\d{1,2})月(\d{1,2})日z(\d{4})(\d{2})(\d{2})Nr)r/r+matchintrS)rWrVymods r&normalize_daterqs, ~~H /:Aqwwqz?C OS_q2 1  bB1*$ /0  56Y]y3y>'Ajy)  *9+T:;G DHY*?4 A3aI'r$w-*<)=SAB DL>$z2B3B2G1HLMd# > AID !$v,/::g&7$w-4C.i( |, #$#!-0   1  %c,&7%8=>5?: Bvh- ,WIZv FG( VH "3y>"2. TXSY Z[}    's+,sH88I# II#__main__rz--pagesrj)r})N),__doc__sysosr+rxrwarningsfilterwarningsrpathdirnameabspath__file___HEREinsertrD crawler_librrDOMAINrTgetenvrrwr'r1r:rLrXrerqr__name__ max_pagesargvargsr|rargr{rm ValueErrorrBisdigitr2r&rs,++! 125277<<tY78" 2  ' BIIk#4 5 a?F 6(0&D2lBL\^ zI 88AB