uj&dZddlZddlZddlZddlZddlZddlZddlZddl Zddl Zddl Z e jdejjejjeZejj%deejj%dejj'eddddlmZdZdZd Zej2d d Zd Zd ZdZdZefdZddZ dZ!ddZ"e#dk(rqdZ$ejJddZ&e'e&D])\Z(Z)e)dk(s e(dze*e&ks e+e&e(dzZ$+e$ e&D]Z) e+e)jYdZ$ne"e$yy#Y]xYw#Y4xYw)u 第五师双河市-环评公示 爬虫 (CMS AJAX) 站点: www.xjshs.gov.cn 栏目: 环评公示 (catid=177, category=35) API: POST api-ajax_list-{page}.html Nignorez..crawler) fetch_pageu第五师双河市-环评公示zwww.xjshs.gov.cnzhttps://www.xjshs.gov.cn SEARCH_DBz/root/search.dbc |sytjt}|jd|jd|jd|jd|jd|j dx}}|D]} |j dxsddd }|j d xs|dd }|j d xsd}|j d xsddd }|j dxsd} |j dxs| } |j dxsd} |jdt | | ||||d| f |jdkDr|dz }n|dz }|j  |jdt f|j |jtd|d||S#|dz }Y@xYw#t$r} td| Yd} ~ Ld} ~ wwxYw)NzPRAGMA journal_mode=WALzPRAGMA busy_timeout=8000zPRAGMA synchronous=NORMALa"CREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT, source_url TEXT UNIQUE, page_url TEXT, title TEXT, publish_date TEXT, summary TEXT, content TEXT, status TEXT, category TEXT, tags TEXT, attachments TEXT )zCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5( title, site_name, summary, content='gov_raw', content_rowid='id' )rtitleisummarycontent publish_date source_urlurl attachmentszINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, attachments) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)activezINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)z [FTS ERROR] u [DB] 新增: , 跳过: ) sqlite3connectrexecutecommitget SITE_NAME total_changes Exceptionprintclose) itemsdbokskipitemr r r pub_daterpage_urlres /root/gov_crawler/crawl_xjshs.py insert_dbr(s   #BJJ()JJ)*JJ*+JJ  JJ  IIKMB XXg&,"ds3Exx *3eTc:Ghhy)/RG06B4'G >G G,G''G,cV|sytjdd|tjtjz}tjdd|tjtjz}tjdd|tj}|j S)Nr z]*>.*?flagsz]*>.*?z )resubDOTALL IGNORECASEstrip)htmls r' clean_contentr2Qsp  66,b$bii"-->W XD 66.D BMM@Y ZD 66-T ;D ::<cp|sy|}tjdd|tj}tjdd|tj}tjdd|tj}tjdd|tj}tjd d|tj}tjd d|tj}tjd d|}tjd d |}tjdd|}tjdd|}tjdd|}tjdd|}tjdd |}tjdd|}|jS)Nr z  r*z

z zzzz5<(?!/?(?:table|tr|td|th|thead|tbody|tfoot|img))[^>]*>z  z<z&&z\n{3,}z[ \t]+z-(?<=[\u4e00-\u9fff])[ \t]+(?=[\u4e00-\u9fff]))r,r-r/r0)r1texts r' extract_textr<Ys<  D 66,d"-- @D 66'64r}} =D 66)T4r}} =D 66(D$bmm ]*)?z]*href=["\']([^"\']*?(?:z))["\'][^>]*>([^<]*)http/u [附件: z]()r5) r,findallr/ startswithrstriplstripappendr0join) r1base_url ext_patternlinkspartshref_r;full_urls r'extract_attachmentsrNps FK JJ(;69TT bmm E  E!>aD??6249MPS9SVZVaVabeVf9f yb !<=> 99U r3rctd|d}d} tjj||j }|j dd|j ddtjj |d }tj|j}|jdg|jddfS#t$r}td |d |Yd}~y d}~wwxYw)Nz/api-ajax_list-z.htmlzajax_type[]=18_xxgk&ajax_type[]=177&ajax_type[]=18&ajax_type[]=xxgk&ajax_type[]=Y-m-d&ajax_type[]=35&ajax_type[]=20&ajax_type[7][]=inputtime+DESC&ajax_type[7][]=displayorder+DESC&ajax_type[7][]=is_top+DESC&ajax_type[]=null&is_ds=1)dataz User-Agentz Mozilla/5.0z Content-Typez!application/x-www-form-urlencodedtimeoutu ❌ API请求失败(page=z): )NrrPtotalr) BASE_URLurllibrequestRequestencode add_headerurlopenjsonloadsreadrrr)pager post_datareqresprPr&s r' fetch_listrcs JodV5 1C ? nn$$Sy/?/?/A$B |]3 ~'JK~~%%c2%6zz$))+& 88FB '1!5 55  +D6QC89sBC C/C**C/c||rd|vryt|d}|sytjd|tj}d}|r |j dj }n_tjd|tj}|r8|j dj }tj dd |}|}|jd }|d kr|jd }|d kry|jd |}|d kry||dzd}d} d } | d kDra| t|krS|| | dzdk(r|| dz| dzd k(s | dz } | dz } n|| | dzdk(r | dz} | dz } n| dz } | d kDr| t|krS|d| dz } t| } t| } t| }| r|jdz| z}||| dS)u,获取详情页内容，处理 302 重定向z xjshs.gov.cnNrQrRzC]*content=["\']([^"\']+)["\']rz([^<]+)u.^第五师双河市人民政府\s*[-–—]\s*r zclass="xxgk_content"rzclass='xxgk_content'r9z#-E QJE 1HC #c!e_ ( QJE 1HC 1HC !)c$ivA,KK(G%k2K(L#**,/EE S " r3c tddtdttdtdtd\}}|ytj|t z }td|dt d||r|d kDr||kr|}td |d |r t |ng}td |dzD]T}t|\}}|r0|j|td |d|dt|dCtd |d|dVtdt|dd x}} g} t|dD]\} } | jdd} | jdd}| jdd}td| dt|d|dddt| }|| dz } p|jdxs|}|d}| j|| | |dd|dd||jddd|dz }td t|d!|ddtddtd"|d#| d$t||z | z | r t| tdtd%t|d&|d'| y)(Nr5z<============================================================u🚀 u 📥 获取列表...ru 总条数: u , 每页: u , 总页数: ru --pages 限制: 取前 u 页u 第 r?u 页: u 条u 页: 空u 共获取: rr r inputtimez [z] <z...r rr)r rrr r r ru ✅ 正文: u 字, 日期: u 📦 写入数据库... 成功: u , 失败: ru✅ 完成! 共处理 u 条，成功 u ，失败 )rrrcmathceil PAGE_SIZElistrangeextendrk enumeraterrxrEr() max_pagesarticlesrT total_pages all_itemspartsrLsuccessfaildb_itemsir#rr date_strdetaildb_title content_texts r'runrs Bvh- E)  VH "# mOHe))EI-.K N5'I;mK= QRY]y;'>  *9+T:;#+XI 1kAo &8Q-a    T " GA3a }F3t9+TB C GA3a }I6 7 8 N3y>*$ /0GdHY*R4hhub!"%88K, A3aI'r%*S9:c" > AID ::g&/%i( $SbM~#!::mR8   1  L 12.#2PQ3R6 Bvh- ,WIZvZPST]P^_fPfgkPkOl mn( VH "3y>"2. TXSY Z[r3__main__z--pages-)r)r)N)-__doc__sysosr,r\rr~urllib.requestrV urllib.error urllib.parsewarningsfilterwarningspathdirnameabspath__file___HEREinsertrF crawler_librrDOMAINrUgetenvrrr(r2r<rNrcrxr__name__max_pargvargsrrargrkintrDr3r'rs^]]]]! 125277<<tY78" -  % BIIk#4 5  4l.(0  6(9vA\F z E 88AB