Hj*dZddlZddlZddlZddlZddlZddlZddlmZddl Z ejdejjejjeZejj#deejj#dejj%edddZdZd Zej,d d Zgd Zd ZdZdZdZddZdZdZdZ ddZ!e"dk(rOdZ#ejHddZ%e&e%D])\Z'Z(e(dk(s e'dze)e%ks e*e%e'dzZ#+e!e#yy#Y;xYw)ur 双江县人民政府-通知通告 爬虫 站点: www.shuangjiang.gov.cn TLS 兼容: --ciphers AES128-GCM-SHA256 N BeautifulSoupignorez..crawleru双江县-通知通告zwww.shuangjiang.gov.cnzhttps://www.shuangjiang.gov.cn SEARCH_DBz/root/search.db)curlz-sz-Lz --max-time20z-kz --cipherszAES128-GCM-SHA256c\tjt|gzddd}|jS)NT)capture_outputtexttimeout) subprocessrun CURL_OPTSstdout)urlresults crawl_shuangjiang.pyrrs( ^^I-dWY ZF ==c |sytjt}|jd|jd|jd|jd|jd|j dx}}|D]} |j dxsddd }|j d xs|dd }|j d xsd}|j d xsddd }|j dxsd} |j dxs| } |j dxsd} |jdt | | ||||d| f |jdkDr|dz }n|dz }|j  |jdt f|j |jtd|d|y#|dz }Y?xYw#t$r} td| Yd} ~ Kd} ~ wwxYw)NzPRAGMA journal_mode=WALzPRAGMA busy_timeout=8000zPRAGMA synchronous=NORMALa"CREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT, source_url TEXT UNIQUE, page_url TEXT, title TEXT, publish_date TEXT, summary TEXT, content TEXT, status TEXT, category TEXT, tags TEXT, attachments TEXT )zCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5( title, site_name, summary, content='gov_raw', content_rowid='id' )rtitleisummarycontent publish_date source_urlr attachmentszINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, attachments) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)activezINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)z [FTS ERROR] u [DB] 新增: u , 跳过: ) sqlite3connectrexecutecommitget SITE_NAME total_changes Exceptionprintclose) itemsdbokskipitemrrrpub_daterpage_urlres r insert_dbr4s   #BJJ()JJ)*JJ*+JJ  JJ  IIKMB XXg&,"ds3Exx *3eTc:Ghhy)/RG06B]*>.*?rflagsz]*>.*?z )resubDOTALL IGNORECASEstrip)htmls r clean_htmlr>Ssj 66,b$bii"-->W XD 66.D BMM@Y ZD 66-T ;D ::<rc|sy|}tjd|tjtjz}g}t |D]1\}}d|d}|j ||f|j ||d}3tjdd|tj}tjd d |tj}tjd d|tj}tjd d|tj}tjd d|tj}tjdd|tj}tjdd|}tjdd|}tjdd|}tjdd|}tjdd|}tjdd |}tjdd|}tjdd|}|D](\}}t|}|j |d|zdz}*|jS)Nrz]*>.*?__TABLE___r!z  r6z

z zzzzz5<(?!/?(?:table|tr|td|th|thead|tbody|tfoot|img))[^>]*>z  z<z&&z\n{3,}z[ \t]+z-(?<=[\u4e00-\u9fff])[ \t]+(?=[\u4e00-\u9fff])) r8findallr:r; enumerateappendreplacer9html_table_to_htmlr<)r=r tables placeholdersitblphmds r extract_textrRZs  D ZZ2D"))bmm:S TFLF#(3s" RI&||CQ'( 66,d"-- @D 66'64r}} =D 66)T4r}} =D 66(D$bmm @H--h]*class="textList1"[^>]*>(.*?)r!z]*>(.*?)ztitle="([^"]*)"zhref="(/artview/\d+/\d+\.html)"%]*>(\d{4}-\d{2}-\d{2})r)rrdate) rr8searchr:groupfinditerr<rIBASE_URL) r2r=r,dl_mareamddtitle_mhref_mrrUdate_mdate_strs rfetch_list_pagerss >D E 99A4 SD  ::a=D [[/ryy A  WWQZ)).3=rBf  a &&(||ACRH&,6<<?" d?   ( Lrctjd|tj}|r|jd}tjd|}|r3|jd}|j ds t |z}d|d}|St t|}t|dkDr|jStjd|tj}|r|jd}tjd|}|r1|jd}|j ds t |z}d|dSt t|}t|dkDr|jSy ) u详情页正文提取zA]*class="articleBox"[^>]*>(.*?)]*class="text"[^>]*>r!z]*src="([^"]*\.pdf)"rVu[附件：PDF文件]()z,]*class="articleBox"[^>]*>(.*?)r) r8rgr:rhr^rjrRr>lenr<)r=rminnerpdf_mpdf_urlr s rextract_content_from_detailr{s7 VX\^`^g^ghA  ;UC kk!nG%%f-"W,*7)15DKJu-. t9q=::<  A4SA  ;UC kk!nG%%f-"W,*7)15 5Ju-. t9q=::<  rc"t|}|syd}tjd|}|r|jdj }|s`tjd|}|rH|jdj }tj dd|j }|r|}d}tjd|}|r|jd}|s)tjd|}|r|jd}t |}|||d S) u 获取详情Nz.]*class="articleTitle"[^>]*>(.*?)r!z([^<]+)u#[-–—|]\s*双江县人民政府$rreu(\d{4}-\d{2}-\d{2})\s+来源)rrrf)rr8rgrhr<r9r{)rr=rtttrrdprs r fetch_detailrs 9D  E Dd KB  !!#  YY0$ 7  !!#A=r1EKKMAH ;T BB 88A;  YY6 = xx{H)$/Gw AArc  tddtdttdtdttd}tdt |dt td}d}t jd |}|rt|jd }t jd |}|rd}|r||kr|}td |t|}td |d zD]}td|d}t|} | std|d|d/|D chc]} | d } } | D cgc] } | d| vs | } } |j| td|d|dt | dt | d tdt |ddx} }g}t|d D]\} }td| dt |d|ddddt|d}|std|d z }K|jdd }|rt |d!krtd"|d z }~|jdxs|d}|jd#xs|d#}|j!||d|d|dd$|dd%|d d&| d z } td't |d(|tddtd)| d*||r t#|tdtd+t |d,| d-|ycc} wcc} w).NrBz<============================================================u🚀 u 📥 获取列表...z /tztg.htmlu 第1页: u 条zmax\s*=\s*"(\d+)"r!u/24页u 总页数: z /tztg/index_z.htmlu 第 /u 页: 空ru 页: u 条 (新增 ruu 共获取: rz [z] r<z...u ❌ 请求失败rru ❌ 正文为空rfr)rrrrrrru ✅ 正文: u 字, 日期: u 📦 写入数据库... 成功: u , 失败: u✅ 完成! 共处理 u 条, 成功 u , 失败 )r*r'rsrjrwrr8rgintrhlistrangeextendrHrr&rIr4) max_pagesitems1html1 total_pagesmmmm2 all_itemspnrr,rN seen_urls new_itemssuccessfaildb_itemsr0detail content_textdb_titler1s rrrs^ Bvh- E)  VH "# z4 5F KF }D )* H:Z( )EK ' /B "((1+& ))Iu %C  Y,  M+ '(V IA{Q' [ ,rd%0$ F2$a }I6 7 '01!QuX1 1 %C15)BQC C# rd!K=s5zl-IGWWXYZ [ OC N+4 01GdHY*M4 A3aI'r$w-*<)=SABd5k* ' ) AID zz)R0 s<014 ' ) AID ::g&7$w-::f%5fu+;$SbM~#   1  L 12. KL7M: Bvh- ,WIZv FG( VH "3y>"2-y RVQW XY[2Cs& K<8 LL__main__r!z--pages)r)r)N)+__doc__sysosr8rr"warningsr[r urllib.parser_filterwarningspathdirnameabspath__file___HEREinsertjoinr'DOMAINrjgetenvrrrr4r>rRrKrsr{rr__name__max_pargvargsrHrNargrwrrrrsW 211! 125277<<tY78 $ ! + BIIk#4 5 \  32l> B@BBKZ\ z E 88AB