uj(DdZddlZddlZddlZddlZddlZddlmZddlZddl m Z dZ dZ ddiZ d Zd Zej d d Zd ZddiZej(Zej,j/edZdZddZdZdZdZdZdZdZ e!dk(re yy)u 建峰集团 - 环境公示 爬虫 (cnjf.com) 栏目: 信息公开 > 环境公示 (classid=26) CMS: ASP.NET 自建 列表: ul.attractbid-nav > li > a 分页: ?classid=26&page=N 详情: div.video-detail > h3(标题), span(日期), p(正文) N)urljoin) BeautifulSoupzhttps://www.cnjf.comz/aspx/ch/dutylist.aspxclassidu建峰集团-环境公示u企业 SEARCH_DBz/mnt/data/search.dbz User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36cV|sytjdd|}|jS)u>清理标题：strip · 实体前缀和多余空白z(·| |\s)+ )resubstrip)texts #/root/gov_crawler/crawl_cnjf_env.py clean_textr&s'  66*C 6D ::<c |syt|d}|ddgD]}|jg}|jD]}|js|jdk(r|j ddD]}|d }t j d |t js.|jd xs|}|jd s t||n|} |j|jd |d| d|jd } | s|j| |jdk(r|jt||jdvr|j ddD]:} | jdr| jd } | s*|j| <|j ddD]} |jt| |jdk(s|j!dd} |j!dd}| s| jd s t|| n| }|jd|d|ddj#|S)u?提取正文：段落 分隔、表格HTML保留、附件嵌入r html.parserscriptstylepaThrefrz"\.(pdf|doc|docx|xls|xlsx|zip|rar)$rhttpz [z](z) table)divsection recursiveimgsrcaltz![)z )r decomposechildrennamefind_allr searchIget_text startswithr replace_with new_stringappendstr find_parentgetjoin) html_content detail_urlsouptagpartschilda_tagra_textfull_urltrtblr#r$full_srcs rclean_content_htmlrA.s.  } 5DXw'(  E6zz  :: $7 UV}99BD"$$O"^^$^7?4F@DPV@Wwz48]aH&&tF82hZr7R'ST  U T*A Q ZZ7 " LLU $ ZZ- -^^C4^8 $==)JJTJ*LLO  $ ~~g~> ' SX& ' ZZ5 ))E2&C))E2&C;>>>&;Q7:s3WZ r#b !45=6@ ;;u rc pt|D]\} tj|d}d|_|jdk(r|j cSt d|jd|dd^y#t$r@}||dz krtjd Yd}~t d |ddd |Yd}~d}~wwxYw) N)timeoutzutf-8u ⚠️ HTTP z for Pu ❌ 请求失败: z: ) rangesessionr3encoding status_coderprint Exceptiontimesleep)urlretriesires rfetchrVZs 7^ : : C ,A AJ}}#vv N1==/s3BxjA B :   :7Q; 1  (Sb "QC8 9 9  :s#8A, A,, B55B0B00B5cg}t|d}|jd}|s|S|jddD]}|jdd}|s|d }|j d st t |}|jd }|r`|jd }|jd } |r|jdnd} | r| jdnd} | r | r| d| n| xs| } nd} |jd} d}| r.| jd }|rt|jd}|s|s|j||| f|S)u6解析列表页，返回 [(title, url, date_str), ...]rzul.attractbid-navliFr rTrrrzdiv.attractbid-timerspanrr -zdiv.attractbid-menu) r select_oner)findr-rBASE_URLr,rr0)htmlitemsr7ulrXrrtime_divyear_tag month_tagyearmonthdate_strmenu_divtitlers r parse_listrijsZ E } -D , -B  kk$%k02 GGCdG # yv&8T*D<< 56 }}S)H f-I4<8$$4$0"D6?I&&T&2RE,0U$q( HH<< 56  c"A"1::D:#9: T LL%x0 192< Lrct|d}|jd}|sy|jd}d}|D]O}|jdd}t j d|}|s.t |jd}||kDsN|}Q|S)u解析总页数rzdiv.pagerGrrr z page=(\d+))rr[r)r3r r*intgroup) r^r7page_divlinksmax_pagerrmrs rparse_total_pagesrqs } -Dz*H    c "EH uuVR  IImT * AGGAJA8|  Orct|}|syt|d}|jd}|sy|jd}|rt |j dnd}|jd}d}|r;|j d}t jd |} | r| jd }g} |jd d D]g} | jd ds| jd dr*| j d} | s?t| dkDsN| jt| idj| } ||| fS)u8获取详情页，返回 (title, date_str, content_html))NNNrzdiv.video-detailh3Trr rYz(\d{4}-\d{2}-\d{2})rGrr rz product-page)class_z masses-pager )rVrr[r\rr,r r*rlr)r2lenr0r1r4)rQr^r7 detail_divrsrh date_spanrfrrp content_partsrr> content_htmls r fetch_detailr{s< :D  } -D!34J   B35Jr{{{. /2E'IH!!!- II,d 3 wwqzHM   5) ==~= 6!--Vc-:d  JJTJ " Q!  Q ( )99]+L (L ((rcddl}|jt}|j}|j d|j |j y)NraCREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, content TEXT, publish_date TEXT, source_url TEXT UNIQUE, page_url TEXT, site_name TEXT, summary TEXT ))sqlite3connectDB_PATHcursorexecutecommitclose)r}conncs rinit_dbrsC ??7 #D AII    KKMJJLrcdtjtjddtdgdy)u跨进程同步FTSz-cz import sqlite3, os db = "a|" conn = sqlite3.connect(db) conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(title, site_name, summary, tokenize=trigram)") conn.execute("INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) SELECT rowid, title, site_name, summary FROM gov_raw WHERE rowid NOT IN (SELECT rowid FROM gov_search)") conn.commit() conn.close() print("FTS synced") T)capture_outputN) subprocessrunsys executablerrrsync_ftsrs5NN #i   rc Dddl}|jdt}|jdtt dt d|j }|j}tdtd td tttd |tddl }|jt}|j}t!ttd }|s td yt#|}t%||}td|g} t'd|dzD]} ttd } | dkDr| d| z } td| dddt!| }|s tdJt)|} tt+| d| j-| t/j0dtdt+| dt3} |j5dD]}| j7|dtdt+| d}d}d}t9| dD]\}\}} }| | vr|dz }t;| \}}}|s|dz }+|xs|}|xs|}t=|| }|r(t?j@dd|ddjCnd} |j5d |||| | t|f|jDr|dz }|d!zdk(s|dkDs|jItd"|d#|d$|d%|jI|jKtd&td'|dtd(|dtd)|d|dkDrtMtd*td+|dy#tF$r}|dz }Yd}~jd}~wwxYw),Nru爬虫: ) descriptionz--pagesu页数（默认u）)typedefaulthelpz=== u 爬虫 ===zURL: u页数: z ?classid=26u❌ 无法获取列表页u 总页数: rGz&page=u 第u页...r T)endflushu ❌ 失败u条g?u 列表总计: u 条zSELECT source_url FROM gov_rawu 已知URL: z<[^>]+>r rEzINSERT OR IGNORE INTO gov_raw (title, content, publish_date, source_url, page_url, site_name, summary) VALUES (?, ?, ?, ?, ?, ?, ?) u 进度: u条新, u 条跳过, u 条错误u === 完成 ===u 新增: u 跳过: u 错误: u FTS同步完成u 新增 )'argparseArgumentParser SITE_NAME add_argumentrk MAX_PAGES parse_argspagesrMr] LIST_PATHrr}r~rrrVrqminrIrirvextendrOrPsetradd enumerater{rAr r r lastrowidrNrrr)rparserargs max_pagesr}rrr^ total_pages all_itemspagerQr_existingrow new_count skip_count error_countrSrhrf detail_title detail_daterz final_title final_datecontentsummaryrUs rmainrs  $ $8I;1G $ HF  YW`VaadEef    D I D ; '( E(I; '( HYK ! I ??7 #D A H:i[ 4 5D  )*#D)Kk9-K K } %&Iaq) 9+[1 !8 VD6? "C dV6"48Sz ,  4  U C ! 3  S^,D 12uHyy9: SV KH  '(IJK%.y!%< a! !E3 (? !OJ 3?s2C/ k< 1 K #+e  ,H %\37CJ"&&R1$37==?PR  IIVgz3YP {{Q r>Q 9q= KKM Jyk*[ U^_ `A aD KKMJJL  Jyk &' Jzl$ '( J{m4 ()1}  !# Ii[ %&)  1 K  s .N NNN__main__))"__doc__r rrOosr urllib.parserrequestsbs4rr]r LIST_PARAMSrGROUPgetenvrrHEADERSSessionrJheadersupdaterrArVrirqr{rrr__name__rrrrs   " $ "o '  "))K!6 7 D  (   w)X  &R$#)L$ f'R zFr