ɅjEdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddlZ e e jdkDr2e jdjree jdndZeedeeze j$dej&j)ej&j+eZej&j1edZdZdZd Zd Zd d d ddZdZddddddddZ ddddZ!dZ"dZ#d,dZ$dZ%dZ&dZ'd Z(d!Z)d"Z*d#Z+d$Z,d%Z-d&Z.d-d'Z/d(Z0d)Z1d*Z2e3d+k(re2yy).u] crawl_eiacloud.py — 环评云(www.eiacloud.com) 多栏目爬虫 ======================================================== 站点: https://www.eiacloud.com/gs/list/1 (环评报告公示) 反爬: JS渲染分页，API POST /gs/gkPublishList 列表: API返回JSON，15条/页 详情: 静态HTML，正文在 div#contentContainer.detail_text 多栏目支持: 报告公示(id=1) / 验收报告(id=2) / 环评报告(id=3) 水土保持(id=4) / 土壤污染(id=5) / 竣工调试(id=6) 用法: python3 crawl_eiacloud.py # 增量爬(所有栏目) python3 crawl_eiacloud.py --full # 全量爬(仅栏目1) python3 crawl_eiacloud.py --plate 3 # 只爬指定栏目 python3 crawl_eiacloud.py --sync # 仅同步 同步机制: - 服务器本地(/root/search.db 存在): 直接写入 - Mac本地: 通过SSH推送到服务器 search.db Nz[AutoPg] max_pages=ignorezeiacloud_results.dbzwww.eiacloud.comzroot@1.94.217.116/root/search.dbz)https://www.eiacloud.com/gs/gkPublishListz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36XMLHttpRequest"https://www.eiacloud.com/gs/list/1z!application/x-www-form-urlencoded) User-AgentzX-Requested-WithRefererz Content-Type)u 报告公示iH)u 验收报告)u 环评报告r r )u 水土保持d)u 土壤污染rr)u 竣工调试rr)u 监测报告2)rrrr  )newskiperrorsctjtd}|jd|jd|j |j y)N<timeoutPRAGMA journal_mode=WALu_CREATE TABLE IF NOT EXISTS crawl_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT DEFAULT '环评云', plate_id TEXT DEFAULT '1', title TEXT, url TEXT UNIQUE, content TEXT, publish_date TEXT, summary TEXT, crawled_at TEXT DEFAULT (datetime('now','localtime')) )sqlite3connectDB_PATHexecutecommitcloseconns #/root/gov_crawler/crawl_eiacloud.pyinit_dbr'AsD ??7B /DLL*+LL    KKMJJLctjtd} |jd|j |j y#YxYw)u给已有表加 plate_id 列rrz>ALTER TABLE crawl_results ADD COLUMN plate_id TEXT DEFAULT '1'Nrr$s r&add_plate_columnr*SsB ??7B /D  UV  JJL  s !AAc jtjtd} |jd||||||f|j |j dkDrt dxxdz cc<nt dxxdz cc<|jy#t dxxdz cc<Y&xYw#|jwxYw) NrrzoINSERT OR IGNORE INTO crawl_results (title, url, content, publish_date, summary, plate_id) VALUES (?,?,?,?,?,?)rrrrr)rrr r!r" total_changesstatsr#)titleurlcontentdatesummaryplate_idr%s r& store_recordr4^s ??7B /D  #C$ :     ! %LA L &MQ M  h1 sABBB B2c d|S)Nu 环评云-)r3 plate_labels r& site_name_forr8rs  } %%r(cJ|ydt|jddzdzS)Nz''')strreplace)vs r&escr>vs(y QT* *S 00r(c@tjjdS)Nr)ospathexistsr6r(r&is_running_on_serverrC{s 77>>+ ,,r(c tdtjtd}|j dj }|j |s tdytr t|yt|y)u4同步所有栏目的数据到服务器的 search.dbu 📤 同步到 search.db...rrzZSELECT title, url, content, publish_date, summary, plate_id FROM crawl_results ORDER BY idu 本地没有数据N) printrrr r!fetchallr#rC _sync_direct _sync_via_ssh)r%rowss r&sync_to_serverrJsf )* ??7B /D <<d hj  JJL  $%Tdr(c tjdd}|jdd}tj Dcic]\}}t ||d}}}|D]p}|\}}} } } } t | |jt | d} |jd||| xsddd | xsd| xsddd | df|jdkDr|d z }r|dkDr|jtj Dcgc](\}}t |t t ||df*c}}D]\} } |jd | f|j|jdjd}|jtd|dt|d|dycc}}w#t$r}td |Yd}~od}~wwxYwcc}}w)Nrrrrr其他zwINSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, summary, site_name, tags) VALUES (?,?,?,?,?,?,?) ,rz Error: zINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?u>SELECT COUNT(*) FROM gov_raw WHERE site_name LIKE '环评云%'u ✅ OK /u 条同步到 search.db (DB共条))rrr!PLATESitemsr;r8getr, ExceptionrEr"fetchoner#len)rIdst new_countkr= plate_maprr.r/r0pub_dater2pidsnetotals r&rGrGs //+R 8CKK)*I*0,,.9$!QQ19I9 #673sGXw 3 c#h A B # KK)gmWf5x~2-R#&B0    1$Q # 1} JP,,.Y$!QQs1vqt!<=Y GC KKU    KKH hj EIIK Ii[#d),J5'QU VWC: # IaS/ " " # Zs$F,?F2?-G2 G;GGc \ddl}tjDcic]\}}t||d}}}dj dtjD}dt d|d} t jdt|gddd }|jjr2t|jjjd n t}g} |D]_}|\} } } } }}| |vrt||jt|d }| j| | | xsd dd| xsd |xsd dd|d fa| st!dt#|dyt!dt#| dd}d}t%dt#| |D]}| |||z}dj d|D}dt d|d} |j'|j)j+}d|dtdt d}t j|dddd }|j,dk7r|D] }t/||t#|z }t!d||zd zd!t#|d"|d#t#| d$ tjDcgc](\}}t|tt||df*c}}D]6\}}d%|d&} t jdtdt d'|dgdd (8 t jdtdt d)gddd* }|jj}t!d+|d,ycc}}w#t}YxYw#|D] }t/||t#|z }YxYwcc}}w#YxYw#YyxYw)-Nrz','c3RK|]\}}tt||d!yw)rN)r8r;).0rZr=s r& z _sync_via_ssh..s#W1 c!fad ;Ws%'zsqlite3 z4 "SELECT page_url FROM gov_raw WHERE site_name IN ('z')"sshT)capture_outputtextr rLrMrNrOu ✅ 所有 u+ 条已在服务器中存在，无需同步u 待同步 u 条...rz, c3K|]c\}}}}}}}dt|dt|dt|dt|dt|dt|dtddeyw)(,rM)N)r>)rdtucdsr__s r&rez _sync_via_ssh..so# $1aAr1AxqQ#a&3q6(!CF81SWIQs2wiq Q# sA)A+zATTACH DATABASE 'z|' AS maindb; INSERT OR IGNORE INTO maindb.gov_raw (title, page_url, content, publish_date, summary, site_name, tags) VALUES z; DETACH DATABASE maindb; echo  | base64 -d | ssh "sqlite3 ")shellrhriru ✅ 批r: u 条 (累计rPrnzINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name='z';z ")rhruA "SELECT COUNT(*) FROM gov_raw WHERE site_name LIKE '环评云%'"ru. 📊 服务器 search.db 环评云系列共 u 条)base64rRrSr;joinSERVER_SEARCH_DB subprocessrun SERVER_SSHstdoutstripsetsplitr8rTappendrErWrange b64encodeencodedecode returncode_sync_one_via_ssh)rIr{rZr=r[all_site_names check_cmdr\ existing_urlsbatchr.r/r0r]r2r^r_ BATCH_SIZEpushedichunk values_clausesqlencodedcmditem pid_labelfts_sqlras r&rHrHs*0,,.9$!QQ19I9ZZW WWN #$%??M>Nd T NN J *dB >?XX^^=MAHHNN,22489SVSX  E 7673sGXw -   3 c#h A B eS7=b'6":HN 2t,b"6 7 7  c$i[(STU LU G ,-J F 1c%j* -!aJ' # (-#   012%o&( )  !&&szz|4;;=G'"5j\L\K]]^_Cs$t$XZ[A||q !,D%d+, c%j F IamAo.bU KxqQTUZQ[P\\]^ _+!8MSLLN[DAq3q6=Q1#>?[  2QQSPTTV X    NN h/?.@7)1$MN#R    NN J())l m odB    >ugTJKi: R ! (!$' ( c%j F\    s>M0A6M B:M2-N*N AN' M/2"N N$'N+cddl}|\}}}}}}}dt|dt|dt|dt|dt|dt|dtdd} |j| jj } d| dt dt d } tj| d d d y#YyxYw) NrziINSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, summary, site_name, tags) VALUES (rmrMz);rurvrwrxTr )ryrhr) r{r>rrrrr}r~r) rr{rorprqrrrsr_rtrrrs r&rrsAq!Q2q q6(!CF81SVHAc!fXQs1vhaBy#b'RT V  ""3::<0779gY1*ZHXGYYZ[s$tRH  s 'ACCctj}|jjdtdi|j ddd|S)NrrrgFrverify)requestsSessionheadersupdate REQ_HEADERSrT)rss r& make_sessionr'sDAIIlK $=>?EE .5EI Hr(c t|t|ttd} |jt|tdd}t j |j}g}|D]}|jdd}|jddj} | r|s:|dk(r@|jd d } | r-tjd tj| d z } n|jd d} |jdtd|d|| | |d|S#t$r} t!d|d|d| Yd} ~ yd} ~ wwxYw)u'调用API获取指定栏目的列表页) gkPlateIdzpageModel.numberNozpageModel.numberSizergF)datarrridrMr. 21010yzwak createDaterz%Y-%m-%direadableCreateDatezhttps://z /gs/detail/z?id=)r/r.r1ru ⚠️ API错误(栏目第u页): N)r; PAGE_SIZEpostAPI_URLrjsonloadsrirTrtimestrftime localtimerDOMAINrUrE) session gk_plate_idpage_numrr\rSresultritem_idr.tsr1r`s r&fetch_list_pager.sH%!(m #I D  LLt["UZL [ 166" DhhtR(GHHWb)//1E,&,*B}}ZT 1JKxx 4b9 MM!&gYO   $  )+c(6!MNsC:D"" E +EE cr|jd}|dk(ry|jd|}|dk(ry|dz}d}|}|t|krq|||dzdk(r-|jd|}|dk7r||dz d k7r|dz }|dz}n*|||d zd k(r|dkDr|dk(r|||S|dz}|d z }n|dz }|t|krqy) Nzid="contentContainer"rM>rrrz)findrW)htmlidxgt inner_startdepthrcis r&extract_content_containerrQs ))+ ,C by 3 B Rxq&K EA c$i- !A#;& 3"BRxDAJ#- QA !AaC[H $QzK** QJE FA FA c$i- r(c t|dzD]} |j|dd}|j}t |dkr||krtjdNyd|vryd}t jd |t j}|r|jdj}|sFt jd |t j}|r|jdj}t|} d} | rt jd d| t jt jz } t jd d| t jt jz } | j} d} t jd|} | r| jd} g} t jd| xs|t jD]}|jdj}|jdj}|jdr| j!d|d|dk| j!dt"|d|d| r| r| ddj%| zz } t jdd| xsd}|r(t jdd|jddn|}|| xsd| |dcSy#||krtjdYYyxYw)NrrgFrriz exception.cssrMz(.*?)z3]*name="ArticleTitle"[^>]*content="([^"]+)"z]*>.*?)flagsz]*>.*?u发表于\s*(\d{4}-\d{2}-\d{2})zI]*href="([^"]+\.(?:pdf|doc|docx|xls|xlsx|zip|rar))"[^>]*>([^<]+)httpz zzrCrJrGrHrrrrrrrr __name__r6r(r&rs04333"499~1diil6J6J6L#diil RV  #g, ./! 77??277??84 5 '',,x!6 7  $ 5o(37    #!!!!!  1*$(&1 -(%XPY x    F4?D:#L.:b zFr(