ujndZddlZddlZddlZddlZddlZddlZddlZddl Zddl Z ddl Z ddl m Z mZddlmZmZej$j'ddZdZdZd Zed Zd Zd d iZdZdZdZe j<edz j?Z dZ!dZ"dZ#dZ$dZ%dZ&e'dk(re&yy)u 韶关市南雄市水务局政府信息公开平台 - 爬虫 CMS: Laravel + Vue SPA (省级政府信息公开平台) 列表: /sgnxshuiwj/gkmlpt/api/all/{catId}?page=N&sid={siteId} (JSON, 100条/页) 详情: requests +
WAF: 无 运行: python3 crawl_nanxiong.py [--full] --full: 全量爬取 (所有3页，含日期过滤) 不加: 仅爬第1页 (增量模式，日跑用) N)datetime timedelta)ThreadPoolExecutor as_completed SEARCH_DBz/root/search.dbu.韶关市南雄市水务局-政府信息公开u韶关市南雄市水务局zhttps://www.gdnx.gov.cnz/sgnxshuiwj/gkmlpt/api/all/1653751222z User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36)g333333?g?iG)daysc2tdtd|dy)N[z] T)flush)print SITE_NAME)msgs #/root/gov_crawler/crawl_nanxiong.pylogr$s Ai[3% -ctd|dt}tjj |t } tjj |d}tj|jj}|jdgS#t$r}td|d|Yd}~yd}~wwxYw) Nz?page=z&sid=headerstimeoutarticlesu 列表页 u 请求失败: )API_URLSITE_IDurllibrequestRequestHEADERSurlopenjsonloadsreaddecodeget Exceptionr)page_nourlreqrespdataes r fetch_listr/(s IVG9E' 3C .. g 6C~~%%c2%6zz$))+,,./xx B''  j  45sA#B C&B<<Cc bg}|D]}|jdxs|jdxsd}t|tr t|}|dkDr|}n|}|r |r |tkr_d}|r% t j |jd}|jd d}|jd r|nt|z}tj|jd dj}|j||||jd tf|S#t|dd}YxYw) u'解析API返回的articles，过滤3年datefirst_publish_timergmBz%Y-%m-%dN r*httptitle publisher)r' isinstancestrintTHREE_YEARS_AGOr fromtimestampstrftime startswithBASE_URLhtmlmodunescapestripappendSOURCE) ris_fullitemsartpub_datedate_strurl_pathfull_urlr6s rparse_articlesrL4s% EP776?Hcgg.B&CHq h $8}H d?HH xH$>   .#11(;DDZP775"%'226:88@S  "!5!;!;!=> hxf1MNO3P4 L .x="-s )$DD.cjtjj|t} tjj |d}|j j }d}tjd|tj} | r | jdj}nFtjd|tj} | r| jdj}|} tjd|} | r[tj| jdj} tjdd | j} | r| } |r%tjd d |j}|| |||fS#t$r}|||d|fcYd}~Sd}~wwxYw) Nrrrz9
]*>(.*?)
\s*]*>(.*?)
z(.*?)u#_全椒县人民政府|_南雄市.*r3z\s+ )rrr r!r"r%r&r(researchDOTALLgrouprBr@rAsub) r*r6r1sourcer+r, html_textr.contentm final_titlets r fetch_detailr[Tso .. g 6C0~~%%c2%6IIK&&( G NPY[][d[deA''!*""$ IIKYXZXaXa b ggaj&&(GK )95A   QWWQZ--/ 0 FF92q A G G I K&&g.446 dGV 447 0UD$//0s?F F2!F-'F2-F2c tjt}|j}d}d}|D]C\}}}}} |j dt |||| |xsdf|j dkDr|dz }n|dz }E|j|j||fS#t$r} td|ddd| Yd} ~ d} ~ wwxYw)NrzINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, source_url, content) VALUES (?, ?, ?, ?, ?, ?)r3rNu入库失败: 2z... ) sqlite3connectDB_PATHcursorexecuterrowcountr(rcommitclose) rconnrainsertedskippedr*r6r1rWrUr.s r save_to_dbriws ??7 #D [[]FHG-5 4)UD'6 4 NN0E3fgmD   "A 1  4 KKMJJL W   4 .Sb $qc2 3 3 4s:B C#B<<Ccdtjv}td|rdnddg}|rtdtdzndg}|D]}t j tjtt|}|td|dI|std|d nFt||}td|d t|d t|d |j|td t|d|s tdytdtdg}tt5}t!|D cic]$\}\} } } } |j#t$| | | | |&} } } } }} d}t| }t'| D]F}|dz } |j)|j+|dzdk(s||k(s6td|d|H dddt/|\}}td|d|ycc} } } } }w#t,$r(| |}||\} } } } |j)| | | d| fYwxYw#1swYfxYw)Nz--fullu开始u全量u增量u 爬取...rNu第 u 页跳过u 页为空，结束u 页: u 条, 3年过滤后 u 条u 共获取 u 条列表数据u没有数据，退出u开始多线程爬取详情 (u 线程)...) max_workersru详情进度: /u入库完成: 新增 u , 跳过 )sysargvrrange MAX_PAGEStimesleeprandomuniform REQUEST_DELAYr/rLlenextend NUM_THREADSr enumeratesubmitr[rrCresultr(ri)rE all_itemspages_to_crawlr)rrFresultsexecutorir*rZdsfuturesdonetotalfidxrgrhs rmainrs\#(("G&W(39=>I07U1i!m,aSN!   6>>=12g&   $wiz* +  $wi23 4 x1 d7)6#h-0DSZLPT UV  *S^$$456  #$ ' }J?@G  4 5[den[oppFWa#qRSUV8??<aA>AppG g& 5A AID 5qxxz* byA~nTF!E734 5 5 #7+Hg z7)<=!q 5aj(~ Q1Q434 5 5 5sH/H7)G:*&H7H0 H7>H7: H7.H41H73H44H77I__main__)(__doc__rnrPrrrtr^osurllib.requestr urllib.errorr#htmlr@rrconcurrent.futuresrrenvironr'r`rrDr?rrr!rqryrvnow timestampr;rr/rLr[rir__name__rrrs NMMMM(? **..&7 8 < & $ J5 6 D     8<<>I5$99DDF. @ 5F4.>b zFr