uj!`dZddlZddlZddlZddlZddlZddlmZmZddlm Z ejjdejjejjeddlmZddlmZdZddlZeej,dkDr2ej,dj/reej,dndZeed eezd Zd Zd Zej>ed z jAdZ!dZ"ddZ#dZ$dZ%dZ&e'dk(re&yy)u 彭州市人民政府 - 环境影响评价审批 爬虫 CMS: EJS (AJAX pagination via ejspagination_go) URL: https://www.pengzhou.gov.cn/pzs/c143008/list.shtml WAF: JS Challenge (requires Playwright with stealth) N)datetime timedelta)urljoin) BeautifulSoup)sync_playwrightz/root/search.dbz[AutoPg] max_pages=cZddl}|jt}|jd|S)NrzPRAGMA journal_mode=WAL)sqlite3connectDB_PATHexecute)r conns &/root/gov_crawler/crawl_pengzhou_hp.pyget_connectionr s% ??7 #DLL*+ Ku%彭州市生态环境局-环评审批z2https://www.pengzhou.gov.cn/pzs/c143008/list.shtmliG)daysz%Y-%m-%dctj}|jjdddg}|j ddddd  }|j d |||fS) z+Create a Playwright browser with WAF bypassTz---disable-blink-features=AutomationControlledz --no-sandbox)headlessargszoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36zzh-CNii8)widthheight) user_agentlocaleviewportzz"crawl_list_pages..Ls!2W! 2W VWrclass_z [List] Page z: no list found, stoppingrliahrefT)stripz(\d{4}-\d{2}-\d{2})N)titleurldate_str: z items (total: )r<z[List] z& items outside 3-year window, stoppingza.pagination-num:has-text('z')z[List] No more pages, stoppingzist] Click error on page z[List] Error: z#[List] Total items within 3 years: )r#new_pagegotoBASE_URLtimesleeprangemin_MAX_PGprintcontentrfindfind_allgetr8get_textrematchgrouplenappendTHREE_YEARS_AGOlocatorcountfirstclick Exceptionclosestop) max_pages all_itemsr r!r"pagepage_numrJsoupr, items_on_pager5r6r7text date_matchr<r:ititems_within_3y items_outsidenext_btnes rcrawl_list_pagesri;s'I$&OAw <<>DB (}e D 1 aY0D9!E!IJ8 H )(37 8 JJsOllnG -8D4(WXB XJ.GHIMkk$' #GGCLf y({{{. XX&JZZ[\ ]-6[bJ?9Zr[O[ NS-AAMq  .TUV+ )# #||.I(UV,WY,Z[H~~'!+ ,,. 1  >@&e8 z   (M2j>_+LMIM /I/? @A =\"!7zA3GH $ qc"##$   NsyFK37K K 6K3A&K ,K3L<'L<K3 K0K+%K3+K00K33 L<L LLL"L9c|d} |j|ddtjd|j}t |d}|d}|j dd d i }|r$|j d r|d j}|d }|j dd di }|r'|j d r|d jdd}|j ddxs|j dd} d} | r t| } n!|j dd} | r t| } |j} d} | r"t | d}|jdddd} t|| || | | dS#t$r}td|d|Yd}~yd}~wwxYw)z(Fetch a detail page and extract content.r;r%r&r'rr+r:metaname ArticleTitle)attrsrJr<PubDateN divarticle__content)idr3r9c$|xr dt|vS)Nzwrapper-detail)strr0s rr2zcrawl_detail..s8X>NRUVWRX>Xr T) separatorr8i,) site_namer:page_url publish_date source_urlrJsummaryz[Detail] Error r=)rBrDrErJrrKrMr8rur;rN SITE_NAMErYrI)itemr^r;rJr`r:mtr<pd content_div body_htmlwrapperr{ summary_texttxt_souprhs r crawl_detailrs u+C2 #- ? 1 ,,.Wm4W  YYvfn%=Y > "&&#yM'')E # YYvfi%8Y 9 "&&#)}**,Sb1Hii*H#,,s$,GML#"$$ #    uBqc*+sE E(( F 1FF ct}|j}d}|D]R} |jd|d|d|d|d|d|d|jd d f|jdkDr|d z }T|j|j|S#t $r+}t d |jdd d|Yd}~d}~wwxYw)zSave records to gov_raw tablerz INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, source_url, content, summary) VALUES (?, ?, ?, ?, ?, ?, ?) rxr:ryrzr{rJr|r9rz[DB] Error inserting ?r=N) rcursorr rMrowcountrYrIcommitrZ)recordsrrinsertedrecrhs r save_to_dbrs  D [[]FHH H NNK #g,J^!4L!3y> 2&  "A H  KKMJJL O  H )#'''3*?)@1#F G G HsAB C !CC c tdtdttdtd}tdt||s tdytdt|d t \}}}|j }g}d }t |D]T\}}t||} | r<|j| |d zd zd k(s1td |d zdt|dP|d z }V|j|jtdt|d|dtdt|} td| dt|dtdy)Nu4=== 彭州市生态环境局-环评审批 爬虫 ===u3年截止日期: z! --- Step 1: Crawl list pages --- )r\zTotal items from list: zNo items found, exitingz --- Step 2: Crawl z detail pages ---rrrpz [Detail] /z donez Detail crawl complete: z success, z errorsz --- Step 3: Save to DB ---z Inserted z new records (total fetched: r>z === Done ===) rIrTrirRr#rA enumeraterrSrZr[r) itemsr r!r"r^rerrorsir~rrs rmainrse @B / 01 ./ q )E #CJ< 01  '( U ,= >?$&OAw <<>DG FU#44&  NN3 A|q  !A#aE |59: aKF MMOFFH %c'l^:fXW MN ()'"H IhZrs ( 277??277??8#<=>/ "499~1diil6J6J6L#diil RV  #g, ./ 4 ?8<<>I5$99CCJO  Nb6r6(V zFr