ujdZddlZddlZddlZddlZddlZddlZddlZddlmZm Z dZ e dzZ dZ dZ dZd Zd d d d ZdZej$Zej(j+eej,ddZej0e dz j3dZdZdZdZd!dZd"dZe dk(rdZ!e"ejFdkDrFejFdjIdr(e%ejFdjMddZ!n<e"ejFdkDr'ejFddk(re%ejFdZ!ee! yy)#u濮阳市生态环境局 - 环评受理公示 (sthjj.puyang.gov.cn) TRS CMS, static pagination: /channel/list/15513_{page}.html 36 pages × 13 items = 465 records Detail: /content/YYYY/NNNN.html Title: h3.title, Date: span.fbsj, Content: div.detail-info N)datetime timedeltazhttps://sthjj.puyang.gov.cnz/channel/list/15513.htmlu濮阳市生态环境局u河南u 环评公示zcrawl_puyang_hj.pyzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z User-AgentAcceptzAccept-Language SEARCH_DBz/root/search.dbiG)daysz%Y-%m-%dc`tj|td}d|_|jS)NF)timeoutverifyutf-8)sessiongetTIMEOUTencodingtext)urlresps $/root/gov_crawler/crawl_puyang_hj.pyfetchrs' ;;sGE; :DDM 99cg}d}tj||tjD]}|jd|jdj |jdj }}}|j ds&|j dr t |zn t dz|z}|j|||f|S)u9Parse list page — div.list-wrap > ul.list-item > li > az
  • \s*]*href="([^"]*)"[^>]*>\s*]*class="title[^"]*"[^>]*>([^<]*)\s*]*class="date[^"]*"[^>]*>([^<]*)http/)refinditerDOTALLgroupstrip startswithBASE_URLappend)htmlitemspatternmhreftitledates r parse_listr,$s EYG [[$ 2*GGAJ (8(8(:AGGAJ]*class="title"[^>]*>(.*?)z<[^>]+>rz(.*?)u-濮阳市生态环境局uN]*class="fbsj"[^>]*>发表时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})r-z6]*class="detail-info"[^>]*>(.*?)\s*z]*>.*?)flagsz]*>.*?dz8]*(?:id|class)=["\']zoom["\'](.*?)\s*)rsearchrsubr r!replacelen)r%rr*r(date_strcontents r parse_detailr8/s E :D"))LAz2qwwqz288:  II-tRYY ? GGAJ&&'BBGMMOEH ceijA771:%%c3/G KTSUS\S\]A''!*&&5r7"))T&&3R R--/ c'lS( IIQSWY[YbYb c ggajGff92wbiiXGff7WBIIVGmmoG (G ##rc  ddl}|D]\}}}}}}} } d|d|d|jtdtddzd|d|d|jtdtddzd|d| d| d} |jdtg| j d d d } | j dk7std | jjddy#t$r} td| Yd} ~ yd} ~ wwxYw)zPush items to search.dbrNzINSERT OR IGNORE INTO gov_raw (source_url, page_url, title, publish_date, site_name, content, group_name, industry, script_name) VALUES ('z', ''rz');sqlite3r T )inputcapture_outputr z[push] SQL error: r1z[push] Error: ) subprocessr4chrrunrencode returncodeprintstderrdecode Exception)r&labelr?r*rr+siter7r indscriptsqlpes rpush_to_searchdbrOUsS $CH F ?E3dGUC d3%tEMM#b'3r719=>d4&TFRVW^WfWfgjkmgnpstvpwxypyWzV{{AF@GGKLOKPPTU[T\\_bC 95SZZ=PaeoqrA||q *188??+>Dc tjt}|j}d}d}t d|dzD]}|dk(rt n t d|dz}tdtd|d|d|t|}t|}|stdtd |d ntdtd t|d |D]\} } } |jd | fj} | r|dz }0 t| } t| | \}}}|s| }|s| }|rt|j!dkrd} |jd| | ||t"|t$t&f|j)|dz }tdtd|d|ddd|d t-j.dt-j.d|j1tdtd|d||S#t$r(}tdtd| d||dz }Yd}~Td}~wwxYw#t$r4}tdtd||j+|dz }Yd}~d}~wwxYw)Nrrz/channel/list/15513_z.html[z] Page rz: z] No items on page z , stoppingz] Found z itemsz)SELECT id FROM gov_raw WHERE page_url = ?z] Fetch detail failed: z - 2u 正文为空z INSERT INTO gov_raw (source_url, page_url, title, publish_date, site_name, content, group_name, industry) VALUES (?, ?, ?, ?, ?, ?, ?, ?) z] +(z ()z] Insert error: g?z [z ] Done. New: z , Skipped: )r;connectrcursorrangeLIST_URLr#rD SCRIPT_NAMErr,r5executefetchonerGr8r! SITE_NAMEGROUPINDUSTRYcommitrollbacktimesleepclose)pagesconnrW all_count skip_countpagerr%r&r*item_urlr6existing detail_htmlrN detail_title detail_dater7s rcrawlrobs ??9 %D [[]FIJa#2!)h6J4&PU4V)V +gdV1UG2cU;<Sz4  Ak]"5dV:F G  +hs5zl&9:).$  %E8X~~;h[hj a  #Ho 2>k81T .L+w$ & c'--/2R7(  , YPWY^`hik Q +c)B|CR7H6IK=XYZ[ JJsOI$ L 3e2h JJL C }M)K | LM A +&=hZs1#NOa  * +&6qc:; a  s17 G.;AH". H7HH" I+*II__main__rz--pages==rz--pages)re) puyang_hj)rP)'__doc__requestsrsysosr;rbjsonrrr#rYr]r^r_rZHEADERSrSessionr headersupdategetenvrnowstrftimeCUTOFFrr,r8rOro__name__rer5argvr"intsplitrrrs_2111( ( 0 0 &  " DO0   (   w BIIk#4 5 (,,.9%0 0 : :: F $$L $=~ z E 388}qSXXa[33J?CHHQK%%c*1-. SXX sxx{i7CHHQK   r