9Jj##~dZddlZddlZddlZddlZddlZddlZdZedzZdZ dZ ejjddZ d d d d Zd ZdZdZdZddZdZdZedk(r[eej0dkDrej0dndZedk(redeZeeyedeZeeyy)ui 盘锦辽滨沿海经济技术开发区 - 通知公告 爬虫 CMS: P8CMS 栏目: /13775/ - 通知公告 Nzhttps://ldwxq.panjin.gov.cnz/13775/u4盘锦辽滨沿海经济技术开发区-通知公告u 经开区DB_PATHz/root/search.dbz]*>(.*?)z WARN: Q25_ListContlist not foundzg]*href="([^"]*)"[^>]*title="([^"]*)"[^>]*>.*?(.*?)\s*(\d{4}-\d{2}-\d{2})<[^>]+>) researchDOTALLprintgroupfinditerstripsubappend) htmlitemsmcontentartitle title_spandate final_titles r parse_listr,s E =tRYYOA  01 ggajG [[DFMOQOXOXY /ggaj   "VVJAGGAJ7==? wwqz!)z +  k3-. / Lrc tjd|}|rt|jdStjd|}|rt|jdStjd|}|rt d|DSy)u获取总页数u \d+/(\d+)页rz pages:(\d+)zlist-(\d+)\.htmlc32K|]}t|ywN)int).0ps r z"get_total_pages..Bs)a3q6)s)rrr0rfindallmax)r#r%pagess rget_total_pagesr78st /4(A1771: .$'A1771: JJ*D 1E )5))) rc d}tjd|}|r|jdj}|sYtjd|}|rA|jdj}|j ddj}d}tjd|}|r|jdj}|sGtjd|}|r/|jdjj d d }d}g}tjd |tj }|r|jd}tjd |D]} | jd} tjd | tjs:tjdd| jdjxs| j d d} | jd r t| z} |j| | dtjd|tj D]} | jd} tjd| tj }g}|D]P}tjd|tj }djd|D}|j|R|j | ddj|zdz}tjdd|tj }tjdd|tj }tjdd|}tjdd|}tjdd |}tjd!d|}tjd"d|}tjdd|}|j d#d$j d%d&j d'd(j d)d*}tjd+d$|}tjd,d |j}|r|}|rt|jd-krad.|d/|d0}tjd1|tj}|r1|d2z }|d3d4D]$}|jd r t|z}|d5|zz }&||||fS)6u解析详情页rz,]*ArticleTitle[^>]*content="([^"]*)"rz(.*?)_rz']*PubDate[^>]*content="([^"]*)"u2发布日期[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})/-z$class="Q25_artcont"[^>]*>(.*?)z$]*href="([^"]*)"[^>]*>(.*?)z"\.(pdf|doc|docx|xls|xlsx|zip|rar)$rr)rnamez]*>(.*?)z]*>(.*?)z]*>(.*?)z | c3fK|])}tjdd|j+yw)rrN)rr!r )r1cs rr3zparse_detail..rs&&XQrvvj"a'@'F'F'H&Xs/1z|  z |z]*>.*?)flagsz]*>.*?z z]*>z

z z ]*>zz  z&&z<z[ \t]+z\n{3,}z

z

zhref="([^"]*\.pdf)"u 附件PDF链接: Nu"

PDF文档

)rrrr splitreplacerrIr! startswithBASE_URLr"r4joinlen)r#rr(r%tr*r& attachmentsrawr'hrefr=table table_htmlrows table_textrowcells cell_text pdf_linkspls r parse_detailr\Es E A4HA   "  II-t 4    "AGGCLO))+E D bddKvvj"aggaj9??AXTZZPS_UWEX??3'#d?D""4#>?  @[[!=sBIIN QEQJ::4j"))LDJ - #?biiP!JJ&XRW&XX !!), -++j%$))J2G*G%*OPC Qff12s"))Lff/S Jff\4-ff["c*ffWfc*ff]B,ffYc*ffZS)kk(C(00#>FFvsS[[\bdghffYS)ffY,224 G c'--/*R/25u=JJ5tRTTB  / /Gm F==%!BB@2EE F $ ,,rcg}tdtztt}t|}tdt |z|j |t |}td|z|r|dkDr t||}td|dzD]q}dt|fz}td||fz t|}t|}tdt |z|j |tjd st}g} |D].} | d} | |vs |j| | j| 0td t | z| S#t$r}td |zYd }~ud }~wwxYw) u 全量爬取zFetching page 1: %s -> %d itemszTotal pages: %drrrz%s/13775/list-%d.htmlzFetching page %d: %s333333?z ERROR: %sNz Total unique items: %d)rLIST_URLrr,rNextendr7minrangerLtimesleep Exceptionsetaddr") max_pages all_itemsr#r$ total_pagespagereseenuniqueitemitem_urls r crawl_allrrsrI ( *+ ?D t E /CJ &' U!$'K k )*Y]+y1 aq) %4(88 $c{23 :Dt$E /CJ. /   U # JJsO  5D F 7 4  HHX  MM$    $s6{ 23 M  -!# $  s1AE F'E;;Fctdtztt}t|}tdt |z|S)u增量爬取（仅第1页）z!Fetching page 1 (incremental): %sr^)rr`rr,rN)r#r$s rcrawl_incrementalrts9 - 89 ?D t E /CJ &' Lrcddl}ddl}|jtd}|j d|j }d}t |D]\}\}}} tdD]} t|} t| |\} } }}| xs|}| xs| }|r|ddn|}|rtj|d nd }|j d td ||||xsd |t|f |jdkDr6|d z }|dks |d zdzdk(r!td|d zt!||dd|fzn|j'd|j+|j-td|z|S#|j"$rE}dt%|vr| dkr|j'dYd}~:td|d|Yd}~d}~wt($r8}| dkr|j'dYd}~ytd|d|Yd}~d}~wwxYw)u保存到生产DB，含重试rNr)rzPRAGMA busy_timeout=30000rF) ensure_asciiz[]zINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category, status, attachments) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'published', ?)rrrGrFz [%d/%d] %s +%d(lockedrz DB ERROR [z]: z ERROR [r_z1 Inserted %d new records (trigger auto-syncs FTS))sqlite3rdconnectrexecutecursor enumeratercrr\jsondumps SITE_NAMEGROUProwcountrrNOperationalErrorstrrerfcommitclose)r$rztime_modconnr? new_countir(rqr*attempt detail_htmlrOdr&rPr+ final_datesummary attrs_jsonrms r save_to_dbrs ??7B? /DLL,- AI&/&6"" "E8TQx G #Ho -9+x-P*1g{j5 Y$ +2'$3- LWTZZ %H]a  GH *gmWeZY Z ::>NI A~!A#q0AaCU[QTRTEUW`3aab' B sE"H KKMJJL > JK %++ s1v%'A+NN1%x;< Q;NN1%Ha89  s1'B8E%%G84#F4F44 G8G3G33G8__main__rfull incrementalu0=== 盘锦辽滨-通知公告 - 增量爬取 ===u0=== 盘锦辽滨-通知公告 - 全量爬取 ===r/)__doc__r rrrdsysosrLr`rrenvironr rr rr,r7r\rrrtr__name__rNargvmoderr$rrrs  ( i  B  **..$5 6QO0   2  M-^'R1f zchh-!+388A;D } @A!#5 @A 5r