uj3<dZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z dZ dZ dZdZed zZd Zd Zd Zd ZdZdZdZdZej2Zde_ej8e_ddddZddZddZ dZ!dZ"da#dZ$dZ%dZ&e'dk(re&yy) u-垦利区人民政府 - 通知公告 (col39604) 爬虫 CMS: Hanweb 大汉版通 (JPage dataproxy 动态分页) 列表: /col/col39604/index.html 首屏内嵌 45条(3组×15条/页) 翻页 POST /module/web/jpage/dataproxy.jsp?startrecord=N&endrecord=M&perpage=15 totalRecord=290, perPage=15 -> 共20页 (290/15=19.3) 详情: /art/YYYY/M/D/art_39604_{id}.html meta ArticleTitle / pubdate + div#zoom (ContentStart~ContentEnd) 正文 附件: 正文内 (相对路径, urljoin 详情页URL) Nurljoin) BeautifulSoupu"垦利区人民政府-通知公告zcrawl_kenli_tzgg.pyu山东zhttp://www.kenli.gov.cnz/col/col39604/index.htmlz/module/web/jpage/dataproxy.jsp39604127350274u垦利区人民政府z/mnt/data/search.db-FzkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z User-AgentAcceptzAccept-Languagect|D]} tjj|t}tjj ||t }|j}t|dkr d|ddvry|jdd cSy#t$rN}||d z k(r'td |d |tj Yd}~ytjdYd}~d}~wwxYw)N)headerstimeoutcontexti,s404dutf-8replaceerrorsu [WARN] 请求失败 : file333333?)rangeurllibrequestRequestHEADERSurlopenctxreadlendecode Exceptionprintsysstderrtimesleep)urlrretriesireqrespbodyes %/root/gov_crawler/crawl_kenli_tzgg.pyhttp_getr4)s 7^  ..((g(>C>>))#w)LD99;D4y36T$3Z#7;;wy;9 9    GaK.se2aS9 K JJsOO  s$A0BB C0")C+C++C0c *t|D]} tjj|j d}tj j ||ittdd}tj j||t}|jjddcSy #t$rO}||dz k(r'td |d |tj Yd}~y t#j$d Yd}~d}~wwxYw)Nrz!application/x-www-form-urlencoded)Refererz Content-Type)datar rrrru [WARN] POST失败 rrrr)rrparse urlencodeencoderrr LIST_URLr!r"r#r%r&r'r(r)r*r+) r,r7rr-r.r1r/r0r2s r3 http_postr<:s 7^  <<))$/66w?D..((4BEBE&. CBE(FC>>))#w)LD99;%%gi%@ @    GaK,SEA37cjjI JJsOO  sB%B:: D)D 2D  Dc|jddjdd}|jddjdd}tjdd|}tjdd|}|jS)u:strip ·  实体前缀 和省略号截断后缀z·rz u​uu^[\s\xa0·\u00b7]+z \s*\.{3,}\s*$)rresubstrip)titles r3 clean_titlerBKsn MM*b ) 1 1(B ?E MM(B ' / /" =E FF("e 4E FF#R /E ;;=cg}tjd|tjD]}|jd}tjd|}|stjd|}|sE|jd}t |jd}t t|}tjd|}|r|jdnd} |sd|vs|j||| f|S) uM从 ...]]> 提取 (url, title, date)z&rz)]*href="([^"]+)"[^>]*title="([^"]*)"z)]*href='([^']+)'[^>]*title='([^']*)'z (\d{4}-\d{2}-\d{2})rart_) r>finditerSgroupsearchrBrBASE_URLappend) htmlitemsmcdataahrefrAr,dmdates r3parse_datastore_recordsrUTs E [[BD"$$ O -  IIBE J FNA wwqzAGGAJ'h% YY:E B rxx{b Vs] LL#ud+ , - LrCc |dkr@ttt}|sgSt|at}||dz tz|tzS|dz dzdz}|dz t zdz}|t zdz }dt tdztdttdd}tttzd|d|dtz|}|sgSt|}|dz dz}||tz|dztzS) u`第page页 (1-based) 列表条目。1-3页在首屏 datastore, 4+页 POST dataproxy 按组取。r/r)colwebidpathcolumnidsourceContentTypeunitidwebnamepermissiontypez ?startrecord=z &endrecord=z &perpage=) _init_recordsr4r;rUPER_PAGE GROUP_RECORDSWEBIDrKCOLUMNIDUNITIDWEBNAMEr< PROXY_URL) pagerMrecsrIstartendr7xmlidxs r3 fetch_pageroks qy  H%D 3D9MTAX)4(?;; AX!Oa E QY- '! +E - ! #C5(S.1a D Hy(]5'SEQZ[cZd+eegk lC  "3 'D !8q.C hqH 4 55rCc  "d}tjd|}|rt|jd}d}tjd|}|r|jd}t |d}|j dd}|s||dgfSd}tjd |} tjd |} | rD| rB| j | jkDr!|| j| j }|jr t |dn|} g} | jd d D]} | d}| jd }tjd|sHd|jvs6d|jvs$d|jvs|jdsxt|}|jds|jdrd|zn|}|r|n|jdd}| j| ||f| D]\} }}| j!d |d}||_| j$s| j$}d}t'|j(D] \}}|| us |}n|>t+|j(d|D]#}t-|dddk(s|j/%| j1|| Dchc]\}}}| c}}"| j2Dcgc]}t-|dds|}}g}|D]}|j4dk(r|jt7|.|j4d vr|jdd }|s|j dsc|jd}t9"fd!|jd d D}|s|s|j dr|jt7||j||jdd }|s|j|t;}g}|D]X}tj<d"dtj<d#d|}|s2||vs7|j?||j|Zd$jA|} | jCd%d&}!| D]\}}}|| vs ||!vs| d'|d(|d)z } tj<d*d| } tj<d+d$| j} ||| | fScc}}wcc}w),u/提取 (title, date, content_html, attachments)rz/zrQT)rRrR)r@z>(?i)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|et|ofd|png|jpg)(\?|$)downloadattachru附件)zhttp://zhttps://z//zhttp:rX_blank)rRtargetNnameimgtable) prqh1h2h3ulolli blockquotec3RK|]}t|dvxs|dv yw)rRNr).0xattached_hrefspage_urls r3 z!extract_detail..s: 6!F),>]!F)~B]] 6s$'\s+<[^>]+>z z&&z

z

u打印本页[^\n]*z\n{3,})"r>rJrBrIrfindrkrlr@find_allget_textlower startswithrsplitrLnew_tagstringparent enumeratecontentslistgetattr decompose replace_withchildrenrystranysetr?addjoinr)#rMrrAmtrTmpsouprr body_htmlm_startm_endfrag attachmentsrQrRtxtabs_urlrynew_ar|idx_acichild_elbody_elsparts inner_tables el_has_attachseen final_partskeycontentcontent_unescapedrs# ` @r3extract_detailrs E Et LB BHHQK( D KT RB xx{ } -D 99Uv9 &D dB"" Iii/6/A/A$/G'G+W37==#5b#9D   41 2 3(4 Swx @ 88 AE&qzz2  EA:E  !!**Ve"45*Eufd3u<)* u 4??-!Qg?N!]]Hrgb&$.GHHH E" 77g  LLR !  77T T++b+-Crwwu~;;w/L 6St4 66M} SW% LL ++b+-C S!'", 5DK "ffVR B!:; 3d? HHSM   q ! " kk+&G5'T4 ' !g5F&F )'2DTF(S SGTff*B8GffY0668G $ ,,W@Hs S7%S=8S=cd}tjdd}d}|t|kru||}|jdrt |j ddd}n,|dk(r'|dzt|krt ||dz}|dz }|dz }|t|krut jtd}|jd|jd |j}d}d}d}d} td|dzD]} t| } | s5| dz } td | d | d tj | dk\r| dkDrnFd} td| dt| d| D]l\} } } |jd| f|jr|dz }2 t'| }|s|dz }Ft)|| \}}}}|r1tt+j,dd|j/dkr|dz }|s| }|s|}t+j,dd|dd}t+j,dd|j/}d|vrdnd} |jd|||| | |t0t2t4|df }|j6}|jd||t0|f|j9|dz }td|d|ddt#j$d#o|jCtd$|d%|d&|d'||z|zy#t j $rt#j$dYwxYw#t j $r}d t;|vr|j=t#j$d! |jd|||| | |t0t2t4|df }|j6}|jd||t0|f|j9|dz }td|d|ddd"n)#t>$r|j=|dz }Yn wxYw|dz }Yd}~gd}~wt j@$r |dz }YwxYw)(Nrrz--pages==z--pages<)rzPRAGMA busy_timeout=60000zPRAGMA journal_mode=WALu [WARN] 第u页无数据 (fail_streak=)rrEu 第u 页: 找到 u 条z'SELECT id FROM gov_raw WHERE page_url=?rWrr r zr?r@ SITE_NAME SCRIPT_NAME GROUP_NAME lastrowidcommitrrollbackr&IntegrityErrorclose) max_pagesargsr.rQconnc total_new total_dup total_skip fail_streakrirNr,rArTdhtmld_titled_daterrsummary has_tablecurridr2s r3mainrsiI 88AB Q fXR ~67, JJsOo7 AF JJL JykI;nZLPZ[den[noy[yZz {|i++  1  8++ $s1v%MMOJJqM (iiI$gwS&)U`blnwyz{}"mm "x#&G"DF !Q fXR ~_MN$( "a (!OJ)) Q  sV-(LA8L4(L10L14Q2P#:A9O43P#4P P#P P##QQ__main__)rW)(__doc__r(r>r*rsslurllib.requestr urllib.parserbs4rrrrrKr;rhrerfrdrgrrbrccreate_default_contextr"check_hostname CERT_NONE verify_moder r4r<rBrUrarorr__name__rCr3rsA@@@  0 #  $ 0 0 -    !    c  "--@O0 ""( 68a-HZ}z zFrC