j:dZddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddl Z e jdZdZdZej"e d z j%d Zd Zd Zd ZdZedzZdZddZdZdZdZedk(reyy)uGcrawl_nc_tzgg.py - 南昌市人民政府-公示公告 (www.nc.gov.cn) 站点: 静态HTML, 无WAF 列表: /ncszf/tzgg/2021_nav_list.shtml (createPageHTML 分页, 共67页, 1000/page?) 分页: /ncszf/tzgg/2021_nav_list_{N}.shtml 详情: /ncszf/tzgg/{YYYYMM}/{hash}.shtml 运行: python3 crawl_nc_tzgg.py [--pages=N] [--dryrun] N)datetime timedelta) BeautifulSoupu南昌市-公示公告zhttps://www.nc.gov.cnim)daysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z/tmp/nc_tzgg.jsonlz/root/search.dbzcrawl_nc_tzgg.pyz'/ncszf/tzgg/2021_nav_list{suffix}.shtmlctjddD]2}|jdst|j ddcSt S)Nz--pages==)sysargv startswithintsplitMAX_PAGES_DEFAULT)as "/root/gov_crawler/crawl_nc_tzgg.pyparse_pages_argrsF XXab\( << #qwws|A' '( ct|D]6} tj|dtidd}d|_|j cSy#t $rtjdYZwxYw)Nz User-AgentF)headerstimeoutverifyutf-8) rangerequestsgetUAencodingtext Exceptiontimesleep)urlretriesirs rfetchr*#se 7^  S<*href="(/ncszf/tzgg/\d{6}/[0-9a-f]+\.shtml)"[^>]*>([^<]{8,100})r)titler&) LIST_TPLformatr*refinditerappendgroupstripBASE)pager&htmlitemsms r list_itemsr;.s qyooRo(ooqZo0 :D  E [[Z\` aN qwwqz//1$:KLMN Lrc<t|}|syd}tjd|tj}|r4tjdd|j dj }d}dD]-}tj||}|s|j d}nd}t|d}|jdtjd tj xs5|jdtjd tj }|rr|jtjd tj D]} | jg} |jgddD]} | jdvr| jdr#| jdk(r| jt!| M| jdk(r=| jgddrr| j#dd} | r| j| | j#dd} | stj$d| r| j| g} | D]!} | s| r | d| k7s| j| #dj'| }g}|jdtjdtjD]}|j)dd}|j#dxs|j+dd}|j-dr t.|z}n|j-ds t.dz|z}|j||d|D]}d |d!d"|d#d$}||vs|d%|zz } t1|j d&kr d |d"|d$}|||fS)'N)rrrru8([^<]*?)(?:\s*-\s*南昌市人民政府)?z<[^>]+>r )z!(\d{4}-\d{2}-\d{2})\s*\d{2}:\d{2}u(发布时间[：:]\s*(\d{4}-\d{2}-\d{2})z html.parserdivzzoom|content|article)idzzoom|content|article|wzcon|text)class_zshare|qrcode|print|wza)ppretabler=T) recursive)r@rArB)r@rArBF)r5uJ^(责任编辑|初审|复审|终审|\[纠错\]|浏览次数|字号|分享)z rz'\.(docx?|pdf|xlsx?|rar|zip|wps|et|dps)$)hrefrE/http)namer&z

rHz

 r)r*r1searchSsubr4r5rfindcompileIfind_all decomposerH find_parentr3strget_textmatchjoinrrr r6len)r&r8r.r: date_textpatcontentsoupzoomelpartstxtdedupr@attsrrErHlinks r parse_detailrc=sl :D  E MtUWUYUYZAz2qwwqz288:Ib IIc4   I   G } -D 99Urzz*A244H9 I [TYYW\egeoeoqSUWUYUYfZYN[D --rzz2KRTT'R-S B LLN -- <-M "Bww,&2>>'+Bww'! SW%ww%770E7Bkk"Dk1LL%++b+-C288$qsvw S! "  A%59> Q ++e$ D ]]3RZZ0Z\^\`\`%a] b1uuVR zzz%<C)< ??3 $;D(#:$D T$/01#ahZ'9!F)HM w  td{ "G# 7==?b  %7whG )W $$rct}dtjv}td|dtdg}t d|dzD]}t |}|std|ddnztd|d t|d d|D]V} t|d \}}}|r |tkr"|j|d |xs|d ||d tjdXtdt|d d|r.|ddD]%} ttj| ddd'yt!t"dd5} |D]+} | j%tj| ddz- dddt&j(j+t,rt/j0t,d} | j3d| j5} d}|D]} | j3d| d f| j7r*| j3d| d t8d | d | d!| d"d#d#t;j<d$| d!rt?| d!jAd%d#ndtBf | jDs|dz }| jG| jItd&|dyy#t$r'} tdt| dddYd} ~ d} ~ wwxYw#1swYxxYw)'Nz--dryrunz[nc_tzgg] pages=z cutoff=T)flushr u 第u页 0 条, 停止u页 u 条r&r.)r&r.daterZu ✗ <g?u [完成] F) ensure_asciiwr)r!rIx)rzPRAGMA busy_timeout=120000rz'SELECT id FROM gov_raw WHERE page_url=?zINSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank, script_name) VALUES (?,?,?,?,?,?,?,?,?,?)u江西rfrZrz\d{4}-\d{2}-\d{2}-u[入库] 新增 )%rr r printCUTOFFrr;rWrcr3r#rSr$r%jsondumpsopenOUTwriteospathexistsDB_PATHsqlite3connectexecutecursorfetchone SITE_NAMEr1rUrreplace SCRIPT_NAMErowcountcommitclose) max_pagesdryrunresultspnr9itr.rXrZer)fconncaddeds rmainrys!I 388 #F YKxx 8EGAy1}%2 E"./t <  bTc%j\.d; B <,8E,C)y'V!3r%y5;OBwKYbovwx JJsO   Ic'l^4 (5 ! ;A $**QU3DS9 : ; c3 )>Q >A GGDJJqu5< = >> ww~~gw4 12 KKM A II?!E( Mzz| II{7Y!E(AfIq|UWY[46HH=QSTU[S\4]QvY&&sB/0cdfqs tzz      (5# <Q .d;; <>>s* J3) J3>1K&3 K#<KK#&K0__main__)r)__doc__r rur1rpr$rryrrbs4rurllib3disable_warningsr~r6rnowstrftimeror rsrxrr/rr*r;rcr__name__rrrs2111( $  (,,.9#. . 8 8 Dv  ; ; 9%x-6` zFr