^jdZddlZddlZddlZddlZddlZddlZddlmZmZm Z ddl Z ddl Z ddlZ e e jdkDr2e jdjree jdndZeedeezej(j+dej(j-ej(j/eddlmZe j6e j8j:dZdZd d iZ d Z!d Z"ejFejHe d z jKdZ&ddgZ'ddZ(dZ)dZ*dZ+dZ,e-dk(re,yy)u 黎川县人民政府 - 两个栏目 1. 乡镇动态(col2434): https://www.jxlcx.gov.cn/col/col2434/index.html 2. 重点民生项目(col3880): https://www.jxlcx.gov.cn/col/col3880/index.html?number=LC0006LC0001 Hanweb, AJAX分页, 正文: ZJEG_RSS.content N)datetimetimezone timedeltaz[AutoPg] max_pages=)push_to_searchdb黎川乡镇动态zhttps://www.jxlcx.gov.cnz User-Agentz]*>.*?(\d{4}-\d{2}-\d{2})r)refinditerDOTALLgroupstripappendBASE_URL)htmlitemsmhreftitledate_strs r parse_listr1/s E [[UW[]_]f]fg9wwqz   "771: eX_h78 9 Lc"t|}|syi}tjd|}|r|jd|d<|j dsptjd|}|rX|jdj ddj ddj }|rt|d kDr||d<tjd |}|r|jd|d <tjd |tj}|rI|jdj }tjd d|}t|dkDr||d<|j dsttjd|tj}|rMt|jdj dkDr"|jdj |d<|j d|j d|j d fS)N)NNNz4]*name="ArticleTitle"[^>]*content="([^\"]+)"rr/z(.*?)|r-r z=]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2}) publish_datezA(.*?)z ]*>2contentz7(.*?)) r r$searchr'rsplitr(lenr&sub)rr+resultr-tr9s r fetch_detailr@9s :D  F I4PA''!*w ::g  II-t 4    %a(..s3A6<<>ASVaZ"#w RTXYA!"~ VX\^`^g^ghA''!*""$&&W5 w<"  'F9  ::i IIPRVXZXaXa b QWWQZ%%'(2- ! 0 0 2F9  ::g  9 5vzz.7Q QQr2c Rtddd|d|ddg}tdtttxstdzD]}t |ddd t |d d d d }td|d tdddt t|}|s tdnEt|}|s tdn+tdt|d|j|tdt|d tjtjddd} td| j!d|fj#D} | j%td t| d!gt} } d"}t)|D]\}\}}}|| vr| j+||r|t,kr|dz }2|| vr7td#t| dzd t|d$|dd%d&ddt/|\}}}|r|}|xs|}|r|t,kr|dz }td't1j2d(d|xsd)j5dd*}t1j2d+d|}| j7||||xsd)|||d,td-t9j:d.|rtd/|d| r t=| |td0|d1t| dt| S#t&$r#} td| t} Yd} ~ d} ~ wwxYw)2u5爬取指定栏目的文章（5页 + 近3年过滤） 2==================================================u 🏠 z (columnid=z) r18/66104u黎川县人民政府0)pageappidwebidpathcolumnidunitidwebnamepermissiontypeu 📄 第 u 页... T)endflushu❌ (请求失败)u0 条 (无更多)u✅ u 条u📊 列表总计: SEARCH_DBz/root/search.db<)rc3&K|] }|d yw)rN).0rs r zcrawl_column..nsZ!Zsz.SELECT page_url FROM gov_raw WHERE site_name=?u# [WARN] 读取已入库URL失败: Nu📚 已入库 u 条，重复跳过详情rz [z] 7z...u ⏭ (超3年)z<[^>]+>r7i,z\s+) site_namer/rr9pub_datesummarytagsu✅g333333?u⏭ 跳过超3年: u ✅ u: 完成! 共 )rrangemin MAX_PAGES_MAX_PGstrr PROXY_URLr1r<extendsqlite3connectosgetenvsetexecutefetchallcloser enumerateaddTHREE_YEARS_AGOr@r$r=r(r)timesleepr) column_id column_namesite_tag output_tagall_listrIrr+r,_cx _existingr all_itemsseenfilteredir/r list_datedtr9dater\r]s r crawl_columnrVs6 Bvhgk]+i[F8 LMHaY(<9=a?@D C#s Ng2cK  $q 73DIY% & ' 4  % &  SZL%&  H d 34oobii 5FGQSTZckk <{n'NNVhjZZ   OC N++D EF#%tIH&/&9" "E3 $;   _4 MH  )   C N1$%Qs8}oRcr |3GSX\](-GT E$9 ?2 MH / " &&S'-R8>>@#F&&g.$uS}"(  e  37: #H:T23J/ F;-~c)n-=T BC y>W  3A378E s9A,K:: L&L!!L&c tj}d}tD]\}}}}|t||||z }tddd|dtj|z dddy)NrrBrCu ✅ 全部完成! 共 u 条 ⏱ 总耗时: z.1fzs )rqCOLUMNSrr)t0totalcol_idcol_nametagout_tags rmainrst B E*1>&#w fhW==> Bvh.ug5J499;WY>Z]J^^abhai jkr2__main__)N).__doc__sysrhr$rqjsonrfrrrrurllib3_SYSr<argvisdigitintrbrrcrLinsertdirnameabspath__file__ crawler_librdisable_warnings exceptionsInsecureRequestWarning SITE_NAMEr*rrdranowutcstrftimerprr r1r@rr__name__rWr2rrsE(''22"499~1diil6J6J6L#diil RV  #g, ./277??277??8#<=>(++BBC % W X E 8<< - u0EEOOPZ[CN  R:FPl zFr2