ujP!dZddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl Z dZ dZ e dzZdZej dd Zd d e d zd dZdZdZddZddZdZdZdZedk(reyy)u 新干县人民政府 — 通知公告/公示公告 Custom CMS, AJAX API (JSON) http://www.xingan.gov.cn/news-list-tzgga.html (catid=639) http://www.xingan.gov.cn/news-list-gongshigonggao.html (catid=47) Usage: python3 crawl_xingan.py [incremental_days] Examples: python3 crawl_xingan.py 639 # 通知公告 全量 python3 crawl_xingan.py 639 90 # 通知公告 增量(最近90天) N)datetime timedeltau新干县人民政府zhttp://www.xingan.gov.cnz/api-ajax_list-{}.htmlg? SEARCH_DBz/root/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36XMLHttpRequestz/news-list-tzgga.htmlz0application/x-www-form-urlencoded; charset=UTF-8)z User-AgentzX-Requested-WithRefererz Content-TypecJdddd}|jt|d|S)Nu 公示公告u 通知公告u 民意征集)47639159cat)getstr)catidnamess !/root/gov_crawler/crawl_xingan.py get_cat_namer"s)!. PE 99SZ3ug //c 4dt|dddddgddg d d S) N6_news6newszY-m-d4020)z is_top DESCzdisplayorder DESCzinputtime DESC1)z ajax_type[]is_ds)r)rs rbuild_post_datar's2 c%j#v T4 B    rc tjtj|t||}|j |j }|S#t$r}td|d|Yd}~yd}~wwxYw)N)headersdatatimeoutz API error (page z): ) requestspostAPI_URLformatHEADERSraise_for_statusjson Exceptionprint)page_no post_datar!rr es rapi_getr/3sl MM'..17(' ; vvx  "7)3qc23sAA A8A33A8cx tj|t|}|jd|_|j }d}tjd|}|r|jd}n>tjd|}|r&tjdd|jd}|jd }|d krdd d S|jd |dz}d} |} | t|krb| d kDr]|jd| } |jd| } | d krn3| d k\r| | kr | dz } | dz} n | dz} | dz} | t|kr| d kDr]||| dz j} | rt| dkrddd Stjdd| } | |dS#t $r}ddt|dcYd}~Sd}~wwxYw)N)rr!zutf-8r)contenttitleerrorz2ArticleTitle[\"\x27 ]*content=[\"\x27]([^\"\x27]+)z([^<]+)u ^新干县人民政府[-_—–]z
zz empty contentz(href|src)=([\'"])(/)(?![/])c|jddz|jdztz|jdzS)Nr4=)groupBASE_URL)ms rzfetch_detail..ks2!''!*s"QWWQZ/(:QWWQZGr)r1 full_title)r"r r&r'encodingtextr)rresearchr=subfindlenstrip)item_urlr!r-htmlr.rAr? zoom_start content_startdepthpos next_open next_closer1s r fetch_detailrR?s= LL7G D  vvJ GNAWWQZ II/ 6  CRQRTJ45JA~ 66IIc:.2M E C D /eaiIIfc* YYx- >  >i*4 QJEa-C QJEq.C D /eai=q)//1G c'lR'88ff'GG j 99U =SV<<=s?F F9 F4.F94F9c |s tdytjt}|j }d}|D]o} t j dd|dj}|jd|d|dtd ||dd |d|d f|jdkDr|d z }q|j|j|S#t$r}td |Yd}~d}~wwxYw)Nu 无数据插入rz<[^>]+>rr1zINSERT OR IGNORE INTO gov_raw (title, page_url, site_name, summary, content, publish_date) VALUES (?, ?, ?, ?, ?, ?)r2url-idater4z DB insert error: )r*sqlite3connectDB_PATHcursorrDrFrIexecute SITE_NAMErowcountr)commitclose)itemscat_nameconncinserteditem clean_textr.s r insert_dbrgrs  !" ??7 #D AH - - BY@FFHJ II0gU {!H:-FDS!4 ?DLB  zzA~A  - KKMJJL O  - 's+ , , -sA,C C1C,,C1c tjt}|j}|j d|j |j tdy#t$r}td|Yd}~yd}~wwxYw)Nz4INSERT INTO gov_search(gov_search) VALUES('rebuild')u FTS 重建完成u FTS 重建失败: ) rWrXrYrZr[r^r_r*r))rbrcr.s r rebuild_ftsrisk*w' KKM HI   "# * $QC())*sA%A(( B 1BB c  ttjdkr tdtjdtjd}t |}ttjdk\}|rt tjdnd}tdtd|d|d |rtd |d n td t|}tjttd|}|r|jd s tdy|d }|jdg}td|dt|d|dzdz}td|td|dzD]} tjtt| |}|r&|jdr|j|dnWtd| dtjdt| |}|r%|jdr|j|d| dzdk(std| d|dt|dtdt|dt!j"} g} |D]} | jdd} | s t!j$t'| dd d!}| |z j*}||ksP| j-| jd"d| jd#d| dd | jd$d%td&t| d| rt/j0t2}|j5}t7}| D]?} |j9d'| d#f|j;}|s,|j=| d#A|j?| Dcgc] }|d#vs |}}td(t|d)t| t|z d*|s td+yg}tA|D]\}} tC| d#}|jd,r6|dzdzdk(r*td-|dzdt|d.t|d/[|j-|jd0xs| d"| d#| d1|d2d3|dzdzdk(std-|dzdt|d.t|d/td4t|dt|tE||}td5|d6tGtd7|d8|d9y#t($rYwxYwcc}w):Nr;z9Usage: python3 crawl_xingan.py [incremental_days]r4r<i'u=== 开始爬取 z - z (catid=z) ===u增量模式: 最近 u 天u 全量模式totalu ❌ 无法获取数据r u 总记录: u , 第1页: u 条r8u 总页数: u ⚠ 第u页空, 等待后重试...r6ru 已扫描 /u 页, u 列表扫描完成: inputtimer z%Y-%m-%dr2rTid)r2rTrVrpu 日期过滤后: z-SELECT page_url FROM gov_raw WHERE page_url=?u 需抓详情: u 条 (已有 u 条)u 无新增数据r3u 详情进度: u (成功: )rArVr1)r2rTrVr1u 详情完成: 成功 u 入库: u 新u === 完成: u , 新增 u 条 ===)$rHsysargvr*exitrintr\rtimesleep PAGE_DELAYr/r rangeextendrnowstrptimer ValueErrordaysappendrWrXrYrZsetr[fetchoneaddr_ enumeraterRrgri)rrais_incremental cutoff_daysr,r rkr` total_pagesr+r{filteredredate_str item_date days_diffrbrc existing_urlsrowitto_fetchsuccessidetailrds rmainrs 388}q IJ  HHQKEE"H]a'N&4#chhqk"$K i[H:XeWE JK %k]$78 n&I JJz 1i D txx( () ME HHVR E M% CJg;Li(   ER<1  $QqSE3x=/CL>QRS T U #CL>3x=/ BC(+H Jxj %&M L )H:X >?s   2Is "T32 U?U3 U?U__main__))__doc__r(rDosrvrrr"rWrr urllib.parseurllibr\r>r$rxgetenvrYr&rrr/rRrgrir__name__rrrs ( # % - -  "))K!2 3D(11F  0  0:f4 *t@n zFr