ގj dZddlZddlZddlZddlZddlZddlZddlmZmZddl m Z ddlZ e e jdkDr2e jdjree jdndZeedeezdZdZed zZed zZd Zd Zej2d dZdZddedZddddddddZd"dZdZd#dZ dZ!dZ"d Z#e$d!k(re#yy)$u 泰州医药高新区（高港区） — 通知公告 Hanweb CMS AJAX (unitbuild API) https://www.cmc.gov.cn/xwzx/tzgg/index.html N)datetime timedelta)urljoinz[AutoPg] max_pages=u0泰州医药高新区（高港区）人民政府zhttps://www.cmc.gov.cnz/xwzx/tzgg/index.htmlz7/api-gateway/jpaas-publish-server/front/page/build/unitz 2023-06-18z!www.cmc.gov.cn:443:218.90.228.239 SEARCH_DBz/root/search.dbi&5zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z!application/json, text/plain, */*)z User-AgentAcceptReferer bulidstatic 5f26120115824722ba2229d0ceaa65b2 158d93f47443484c9ef19d54c1ab9ab2columnu 信息列表nulldE67Ga7Sg0xeYjn7E3pZ5) parseTypewebIdtplSetIdpageTypetagIdeditTypepageIdc tt}tj||dd|d< t j t |t|}|j|j}|j dr|ddStd |d |j d d y #t$r}td|d |Yd }~y d }~wwxYw)z(Call the Hanweb API for a specific page.)pageNopageSizeF) ensure_ascii paramJson)paramsheaderstimeoutsuccessdatahtmlz API error (page z): messageunknownNz Request failed (page ) dict BASE_PARAMSjsondumpsrequestsgetAPI_URLHEADERSraise_for_statusprint Exception)page_no page_sizerrrr es /root/gov_crawler/crawl_cmc.pyapi_getr45s + F**Y%O^cdF;  LL' R vvx 88I <' ' &wis488Iy3Q2RS T  'yA378sAB*!B** C3C  Cc g}tjdtj}|j|D]}|j d}tj d|}tj d|}tj d|}|sY|s\|s_|j |j dtt|j d|j dd|S)z4Parse list HTML to extract items (title, url, date).z(]*class="clearfix"[^>]*>(.*?)rztitle="([^"]*)"zhref="([^"]+)"z\[(\d{4}-\d{2}-\d{2})\])titleurldate) recompileDOTALLfinditergroupsearchappendrBASE_URL)r!items li_patternlili_htmltitle_mhref_mdate_ms r3 parse_listrHGs EGSJ!!$' ((1+)).8,g65w? v& LL q)xa9 Q   Lc tj|t|}|jd|_|j }tjd|}|r|jdjnd}tjd|}|r|jdnd}d} tjd |tj} | r | jdj} nFtjd |tj} | r| jdj} | s||dd dS||| d S#t $r}dddt|dcYd}~Sd}~wwxYw) z3Fetch detail page and extract title, date, content.)rrzutf-8)r6r8contenterrorNz([^<]*)rz1]*>(.*?)\s*\s*z!]*>(.*?) empty content)r6r8rL) r(r)r+r,encodingtextr.strr9r>r=stripr;) r7rr1r!r2rEr6rGr8rLzoom_ms r3 fetch_detailrTZs:I LLgw ?  vv ii148G(/GMM!  " " $REYYKT RF$6<<?"DG YYMtUWU^U^ _F ,,q/'')?ryyQ ll1o++-G oVVDW ==3 IRBQHHIs?E E& E!E&!E&c X|s tdytjt}|j }d}|D]} t j dd|dj} tj|dd}t|jd}|jd |d |d t|d d |d|df|jdkDr|dz }|j!|j#|S#d}YixYw#t$r}td|Yd }~d }~wwxYw)zInsert items into search.db.u 无数据插入rz<[^>]+>rKrLr8z%Y-%m-%dz%Y%m%dzINSERT OR IGNORE INTO gov_raw (title, page_url, site_name, summary, content, publish_date) VALUES (?, ?, ?, ?, ?, ?)r6r7Nirz DB insert error: )r-sqlite3connectDB_PATHcursorr9subrRrstrptimeintstrftimeexecute SITE_NAMErowcountr.commitclose) rAconncinserteditem clean_textdt date_rankr2s r3 insert_dbrj}s/  !" ??7 #D AH- - BY@FFHJ &&tF|Z@ H 56  II0gU Y 4C8Hi$v,0  zzA~A %-* KKMJJL O   - 's+ , , -s0(D)3C??D?DD D)D$$D)c tjt}|j}|j d|j |j tdy#t$r}td|Yd}~yd}~wwxYw)zRebuild FTS index.z4INSERT INTO gov_search(gov_search) VALUES('rebuild')u FTS 重建完成u FTS 重建失败: N) rVrWrXrYr^rarbr-r.)rcrdr2s r3 rebuild_ftsrlsk*w' KKM HI   "# * $QC())*sA%A(( B 1BB c tdtdtdttd}|s tdyt j d|}|rt |jdnd}d}||zdz |z}td |d |t|}|r|d d nd }tdttxs||dzD]g}t|}|st|}|r#|j||d d }|tkr |dzdk(sJtd|d|dt|ditdt|d|D cgc]} | d tk\s| } } tdt| dg} d} d} t| D]\}} t| d}|j!drBd|dvr| dz } n| dz } |dzdzdk(r$td|dzdt| d| d| d g| j#|dxs| d| d|d xs| d |dd |dzdzdk(std|dzdt| d!t| d"| d td#t| d$| d%| t%| }td&|dt'td't| d(|d)ycc} w)*Nu=== 开始爬取 u - 通知公告 ===u截止日期: ru! ❌ 无法获取第一页数据z count="(\d+)"ru 总数: u , 总页数: r8rK u 已扫描 /u 页, 累计 u 条u 列表扫描完成: u 3年内: r7rMrN2u 详情页进度: u (空正文: u , 错误: )r6rL)r6r7r8rLu (成功: u , 空正文: u 详情页完成: 成功 u, 空正文跳过 u , 错误 u 入库: u=== 完成: 共 u 条成功, 新增 u 条 ===)r-r_ CUTOFF_DATEr4r9r>r\r=rHrangemin_MAX_PGextendlen enumeraterTr)r?rjrl)r!count_m total_countr0 total_pages all_items last_dater/rArffilteredr skipped_empty skipped_erroridetailres r3mainrs< i[(; <= N;- () 1:D  12ii($/G+2#gmmA&'KI*Q.9?4 I)2 " f%IC 6; DqHI[w 4     U #b &)I;& R<1  L ;-}S^DTTXY Z[  "3y>"2$ 78"+Jd6lk.IJHJ KH d +,GMMX&t4d5k* ::g &/1" " A|q +AaC5#h- m_\fgtfuuvwx G_5W ;6N2d6li(   ER<1  '!uAc(m_Js7|nTaboappqr s#t& &s7|n4F}oU^_l^m no!H Jxj %&M S\N*=hZx PQGKs J?!J?__main__)rnrn)rn)%__doc__r&r9ossysr(rVrr urllib.parser_SYSrzargvisdigitr\rxr-rQr_r@ COLUMN_URLr*ruRESOLVEgetenvrXDATE_RANK_BASEr+r%r4rHrTrjrlr__name__rIr3rs$  ( "499~1diil6J6J6L#diil RV  #g, ./ ? # / / N N  . "))K!2 3D1  /2 % $& >FD *LR^ zFrI