yj)NdZddlZddlZddlZddlZddlZddlZddlZejdZ dZ dZ dZ dZ dZd Zd Zd d d ddZej$ddZej(Zej,j/ede_dZdZdZdZedk(reyy)u托克托县人民政府 — 部门文件 (服务器正式版) 栏目: http://www.tuoxian.gov.cn/zfxxgkzl/fdzdgknr_22198/bmwj/ CMS: TRS，静态分页 index_N.html (共 334 页) 详情: meta ArticleTitle + meta PubDate + div.TRS_UEDITOR ⚠️ 2026-08-10: 服务器 IP 被奇安信云 WAF 随机拦截(502, WZWS-RAY 头)， 本地 Mac 可稳定访问 → 当前走「本地抓取→JSONL→scp→import_tuoxian_bmwj.py 导入」降级闭环。 本脚本为 WAF 解锁后恢复用，配置注册 enabled=false。 Nzhttp://www.tuoxian.gov.cnz7http://www.tuoxian.gov.cn/zfxxgkzl/fdzdgknr_22198/bmwj/u%托克托县人民政府-部门文件u 内蒙古u 政府公告zcrawl_tuoxian_bmwj.pyz 2023-08-10zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zUtext/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8zzh-CN,zh;q=0.9zhttp://www.tuoxian.gov.cn/)z User-AgentAcceptzAccept-LanguageReferer SEARCH_DBz/root/search.dbFc vtdD]s} tj|dd}|jdk(r|jcSt d|dzd|jd tjd |d zzuy#t $r,}t d|dzdt|dd d Yd}~Nd}~wwxYw) NrT)timeoutallow_redirectsz retry : flush<) rangesessionget status_codetextprint Exceptionstrtimesleep)urlires '/root/gov_crawler/crawl_tuoxian_bmwj.pyfetchr!$s 1X ? CT BA}}#vv HQqSEAMM?34 @ 1q1u9  ? HQqSECF3BK=1 > > ?s2B B B8 "B33B8cfg}t}tjd|tjtjz}|D]h}tj d|}|s|j d|j dj}}tjdd|j}t|dks%tj d|tjs|jd r t|ddz}n8|jd r t|z}n|jd r|}n t|z}|jts||vr|j|d } tj d |} | r| j d} |j||| fk|S)Nz]*>.*?z&]*href="([^"]+)"[^>]*>([^<]*)r z\s+ z\.html?$z.//http(\d{4}-\d{2}-\d{2}))setrefindallSIsearchgroupstripsublen startswith LIST_BASEBASE_URLaddappend) htmlitemsseenrowsrowmhreftitlefulldate_strdms r parse_listrD1sa E 5D ::*D"$$+ >D. II? E ggaj!''!*"2"2"4evsE*002 u:>;bdd!C  ??4 tABx'D __S !d?D __V $Dt#Dx(  4<   YY-s 3 xx{H eT8,-3.4 LcDd}tjd|}|r|jdj}|sGtjd|}|r/|jdj ddj}d}tjd|}|r8tjd|jd}|r|jd}d}tjd|}|r|j }d}|}|t |krX|d kDrS|jd |r |dz }|d z }n"|jd |r |dz}|d z }n|dz }|t |kr|d kDrS|||}|rtjdd|tjtjz}tjdd|tj}tjdd|tj}tjdd |}|j}|||fS)Nr(z3]*name="ArticleTitle"[^>]*content="([^"]+)"r z([^<]+)u托克托县人民政府_z.]*name="PubDate"[^>]*content="([^"]+)"r)z,]*class="[^"]*TRS_UEDITOR[^"]*"[^>]*>rzz<(script|style)[^>]*>.*?)flagsz\s*\s*(?:\s*)+$) r+r/r0r1replaceendr3r4r2r-r.) r9r@r>rBrCcontentstartdepthrs r parse_detailrNRs E H$OA   "  II/ 6 GGAJ&&'BBGMMOEH CTJA YY-qwwqz : xx{HG A4HA #d)m vq) Q1- QQ#d)m uQ-&&92wbddUWUYUYkZ&&7WBDDQ&&LbRYacaeaef&&4hH--/ (G ##rEc tjt}|j}g}t dt dzD]}t |dkDrd|dndz}t|}|stdtd|dd nst|}|stdtd|d d nL|j|tdtd|d t|d d tjd tdtdt|d dx}}|D]\} } } |jd| fj!} | r|dz }0t| } | s|dz }Ct#| \}}}|s| }|s| }|rt|j%dkrd} |jd| | ||t&|t(t*tf |j-|dz }tdtd|d |ddd|d d tjd|jdt&f|j-|j3tdtd|d|y#t.$r}|j1|dz }Yd}~d}~wwxYw)Nr index_z.htmlr([z] Page z : fetch failTrz: 0 items (end)r z itemsg?z ] Total: rz)SELECT id FROM gov_raw WHERE page_url = ? u 正文为空zINSERT INTO gov_raw (source_url, page_url, title, publish_date, site_name, content, group_name, industry, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)z] +(z ()g?zINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id, r.title, r.site_name, substr(r.content,1,500) FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name = ?z [z ] Done. New: z , Skipped: )sqlite3connectrcursorr MAX_PAGESr5r!r SCRIPT_NAMErDextendr3rrexecutefetchonerNr1 SITE_NAMEGROUPINDUSTRYcommitrrollbackclose)connrW all_itemspgrr9r: new_count skip_countr@item_urlrBexisting detail_html real_title real_daterKrs r crawlrmzs ??9 %D [[]FIAy1}% aVB4u-R@Sz Ak]'"\:$ G 4  Ak]'"_=T J  +gbTCJdK 3  Ak])C N#36 :;I %.!x>> 7( (*   !OJ Ho  !OJ )5k)B& IwJ I#gmmo.3$G  NN98ZIwPUW_alm  KKM NI Ak]#i[:cr?2C2i[PQRZ^ _ 3=@ NN $   KKMJJL C }M)K | LM  MMO !OJ sA!I J!I;;J__main__)__doc__requestsr+sysosrrUurllib3disable_warningsr6r5r]r^r_rYrXCUTOFFHEADERSgetenvrSessionrheadersupdateverifyr!rDrNrm__name__rEr r~s,++ & E 3  % De'+   BIIk#4 5 (   w B%$P=N@ z GrE