jEJdZddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddl Z e jdZdZdZej"e d z j%d Zd Zd Zd ZdZgdZdZdZdZddZdZdZdZdZ e!dk(re yy)uacrawl_lixin_jcsd.py - 利辛县-基层政务公开专题 (www.lixin.gov.cn) 站点: 静态HTML, 无WAF 列表: /Jcsd/showList/{subId}/page_{N}.html (21个环保子栏目, 专题首页 111000000) 子栏目: 111001001环评审批 111001002固废许可 111001003危废许可 111001004排污许可 111001005夜间施工 111001006排污口审批 111002000行政处罚 111003000行政强制 111004000行政规划 111006000环境信访 111007000其他权力 111007001现场检查 111007002清洁生产 111007003专项资金 111008000公共服务 111008001环保宣传 111008002环境质量 111008003重污染天气 111008004固废危废 111009000污染综合防治 详情: /Jcsd/show/{id}.html → div#zoom.j-fontContent + meta ArticleTitle/PubDate 运行: python3 crawl_lixin_jcsd.py [--pages=N] [--sub=111001001,...] [--dryrun] N)datetime timedelta) BeautifulSoupu利辛县-基层政务公开zhttps://www.lixin.gov.cniG)daysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z/tmp/lixin_jcsd.jsonlz/root/search.dbzcrawl_lixin_jcsd.py) 111001001 111001002 111001003 111001004 111001005 111001006 111002000 111003000 111004000 111006000 111007000 111007001 111007002 111007003 111008000 111008001 111008002 111008003 111008004 111009000ctjddD]2}|jdst|j ddcSt S)Nz--pages==)sysargv startswithintsplitMAX_PAGES_DEFAULTas %/root/gov_crawler/crawl_lixin_jcsd.pyparse_pages_argr(%sF XXab\( << #qwws|A' '( ctjddD]8}|jds|jddjdcSy)Nrz--sub=r,)rr r!r#r%s r'get_sub_filterr,,sI XXab\. << !773<?((- -. r)ctjdd|xsdjjddjddS)Nu ^[•·\s]+u​u)resubstripreplace)ts r' clean_titler43s> 66."ag2 . 4 4 6 > >x L T TU]_a bbr)ct|D]6} tj|dtidd}d|_|j cSy#t $rtjdYZwxYw)Nz User-AgentF)headerstimeoutverifyutf-8r.) rangerequestsgetUAencodingtext Exceptiontimesleep)urlretriesirs r'fetchrI7se 7^  S<*]*>\s*([^<]{8,100})rr;)titlerE)BASErIr/finditergroupr4append) sub_idpagerEhtmlitemsmhrefid_rLs r' list_itemsrXBs F/&vU ;C :D  E [[QSW X;771:qwwqz;qwwqz3J5c uTD[9:; Lr)c|jdk(r t|gS|jdvro|jdrgS|jdd}|rt j d|rgSt j ddt|jgS|jdk(rdg}|jgd d D]}|jt||s*|jdd}|r|jd |d |SgS)uV递归提取 HTML 块 (保持顺序): p/pre 保留内部HTML, table 保留, div 递归table)ppre Tr1uJ^(责任编辑|初审|复审|终审|\[纠错\]|浏览次数|字号|分享)z\s+divr[r\rZr_F recursivez

z

) namestr find_parentget_textr/matchr0r1find_allextend extract_blockrP)eltxtblockscs r'rjrjPs ww'By ww, >>' "Ikk#Tk*bhhlnqrIvsCG,22455 ww%9UK ,A MM-* + ,++c+.C C5o. Ir)ct|}|syd}tjd|}|r|jdj }|s[tjd|tj }|r4tj dd|jdj }d}tjd|}|r|jd}d}t|d}|jd d }g}|rm|jgd d D]} |jt| g} |D]!} | s| r | d| k7s| j| #dj| }g} |jddD]} | jdd}tjd|tjsd|vs?tj dd| j!dj xs|j#dd}|j%dr t&|z}n|j%ds t&dz|z}| j||d| D]} d| dd| dd }||vs|d!|zz } |||fS)"N)r.r.r.r.z+([^<]*)u\s*_?利辛县人民政府.*$z1rcz

 )rIr/searchrOr1Sr0rfindrhrirjrPjoinr>Irfr#r!rM)rErSrLrU date_textcontentsouprppartsrkdedupr[attsr&hrclinks r' parse_detailrgsY :D  E @$GA   "  II/rtt < FF;RLRRTEI FMAGGAJ G } -D 99Uv9 &D E -- <-N ,B LLr* + , A%59> Q ++e$ D ]]3T] *2 EE&"  99Da NRgklRl66,AJJTJ,BCIIK_qwwWZ|\^O_D||C 1H\\&)3JN KKa0 12#ahZ'9!F)HM w  td{ "G# )W $$r)cvt}t}dtjv}t d|d|r t |nt t dtdg}|r|nt }|D]}td|dzD]}t||}|s%t d|d |d t |d d|D]s} t|d \} } } | r | tkr"|j|d | xs|d | | |dt d|dd ddd| dtjdut dt |d d|r.|ddD]%} t t!j"| ddd'yt%t&dd5}|D]+} |j)t!j"| ddz- dddt*j,j/t0rt3j4t0d }|j7d!|j9}d"}|D]} |j7d#| d f|j;r*|j7d$| d t<d%| d | d&| d'd(d(t?j@d)| d&rtC| d&jEd*d(nd"tFf |jHs|dz }|jK|jMt d+|dyy#t$r'} t dt| dddYd} ~ d} ~ wwxYw#1swYxxYw),Nz--dryrunz[lixin_jcsd] pages=z subs=z cutoff=T)flushrz [sub=z] pagez: u 条rErL)rErLdater|r0u ✓ rr(z | u ✗ <g?u [完成] rF) ensure_asciiwr:)r@rux)r8zPRAGMA busy_timeout=120000rz'SELECT id FROM gov_raw WHERE page_url=?zINSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank, script_name) VALUES (?,?,?,?,?,?,?,?,?,?)u安徽rr|r.z\d{4}-\d{2}-\d{2}-u[入库] 新增 )'r(r,rr printlenSUB_IDSCUTOFFr<rXrrPrBrdrCrDjsondumpsopenOUTwriteospathexistsDB_PATHsqlite3connectexecutecursorfetchone SITE_NAMEr/rgr"r2 SCRIPT_NAMErowcountcommitclose) max_pages sub_filterdryrunresultssubsrQpnrTitrLr{r|erHfconnrnaddeds r'mainrsB!I!J 388 #F  {&JZTWX_T`0aaijpiq rz~G#:D 9q=) Bvr*E GF86"RE |4@ M  @0$?s9+NVZ[ 3   ( Ic'l^4 (5 ! ;A $**QU3DS9 : ; c3 )>Q >A GGDJJqu5< = >> ww~~gw4 12 KKM A II?!E( Mzz| II{7Y!E(AfIq|UWY[46HH=QSTU[S\4]QvY&&sB/0cdfqs tzz      (5#!@HSVCR[M2$??@>>s*4K;=K;1L.; L+ L& &L+ .L8__main__)r;)"__doc__rrr/rrCr=rrrbs4rurllib3disable_warningsrrMr$nowstrftimerr?rrrrr(r,r4rIrXrjrr__name__r)r'rs 2111( * ! (,,.9%0 0 : :: Fv #  c .+%\46n zFr)