V_jdZddlZddlZddlZddlZddlZddlZddlZddlmZdZ dZ dZ ejddZ d d d d Zd ZdZdZdZdZdZddZdZedk(reyy)uY雅安市生态环境局 - 公示公告 (sthjj.yaan.gov.cn) CMS: 自建CMS (标签式内容发布) 列表: /xinwen/list/{uuid}.html + ?page=N (N=2~8, 共8页) 详情: /xinwen/show/{hash}.html, content=div.nr-rap > div.xqing-web-box 用法: python3 crawl_yaan_sthjj.py # 全量8页 python3 crawl_yaan_sthjj.py 1 # 增量1页 N)datetimezhttps://sthjj.yaan.gov.cnz$f8afd4bc-28ae-407b-ac92-139d91beb15fu%雅安市生态环境局-公示公告 SEARCH_DBz/root/search.dbz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z User-AgentAcceptzAccept-Languagec$td|dy)Nz T)flush)print)msgs %/root/gov_crawler/crawl_yaan_sthjj.pylogr s Bse*D!c b tj|td}d|_|jdk(r$t |j dkDr |j Std|d|jdt |j y#t$r}td |d |Yd}~yd}~wwxYw) N)headerstimeoutzutf-8z[WARN] u → status=z, len=z[ERROR] : ) requestsgetHEADERSencoding status_codelentextr Exception)urlrespes r http_getr s ||C"=   s "s499~';99  gcU,t'7'7&8s499~>N OP  hse2aS !"sAB 1B B.B))B.cP|dk(rtdtdStdtd|S)Nz /xinwen/list/z.htmlz .html?page=)BASE LIST_UUID)pages r get_list_urlr&+s3 qy}YKu55V= ;tf ==r cg}tjd|tjD]}|jdj }|jdj }|jdj }|sc|sf|si|j ds t |z}|j|||f|S)uN从列表页提取 (title, url, date) 元组。title 优先取 title 属性。z]]*title="([^"]*)"\s*href="([^"]*)"[^>]*>.*?\s*([^<]*)r"http)refinditerDOTALLgroupstrip startswithr#append)htmlitemsmtitlerdates r parse_listr71s E [[h bii -   "ggaj wwqz! ST>>&)Sj LL%d+ , - Lr crt|}|sy|}tjd|}|r#|jdj }|r|}d}tjd|}|r|jd}d}tjd|tj } | r| jd} n;tjd|tj } | r| jd} nd} | rtj dd| tj } tj d d| tj } tj d d| tj } | j } | r| }g} tjd |tjD]} | jd}| jd j }|sdt| dz}|jdr t|z}|jdstdz|jdz}| j||dtj dd|j }tj dd|j }|s%| r#djd| D}d|d|d|}|||| dS)u+抓取详情页，提取正文HTML、附件Nu3(.*?)\s*-\s*雅安市生态环境局r"z(\d{4}-\d{2}-\d{2})zX]*class="nr-rap"[^>]*>\s*]*class="xqing-web-box"[^>]*>(.*?)\s*z8]*class="xqing-web-box"[^>]*>(.*?)\s*z]*>.*?)flagsz ]*>z]*>.*?zJ]*href="([^"]*\.(pdf|doc|docx|xls|xlsx|zip|rar|7z))"[^>]*>([^<]+)r)u附件/r*)namer<[^>]+> z\s+ c3:K|]}d|dd|ddyw)[r<z](r)N).0as r z!extract_detail..s'Pq!F)Bqxj:Psz

u

PDF附件： )r5r6content attachments)r r+searchr.r/r-subr,Irr0r#lstripr1join)rtitle_from_listr2r5tmtr6dmrGcmrawcm2rHamatt_urlatt_name text_only pdf_linkss r extract_detailrZCsr C=D  E I4 PB HHQK    E D )4 0B xx{G c bii B hhqkiiSUY[][d[de ))A,CC ff12s"))Lff^RBII>ff/S Jiik GKkkU bdd ?((1+88A;$$&K 0 234H   c "WnG!!&)Sj7>>##66GHW=> ?z30668IvsI.446I IIPKPP  Rw.H T"  r ct|}td|d|t|}|sgddfSt|}tdt |dd}d}g}t |D]\}\} } } t | | } | s|dz }|j| dt| | | dxs| | d tjd d | d jxs| dd d d| drtj| ddndd tjd|rt!j"t$} | j'd| j)}|D]S} |j'd|d|d|d|d|d|d |d|d|df |j*dkDr|dz }n|dz }U| j/| j1|||fS#t,$r}td||dz }Yd }~d }~wwxYw)u4爬取单页列表，返回 (items, stored, skipped)u 列表页 rru 发现 u 篇文章r"r5r6rGr=r>Nru 公示公告rHF) ensure_asciir9) r5 site_namepage_url source_url publish_daterGsummarycategoryrHg333333?zPRAGMA busy_timeout=10000zINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category, attachments) VALUES (?,?,?,?,?,?,?,?,?)r]r_r^r`rarbz DB error: )r&r r r7r enumeraterZr1 SITE_NAMEr+rJr/jsondumpstimesleepsqlite3connectDB_PATHexecutecursorrowcountrcommitclose)r% incrementallist_urlr2r3storedskippedenrichedir5rr6detailconncitemrs r crawl_pager{s8D!H*TF"XJ'( H D 1ax t E)CJaKFqLG "  &' !!  j$%1  sAG  G2G--G2cd}t}ttjdkDrtjddk(rd}d}t ddt dt d|d t d t t ddd }d }td|dzD]}t|\}}}||z }||z }t dd t d |d|t d|dy)Nu全量r"1u增量r?z2==================================================u🏠 z (rBz DB: ruZ──────────────────────────────u ✅ 新增: u | 跳过: u📤 完成！共 u 条新数据 ) TOTAL_PAGES_ALLrsysargvr rdrkranger{)mode max_pages total_stored total_skippedr%r3rsrts r mainrs DI 388}qSXXa[C/  Bvh- E)BtfA &' GG9  VHB-LMaQ'!!+D!1vw  !  Bxj/ Lk- AB |nO <=r __main__)F)__doc__r+rriosrrgrerr#r$rdgetenvrkrr~r r r&r7rZr{r__name__rCr r rs2111" 2 3 "))K!2 3oO0  " > $JZ9"x>2 zFr