Sjz' dZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl Z dZ dZ dZe je dz j!d Zd Zd Zd Zd e dzddZdZdZdZdZdZdZdZdZedk(reyy)ul永丰县人民政府 - 公告公示栏目爬虫 站点: http://www.jxyongfeng.gov.cn (江西吉安永丰县, 维网科技 CMS) 栏目: news-list-gggsaa.html (公告公示, catid=4734) WAF: 无 (requests 直连 HTTP 200) 列表: POST /api-ajax_list-{page}.html jQuery 数组序列化 body: ajax_type[]=5_news&ajax_type[]=4734&ajax_type[]=5&ajax_type[]=news&ajax_type[]=Y-m-d&ajax_type[]=40&ajax_type[]=20&ajax_type[]=["is_top DESC","displayorder DESC","inputtime DESC"]&ajax_type[]=&is_ds=1 返回 JSON {total, data:[{id,title,url,inputtime,...}]}, 20条/页 详情: /news-show-{id}.html UTF-8 (必须 r.content.decode('utf-8'), 不能用 r.text 否则 latin-1 乱码) 正文 div.xxq_essay (id=content); 标题 meta ArticleTitle; 日期 meta PubDate 纯PDF型: iframe src=/uploadfile/5/Attachment/xxx.pdf → 正文=标题+URL内嵌段 附件: 文件名.xlsx 相对路径 → BASE 绝对化 入库: 自动检测 /root/search.db 存在则 INSERT OR IGNORE 直插 gov_raw (FTS 触发器自动同步 gov_search) 运行: python3 crawl_yongfeng_gggs.py [--pages=N] [--no-db] N)datetime timedeltau永丰县-公告公示zhttp://www.jxyongfeng.gov.cn4734iG)daysz%Y-%m-%dz/tmp/yongfeng_gggs.jsonlz/root/search.dbu 公告公示zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z/news-list-gggsaa.htmlXMLHttpRequest)z User-AgentRefererzX-Requested-Withctjdd|xsdj}|jddjdd}|S)Nu^[•··.\s]+u​u)resubstripreplacets (/root/gov_crawler/crawl_yongfeng_gggs.py clean_titler"sD  "ag2.446A (B''"5A Hcddlm}||d}|jdD]}|j|jddgD]}|j  |jdDcgc]}|j ds|}}|sn|dj G|jdd D]}|j t |Scc}w) Nr) BeautifulSoupz html.parserc:|jxrd|jvS)N:)namers rzclean_content..*sqvv'?#-rspanfontpdiv vw_editor)class_)bs4rfind_all decomposeunwrapfindstr)htmlrsouptagrnestedrs r clean_contentr*'s! } -D}}?@ }}ff-.  !]]3/?166#;!?? q   }}U;}7  t9@s 3C Cc|r|jdr|S|j}|jdr|jddS|jdr t|zStdz|zS)N javascripthttpzhttp:///) startswithr rBASE)hrefs r absolutizer2:sb 4??<0 ::]+href="([^"]+)"[^>]*>(.*?)r,z\.(pdf|docx?|xlsx?|zip|rar|7z)$<[^>]+>r dr.)rurlr9)r finditerISgroupr r/searchHunescaper appendsplitr2setadd) html_fragattsmr1 name_htmlrseenuniqas rextract_attachmentsrLDs D [[@)RTTTVTXTX[ YX''!***,aggajit|4  997rtt D::bffZY?@FFH#ND KK!<C)d K JJqMM sAA// B08.B++B0c<tdD]M} tj|td}|jdk(r|j j ddcSOy#t$r5}td|d zd |d tjd Yd}~d}~wwxYw)NrTrV)rXrYrZutf-8r)errorsu 详情失败(r4r[Tr\r5) r_r`getrbrccontentdecodererfrgrh)r9rjrkrls r fetch_detailrtes8  S'2>A}}#yy'' 'BB$   OGAI;c!5T B JJqMM sAA B&+BBc|sytjjtst dtddyt j td}|jdd x}}|j}|D]} |jd t|d |d |d |d |d|dxsdddtd|d|jdd|jddf |jd kDr|dz }n|dz }|j|j||fS#t j$r}t d|d|dz }Yd}~Sd}~wwxYw)N)rru 未发现 u, 跳过入库 (仅写 JSONL)Tr\i")rYzPRAGMA busy_timeout=290000rzINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments, group_name, industry) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)r9titlepub_daterrr izcrawl_yongfeng_gggs.py attachments group_name江西industryotherr4z DB err: )ospathexistsDB_PATHrfsqlite3connectexecutecursor SITE_NAMECATEGORYrqrowcountErrorcommitclose)resultsconnaddedskippedcurrkrls r push_to_dbrps[  77>>' " WI%BC4P ??7C 0DLL-.EG ++-C   KK7AeHah' AjM1Y<I,$"ds+X7OQRS`Qa|X.j'0JL  ||a 1  #$ JJL '> }}  Jqc"$ / qLG sA|d?dt d@|ddA|d%dB|d,dC|dd? dy#1swYtxYw)DNz--no-dbz--pages==r4u站点: z CUTOFF: Tr\r)list detail_okemptycutofftotalpagesFru 第u页无数据, 停止rRru共 u 条 / u 页rWu页无条目, 停止u页: u 条rrvr inputtime rr9z /news-show-idrUu 详情失败: rVrz3]+name="ArticleTitle"[^>]+content="([^"]+)"z.]+name="PubDate"[^>]+content="([^"]+)"z6
]*>(.*?)
]+src="([^"]+\.pdf[^"]*)"r6u 空正文跳过: rz

点击查看PDF公告

z

z(]+src=")([^"]+)(")c~|jdt|jdz|jdzS)Nr4r5rT)r=r2)mms rrzmain..s.rxx{Z -DDrxxPQ{Rrrzr|) ensure_ascii) site_namervrwrr source_urlr9ryr{rxwro)encoding u已写 u 条到 u=== 入库: 新增u 跳过z ===u=== 完成: 列表u 详情OKu 空u CUTOFF截)!sysargvr/intrBrfrCUTOFFrmrqlenrr0rtr r>r?r@r=r<findallr r r2r*rLrddumpsrAopenOUTwriter)no_db max_pagesrKrstats cutoff_hitnjitemsit list_titlerw detail_url detail_htmlrGrv content_htmliframestextpdf_urlrrrFrecfrkrrs rmainrs  !EI XX0 << #AGGCOA./I0 HYKy 1>GQa!VW XEJ A  Y  qMg E!01 >  >Q wZE'Nj2o"4E'N Dw(g/?tDD Qfb! E!01 >  aSc%j\.d;4 B &MQ M$RVVGR%89J{+1r3B7HHv-h1$! QTF+bffTl^5*QJ&z2K*:cr?*;Q >A GGDJJqu5< = >> GCL># /t< #G,w "5' >dK uV}oYu[7I6J$uU\~N^^hinowixhyy} ~GKL>>s 1RR __main__)__doc__rdr}r rrgrr&r?rrr`rr0r^nowstrftimerrrrrbrr*r2rLrmrtrr__name__rrrs 3222( $ % (,,.9%0 0 : :: F   D..(   &" >^L@ zFr