uj, ddlZ ddlZddlZddlZddlZddlZddlZddlmZmZddl m Z ddl Z dZ dZ dZdZej dd Zej$ed z j'd Zd ZdedefdZdededededzfdZdedefdZdedefdZdedefdZdZdZ e!dk(r;e"ejFdkDrejFdndZ$e$dk(re yeyy)N)datetime timedelta)urljoinzhttps://sthjj.hefei.gov.cnz/hbyw/hpsp/jsxmhpgs/index.htmlz/content/column/6800451u"合肥市-建设项目环评公示 SEARCH_DBz/root/search.dbiG)daysz%Y-%m-%dz/tmp/solve_hefei_jsl.jsurl_pathreturncF tjdtgddd}|jj }|j dD]F}|j }|j ds% tj|}d|vrd|vr|cSHtd |d d d |jrtd |jd d d iS#YxYw#tj$rtdd YiSt$r}td |d Yd }~iSd }~wwxYw)zERun Node.js script to solve JS challenge, returns {jsluid, clearance}nodeT)capture_outputtexttimeout {jsluid clearancez [JSL ERROR] Nflushz [JSL STDERR] z [JSL ERROR] Timeout) subprocessrun JSL_SCRIPTstdoutstripsplit startswithjsonloadsprintstderrTimeoutExpired Exception)rresultoutlinedataes /root/gov_crawler/crawl_hefei.py solve_jslr*s(0 Z dB mm!!#IIdO D::CCC D =D DD rrc tjt|dd|d|dd}|jdk7ry|j} |j |j xsd d }d |vr|S d}d dl}|jd||j}|r-|jdj j}|sL|jd||j}|r-|jdj j}|r|gnggdzD]!} | s |j | d }d |vr|cS#|j d d S#YxYw#YBxYw#t$r} td| dYd} ~ yd} ~ wwxYw)u2使用JSL cookies请求页面，自动检测编码zrs1]*charset[\s]*=[\s]*["']?([a-zA-Z0-9_-]+)s)Content-Type[^>]*charset=([a-zA-Z0-9_-]+))r.gbkgb2312gb18030replaceu [ERROR] 请求失败: Tr)requestsgetBASE_URL status_codecontentdecodeencodingresearchIgrouplowerr#r ) rrrresprawr$charset_remencr(s r)fetch_with_jslrH2s,||j #\'x/CI;O     s "ll ZZ 8ZJFF" #  JJPRUWZW\W\ ] ggaj'')//1G LcSVSXSXYA''!*++-335!(WIb4YY C C9&!M'  zz')z441  (   *1#.d;sS8E! E!&E/B3E!#E;E!>E!EE!EE!! F*E??Fhtmlcg}tjd|tj}|s|S|jd}tjd|tj}|D]}tjd|}|r|jdnd}tjd|}|r|jdnd} tjd|} | r| jdj nd} | sLtjd|} | r4tj d d| jdj } | s| s|j| | |d |S) u解析列表页的文章条目z%class="doc_list[^"]*"[^>]*>(.*?)r1z]*>(.*?)z(\d{4}-\d{2}-\d{2})zhref="([^"]+)"ztitle="([^"]*)"z]*>(.*?)<[^>]+>)titleurldate)r=r>DOTALLr@findallrsubappend) rIitemsrFul_htmllislidate_mrOurl_mrNtitle_mrMspan_ms r)parse_list_itemsr\bs$ E :D"))LA  ggajG **+Wbii @CE126"(v||Ab +R0 %ekk!n2)).3,3 a &&(YY92>Fz2v||A?EEG 5 LL5dC D!E" Lcd}tjd|tj}|rD|jdj }tj dd|j }d}dD]_}tjd|d|tjtj z}|s@|jdj }n|sSdD]N}tjd |d |tj}|s/|jdj }n|sSd D]N}tjd |d |tj}|s/|jdj }n|r>tj d dtd|}tj ddtd|}||fS)u%提取详情页的正文HTML和标题rKz(.*?)r1u&[_-].*?(合肥|生态环境|政府).*$)zoomZoomarticler: news-contentzid\s*=\s*["\']?z["\']?[^>]*>(.*?)) TRS_EditorconTxtzarticle-contentCustom_UnionStylez ewb-article newscontnetminh500z class="[^"]*z[^"]*">(.*?))r`rar:rbz id="[^"]*zhref="/zhref="/zsrc="/zsrc=")r=r>rPr@rrR IGNORECASEr8)rIrMrZr:cidrFclss r)extract_detail_contentrls Eii/ryyAG a &&(@"eLRRTGE II-DEtRYYY[YfYfMf g ggaj&&(G   = C \#.@A4SA''!***,   A C Yse+=>biiPA''!***,   &&vhZq%97C&&eH:Q$7A '>r] html_contentc|sytjdd|}tjdd|j}|ddS)NrKrLz\s+ r)r=rRr)rmrs r) get_summaryrpsA  66*b, /D 66&#t $ * * ,D :r]c tddtdtdtddtt}|stddytd|ddd d dt j t }|j}d }d }d }d }d }||kr|d z }|d k(rt}n td|}td|d|dt||d|d} | stddnut| } | stddnYtdt| dd| D]} | d} | r| tkr|d z }| d} | d}t| jtd|d|d}|}d}|rt|\}}|r|}ntd| dddt|xs|} |j!d|| t"||| f|j$d kDr|d z }n|d z }t)j*d |d k(rIt-j.d!| }|r+t1|j3d }td"|dn| sn| snt)j*d ||kr|j5|j7td#dtd$|dtd%|ddtd&|ddtd'|ddy#t&$r!}td|ddd|dYd}~d}~wwxYw)(Nu([INFO] 合肥市环评审批爬虫启动Tru[INFO] 截止日期: u [JSL] 正在破解JS挑战...u[ERROR] JS挑战破解失败u [JSL] 通过! jsluid=rz...r2r1z ?pageIndex=z [PAGE z] ru [ERROR] 页面获取失败u) [WARN] 未找到文章列表，结束z  条rOrNrMrKu [WARN] 详情获取失败: itINSERT OR IGNORE INTO gov_raw (title, page_url, site_name, summary, content, publish_date) VALUES (?, ?, ?, ?, ?, ?)u [ERROR] 入库失败: r  - g333333?zpageCount:\s*(\d+)u 总页数: u [DONE] 合肥市爬取完成u 爬取页数: u 新增: u 已存在: u 跳过(日期超限): )r CUTOFF_DATEr* LIST_PATHsqlite3connectDB_PATHcursor COLUMN_APIrHr\lenr5r8rlrpexecute SITE_NAMErowcountr#timesleepr=r>intr@commitclose)cookiesconncur total_new total_old total_skippage max_pagesrrIrTitemrOpage_urlrM detail_html detail_title content_htmldtsummaryr(total_ms r)crawlrsP 4TB !+ /t< +D9 "G  ,D9 #GH$5bq$9#:# >dK ??7 #D ++-CIIJ DI     19 H$[7H b +48h(97;;OP 24 @  & =d K  SZL%T2# Drs1  0//(  ' , & 0 "))K!2 3x||~ u 55?? K ' 6.S.#.#.#*.`34<   FccdCN':T zchh-!+388A;D }   r]