uj"ddlZ ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZejj!ejj#eZej(ddZdZdZdZd Zd Zd Ze j8Zd d iZddZdZ dZ!dZ"dZ#dZ$dZ%dZ&dZ'e(dk(re'yy)N)datetime)urljoin SEARCH_DBz/root/search.dbu!新疆准东经济技术开发区zwww.xjzdkfq.cnzhttps://www.xjzdkfq.cnz&https://www.xjzdkfq.cn/p163/sthj1.htmlz-https://www.xjzdkfq.cn/p163/sthj1_{page}.htmlz User-Agentz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36ctjj|t} tjj ||t }|j }|jdd}|S#t$r }Yd}~yd}~wwxYw)N)headers)timeoutcontextzutf-8replace)errors) urllibrequestRequestHEADERSurlopenCTXreaddecode Exception)urlr reqresprawtextes "/root/gov_crawler/crawl_xjzdkfq.pyhttp_getr#sp .. g 6C~~%%c7C%Hiikzz')z4 sA A22 BBczg}t}tjd|tj}|s%tjd|tj}|r|j dn|}tj d|tjD]}|j dj }tjdd|j dj }|r|s[|jdstt|}||vr|j||j||d |S) u#提取 xxgkzd 区域的文章链接z-class="xxgkzd"[^>]*>(.*?)]*id="myPage"zclass="xxgkzd"[^>]*>(.*?)z8]*href="(/p163/sthj1/\d+/\d+\.html)"[^>]*>(.*?)<[^>]+>http)titler) setresearchDOTALLgroupfinditerstripsub startswithrBASE_URLaddappend)htmlitems seen_urlsarea_mareamhrefr$s rparse_list_pager8.s EIYYGryy YF =tRYYO$6<<?$D[[C bii 4wwqz!z2qwwqz288:D v&8T*D 9   d uT23 4 Lc t|}|syd}tjd|}|r4tjdd|j dj }d}tjd|}|r$|j d}|ddd|dd d|d d }d}tjd |tj }|r|j dj }tjd d|tj tjz }tjdd|tj tjz }tjdd|}t|dkDr|dddz}|||dS)u.获取详情页：标题、正文HTML、日期Nr!z([^<]+)u/_国家级新疆准东经济技术开发区\s*$rz/(\d{8})/\d+\.html-z']*id="?Zoom"?[^>]*>(.*?)z]*>.*?)flagsz]*>.*?z

\s*\s*

i'z...)r$contentdate) rr&r'r,r)r+r(Ilen) rr1r$tmrAdmdr@cms r get_detailrHKsi C=D  E ,d 3B I2rxxXY{[aac D (# .B HHQKBQ%!Aa&1Qq6(+G 2 bii B ((1+##%&&5r7"))TVTXTX.Y&&3R RTRVRVW&&2B@ w<% fuo-G r9c8|sytjdd|}tjdd|}tjdd|}tj|}tjdd|}tjd d |}|j S) Nr!zz \n\nz  r z[ \t]+ z\n{3,}z )r&r,html_modunescaper+) html_contentrs rextract_plain_textrOrs  662L, OD 66,d +D 66*b$ 'D   T "D 66)S$ 'D 66)VT *D ::<r9c|dz|zjg}dgdfdgdfdgdfdgd fd gd fd gd ffD]+\}}tfd|Ds|j|-dj|S)NrKu 环评阶段)u环评u环境影响评价u 征求意见u第一次公示u 审批报批)u审批u批复u 拟批复u报批u许可u批准u 竣工验收)u验收u竣工u 试运行u 招标采购)u招标u投标u采购u 规划设计)u规划u设计u可研u可行性研究u选址 安全评价)rQu安评u安全预评价u 安全验收c3&K|]}|v yw)N).0kwts r zcategorize..s%2rQw%s,)loweranyr0join)r$rtagsstagekwsrVs @r categorizer_}s t ""$A D \] XY :; 78 TU VW   s %% % KK   88D>r9ctjt}|jd |jd|j |j y#tj$rY6wxYw)u)确保search.db有gov_raw和gov_search表aaCREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT, source_url TEXT, page_url TEXT, title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0, summary TEXT, status TEXT, category TEXT DEFAULT '', visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT '' )zCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5( title, content, site_name, content='gov_raw', content_rowid='id', tokenize='unicode61' )N)sqlite3connectrexecuteOperationalErrorcommitclose)conns r ensure_tablerhse ??9 %DLL        KKMJJL  # #   sAA43A4ctjt}|jd|tf}|j }|j |duS)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?)rarbrrc SITE_NAMEfetchonerf)rrgcurrs ris_url_crawledrnsF ??9 %D ,,PSVXaRb cC AJJL D=r9c |d}|jdd}|jdt|dd}|jdd}|d}t||}tjt } |j d||||tt||f|jd kDr@|j d |fj}|r |j d |d |tf|j|jd kD|jS#tj$rYDwxYw#|jwxYw) Nrr@r!summary,rAr$zINSERT OR IGNORE INTO gov_raw (title, content, publish_date, page_url, source_url, site_name, summary, tags) VALUES (?,?,?,?,?,?,?,?)rz'SELECT id FROM gov_raw WHERE page_url=?z>INSERT INTO gov_search(rowid, title, site_name) VALUES (?,?,?))getrOr_rarbrrcDOMAINrj total_changesrkIntegrityErrorrerf) itemrr@rprAr$catrgrow_ids r store_itemrys0 u+Chhy"%Ghhy"4W"=ds"CDG 88FB D ME UG $C ??9 %D  'GT3 7C H     !\\"KcVT]]_FLLXE95 !!A% -- s13AD,D$D,D)&D,(D))D,,D>cBtjdd}d|v}|r t}d}nK|Dcgc]}|jds|}}|r t |dj ddnd}d|d}t d td |d t d td |dt td}d}td|dzD];}|dk(rt} ntj|} t| } | st d|dd|dGt| } | st d|dd|dhd} d} | D]}t|dr | dz } |dz }t!|d}|rH|dxs|d|d<|dxsd|d<|dxsd|d<t#|j%dddd|d<t'|| dz } |dz }t d|dd|dt)| d| d| d d zd!"t+j,|rd#nd$>t t d%|d&|ycc}w)'Nrz--fullu全量z--pages=r=u 增量（u页）u🔍 u 爬虫 — u模式u 总页数: u, 本次爬取: u页)pagez [p2d/u] ✗ 获取列表页失败u] ✗ 未解析到条目rr$r@r!rArqrpu] ✓ u 条 (新增u 跳过)z  )endg?g333333?u 📊 完成！新增: u | 跳过: )sysargv TOTAL_PAGESr-intsplitprintrjrhrange LIST_TEMPLATELIST_TEMPLATE_Nformatrr8rnrHrOrrryrCtimesleep)args full_modepages_to_crawlmodea pages_match new_total skip_totalr|rr1r2page_new page_skiprvdetails rmainrs 88ABI[^11#6q9:q>*&1 E)KvV 45 M+&6~6Fc JK GNIJa!+,$. 19C!((d(3C} Db >"22MN O % Db >"22JK L   Dd5k*Q a U ,F &w @4=W "("3"9rY%f~3V "4VZZ 25N"OPTQT"UY t  MH NI " T"IQ~.fSZL H:U\]f\gghilttz~ )3-I$.L G $YK{:, GHgDs HH__main__)))osrr&rahashlibr subprocessr1rLsslurllib.requestr r urllib.parserpathdirnameabspath__file__BASE_DIRgetenvrrjrsr.rrr_create_unverified_contextrrrr8rHrOr_rhrnryr__name__rSr9rrs ^]]]]  77??277??84 5 BIIk#4 5 /  #8 A %c$$&3  :%N   (B;Iz zFr9