ώjA dZddlZddlZddlZddlZddlZddlZejjddZ ddddZ d Z d Z d Z d Zd ZdZdZd#dZdZdefdZedk(rdej,vr edydej,vrReedZeeZedeezedd D]"Zededdddedded dd!$yede"yy)$ug 辽宁（营口）沿海产业基地-通知公告 https://ykcyjd.yingkou.gov.cn/003/003003/about.html NDB_PATHz/root/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zJtext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z User-AgentAcceptzAccept-Languageu1辽宁（营口）沿海产业基地-通知公告u营口z(https://ykcyjd.yingkou.gov.cn/003/003003c6|dk(r tdzStd|zzS)Nz /about.htmlz/%d.html)BASE_URL)pages !/root/gov_crawler/crawl_ykcyjd.py get_list_urlr s$ qy-''*t+++cbtj|tdd}d|_|jS)NT)headerstimeoutallow_redirectszutf-8)requestsgetHEADERSencodingtext)urlrs r fetch_pagers& S'2tLAAJ 66Mr rcg}tjd|tj}|s|S|jd}d}tj||tjD]}|jd}|jdj }|jdj } |j drd|z}n|j dsd |z}|j ds|j||| d |S) Nz%]*id="infolist"[^>]*>(.*?)rzx]*>.*?]*href="([^"]*)"[^>]*title="([^"]*)"[^>]*>(.*?).*?]*class="ewb-list-date"[^>]*>(.*?)/https://ykcyjd.yingkou.gov.cnhttpzhttps://ykcyjd.yingkou.gov.cn/)rtitledate)researchDOTALLgroupfinditerstrip startswithappend) htmlr itemsmul_html li_patternm2rr!date_strs r extract_list_itemsr2$s E :D"))LA  ggajGMJkk*gryy9 Ehhqk !!#88A;$$& >># 1C7C'2S8C~~=>  S5(CD E Lr cd}tjd|tj}|r |jdj }nmtjd|tj}|rF|jdj }tj dd|}|j }d}tjd|}|r|jdj }d}g}d}tjd|tj}|r|jd}|r"tj d|tjtjzD]} | jd} tj d d| jd j } | jd s| jd rd | zn| } |j| xs| jd d| dtjd|D]Q} | jd s| jd rd | zn| } |j| jd d| dSg} tj d|tjtjzD]}|jd}|jdr| j|7|jd}tj d d|}tj dd|}|j }|s| j|dj| }|rt|j dkrNtj d d|}tj dd|j }|rt|dkDr|}|||||ttdS)Nz&]*id="ivs_title"[^>]*>(.*?)rz(.*?)u\s*[-—|]\s*.*$u&发布时间：(\d{4}-\d{1,2}-\d{1,2})z<]*class="ewb-article-content"[^>]*>(.*?)\s*