ujydZddlZddlZddlZddlZddlZddlZddlmZm Z ejddZ ddiZ dZ d Zd Zd Zd Zed k(reyy)uv 湘潭县人民政府 - 环境保护栏目 creatorCMS系统, GB2312编码 https://www.xtx.gov.cn/6571/8155/index.htm N)ThreadPoolExecutor as_completed SEARCH_DBz/root/search.dbz User-Agentz]*title='([^']*)'[^>]*>([^<]*)([^<]+)/)urltitledatesite:  items2g333333?)BASErequestsgetHEADERSencodingtext Exceptionprintrefindallstripappend SITE_NAMElentimesleep) itemspagerrhtmleentrieshref title_attr link_textrrs #/root/gov_crawler/crawl_xtx_hjbh.pyget_list_itemsr0s] E D %)QYj!tfGD64N  S'2>A!AJ66D ** b   " L29  -D*i$$&;)//*;E LLq' !    vRG ~V45 w<"   L   3?    GD6!- . 2 L7 s/D D7D22D7c|d} tj|td}d|_|j}|d}t jd |}|r#|jd j}|r|}d}t jd |t j}|r|jd j}t jd d|t jt jz }t jdd|t jt jz }|}|d} | s)t jd|}|r|jd } |||| |dfS#t $r||dd|d|dfcYSwxYw)u抓取详情页rr r r rrrz([^<]+)rz)]*class="xl-xqnr"[^>]*>(.*?)z]*>.*?)flagsz]*>.*?u.发布时间[：:]\s*(\d{4}[-/]\d{2}[-/]\d{2})) rrrrrrrsearchgroupr DOTALLsubI) itemrr(r)rmtcontentrawrs r/ fetch_detailr>9sd u+CD LLgr : vv ME +T2A GGAJ    EG >biiPAggaj ff12s"))bddBRSff/S BDD@PQ zmain..ls!sz.SELECT page_url FROM gov_raw WHERE site_name=?rz Existing: z, New: zAll up to date!z Fetching z details (5 threads)...) max_workerszwINSERT OR IGNORE INTO gov_raw (title, page_url, source_url, content, publish_date, site_name) VALUES (?, ?, ?, ?, ?, ?)rz DB error for rrz rz processed, new=z Done! z processed, z new.)rr0r#sqlite3connectrexecutesetr"fetchallclosersubmitr>rresultcommitrowcountr) total_newr&connexistingit new_itemsdonepoolr9fut_mapfrrr<rrcr*s r/mainrZ^sRI   E Ic%j\ ()    ??9 %DLL,-ll ?@     Ic)n%%< => D  *OdENOT4;;|T2D8OOg& OA./hhj +Ct 4LLNCgtT: ::>NI AIDbyA~Y!74&#i.!11A)MN OO$ HS^$L 5 ABJJL;BP 4uBqc233 4OOsZ! H.HI H>&I%rZ__name__rAr/rcsf ,++? BIIk#4 5 W X)  %P"5J1h zFrb