uj)dZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl Z dZ e dzZ e dzZdZdZd Zej$j'd d Zd d iZdZeej0D]=\ZZedk(s edzeej0ks%eej0edzZndZddZdZdZ dZ!dZ"e#dk(re"yy)u0扬州市人民政府 - 项目审批核准备案N) BeautifulSoupzhttps://www.yangzhou.gov.cnz0/zfxxgk/fdgkzdnr/zdjsxmpzyss/xmsphzba/index.htmlz7/api-gateway/jpaas-publish-server/front/page/build/unitu"扬州市-项目审批核准备案u扬州市人民政府u 环评平台 SEARCH_DBz/root/search.dbz User-Agentzzparse_detail..sIcs5zT1I)rQrtrtdthc3.K|] }|dk(ywrrrs r"rzparse_detail..s&Qcs5zT'9&Qrz| z | z |r|z--- rysrcaltz//zhttps:/z![z]()z![](rz )rrQstringrMrTrArVrWrXrirrrfind_allrq find_parentrUIrYrZanyr[joinlensplitinsertrsplitsetadd)r9rr\rQ meta_titler_ meta_dater` content_divrvpartseltextrIatitlergrcolscell cell_text col_countseprrseenunique_attachmentsrrtrKs @r" parse_detailrWs } -D E zz !!'') YYvfn-EYF *..3y)//1EI &(; bnnW5  77g ".."9  77c>;;s$;/D[[4[0 Ky(%d):biiHgimoqosos>tZZdZ3LquuZ7LF??62'$I[II#**VD+IJ K T" WW Dkk$' @KKt 5 +D $ c >I!]]3T]:S y0%-d1BbiiPoquwyw{w{F|%&ZZdZ%;%TquuZQS?TF#'??6#:'/$#&&Q[&Q#Q + 2 2VD3Q RSKK * +KKuzz$'7 7$ >? @Q c 23a7 UZZ)(;<>$'"S.C^^C("S.C/**S!,Q/#5;C 3r#bQ/d3%qMJk5Kn 5D+ u:T ! HHSZ  % %c *+ kk% GI'Rd eerbc tdd}|s0tdtjtjdt |\}}|dkDr ||zdz |znd}td|d|d|tjt r9|t kDr0td t d t d tjt }t|}td t|d |tjtd|dzD]}td|d|tjt||}|std|tjQt|}td t|d|tj|j|tdt|tjtjt}|jd|j} d} d} d} |D]} | jd| df| j!r+td| ddddtjt#| d}|s"td| dtjt%|| d}|dxs| d}|dxs| d}|d}|d}t'j(dd |}t'j(d!d"|j+}|r|dd#nd }|s#| dz } td$|tjd}|rVt'j,d%|}|r>t/|j1d|j1dz|j1d&z}|rt3j4|d'(nd)} | jd*|t6t8| d|||||f | j:dkDr| dz } | t|z } |j?|jAtd.t6d/| d0| d1| d2 y#t<$r/}td+|dd,d-|tjYd}~Nd}~wwxYw)3Nrrdz'[ERROR] Cannot fetch first page via APIrrz[INFO] Total count: z , page size: z, total pages: z[INFO] Limiting to z pages (--pages=rz [INFO] Found z items on page 1/rz[INFO] Fetching page rz[WARN] Empty response for page z items on page z[INFO] Total items from list: zPRAGMA busy_timeout = 30000z'SELECT id FROM gov_raw WHERE page_url=?rz[INFO] Fetching detail: rQ2z...z&[WARN] Cannot fetch detail, skipping: rRrtrvz<[^>]+>rz\s+r{z[WARN] Empty content: z(\d{4})-(\d{2})-(\d{2})Fr-z[]zINSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank) VALUES (?,?,?,?,?,?,?,?,?)z[ERROR] DB insert failed: r rz [RESULT] r:z new, z empty content, z attachments)!rFrrrexitrn_MAX_PGrarrangeextendsqlite3connectDBexecutecursorfetchoner#rrVsubrMrWrjrXr;r< SITE_NAME GROUP_NAMErowcountrcommitclose) first_html total_countrC total_pages all_itemspgr9r]connc new_count empty_contenttotal_attachmentsitemdetailrQr_rtrvplainsummary date_rankr`att_jsonr!s r"mainrs1b!J  7cjjI  +J7K@Ka;*Q.9*: ;#**M ??2 DLL./ AIM+T ;d5k^L ::<  (gs);(SW T IIl :tE{IwPWYaclm zzA~Q !S%55!S+TZ KKMJJL K {"YKvm_DTUfTggs tu  T .uSbzl#aSA S S TsAP## Q,$QQ__main__)rd)$__doc__rVrosr; urllib.parserurllib.request urllib.errorbs4rrrZLIST_URLr=r SITE_DISPLAYrenvironrArrr enumerateargvirIrrjr#rFrarnrr__name__rrbr"rs6&&&# ( H H N N 0 &  ZZ^^K!23 W X  chh DAqI~!a%#chh-/chhq1uo&  ,* _fDUvp zFrb