vjdZddlZddlZddlZddlZddlZddlZddlZdZ dZ e dZ dZ dde d d Z d Zd Zeezd z ezZdZdZdZdZedk(reyy)u| 掇刀区人民政府 - 通知公告 (crawl_duodao_tzgg.py) TRS jPage (dataproxy.jsp AJAX) 总计1407条, 94页(15条/页) N duodao_tzggzhttps://www.duodao.gov.cnz/module/web/jpage/dataproxy.jspz/root/search.dbz> ! !#r ! 2D 99;  gi  88cvg}tjd|tjD]}|jdj }|jdj }|jdj }|j ds t |z}|j|||d|S)z"Extract articles from XML responsezJ]*href="([^"]*)"[^>]*title="([^"]*)"[^>]*>.*?([^<]*)rhttp)titleurldate)refinditerDOTALLgroupstrip startswithBASE_URLappend)xmlarticlesmhrefr4r6s r- parse_listrC$sH [[U RYY Ewwqz!   "wwqz!v&d?D%dCD E Or/c^ tjj|dtdi}tjj |d}|j j dd}d}tjd |tj}|r|jd }n8tjd |tj}|r|jd }|rNtjd d|tj }tjdd|tj }ttjdd|j}|dk\r|SdS#tjj$r}|j dk(rdndcYd}~Sd}~wt"$rYywxYw)zFetch article detail contentr)rrrrrrzf]*class="con zoom"[^>]*>.*?]*class="main-txt"[^>]*id="content"[^>]*>(.*?)\s*rz;]*class="main-txt"[^>]*id="content"[^>]*>(.*?)z]*>.*?)flagsz]*>.*?z<[^>]+>iN)rr#r$r&r'r(r)r7searchr9r:sublenr;error HTTPErrorcode Exception)r5r+r,htmlcontentrAtext_lenes r- fetch_detailrR4sinn$$S<AV2W$X~~%%c2%6yy{!!')!< IIBFHJHQHQ R ggajG XZ^`b`i`ijA''!* ff92wbiiXGff7WBIIVGrvvj"g6<<>?"b.w0b0 << ! !+VVs]r* s*E#E(&E((F,FF, F,+F,cLtjddtjv}t d|rdt fg}t dt dnWt dtdzt Dcgc]}|t|t zdz tf }}t dt|dg}t}|D]\}} t||}t|}d } |D]4} | d |vs |j| d |j| | dz } 6t d |d |d t|d| dt| t!j"dt dt|d|syt%dd5} t'j(|| ddddt dt dt+j,t.} | j1}|j3d| j5d }d }d }t7|D]\}} |j3d| d f|j9d d kDr|dz }8t;| d }|r; |j3d| d|| d t<| df| j5|dz }n|dz }|d zd k(s|t|dz k(r#t d!|d"t|d#|d$|d%| t!j"d&| j?t d't d(t|t d)|t d*|t d+|ycc}w#t$r} t d |d |d| Yd} ~ 0d} ~ wwxYw#1swYxYw#t$r} t d| Yd} ~  d} ~ wwxYw),Nz/rootz --incrementalz === Phase 1: Collecting URLs ===rzMode: INCREMENTAL (1 page, z items)z Mode: FULL (z pages)rr5z -z: z items, +z, total=z ERROR: g?z === z articles collected ===z/tmp/duodao_articles.jsonwF) ensure_asciiz"Saved to /tmp/duodao_articles.jsonz" === Phase 2: Fetching details ===zCREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, content TEXT, source_url TEXT UNIQUE, site_name TEXT, publish_date TEXT )z1SELECT COUNT(*) FROM gov_raw WHERE source_url = ?zjINSERT OR IGNORE INTO gov_raw (title, content, source_url, site_name, publish_date) VALUES (?, ?, ?, ?, ?)r4r6z DB error: 2z [r z] done=z skip=z empty=g?z === Done ===zTotal: z Inserted: z Skipped: zEmpty content: ) oschdirsysargvprintr!range TOTAL_RECORDSminrIsetr.rCaddr>rMtimesleepopenjsondumpsqlite3connectTEMP_DBcursorexecutecommit enumeratefetchonerR SITE_NAMEclose) incrementalpagess all_articles seen_urlsstartendr?r@newartrQfconncinsertedskippedemptyirOs r-mainrMsHHW!SXX-K ,-X +H:W=> =1#4h?ASX)=9:AA SZL01LI  s 1UC(C!#HC u:Y.MM#e*- '',1HC   BugQse2c(m_IcU(3|K\J]^ _ 3  F3|$%%< =>  )3 /71 ,67 ./ /0 ??7 #D AII   KKMHG EL)3 EE }U ::<?Q  qLG s5z*  * A\7CJ 3v;O A  QJE r6Q;!s<0144 Cs!C -.ghZvgYgV[U\] ^ 3/2 JJL N GC %& '( Jxj !" IgY   OE7 #$YA$ 1 BugQse8A3/ 0 0 177D * QC()) *sI,#M>%M $AM :M49N M1M,,M14M> N# NN#__main__)__doc__rerXrZrbr7urllib.requestrrgror=r%rir&r^r! TOTAL_PAGESr.rCrRr__name__r/r-rs 8777  &j7 8 P723   x'!+8  9  2V%r zFr/