vj"dZddlZddlZddlZddlZddlZddlZddlZdZ dZ e dZ dZ ddiZ d Zd Zd Zd Zd ZdZedk(reyy)uk 纳雍县人民政府 - 通知公告 (crawl_nayong_tzgg.py) TRS CMS - 静态分页 总计3254条，163页 N nayong_tzggzhttps://www.gznayong.gov.cnz /xwdt/tzggz/root/search.dbz User-Agentz> ! !#r ! 2D 99;  gi  88cvg}tjd|tjD]}|jdj }|jdj }|jdj }|j ds t |z}|j|||d|S)zExtract articles from list pagezM]*title="([^"]*)"[^>]*href="([^"]*)"[^>]*>.*?\s*([^<]*)rhttp)titlerdate)refinditerDOTALLgroupstrip startswithBASE_URLappend)htmlarticlesmr$hrefr%s r parse_listr2sH [[X bii E   "wwqz!wwqz!v&d?D%dCD E Orc tjj|t}tjj |d}|j j dd}d}tjd|tj}|r|jd }nqtjd |tj}|r|jd }n8tjd |tj}|r|jd }|rNtjd d|tj }tjdd|tj }ttjdd|j}|dkry|S#tjj$r}|j dk(rYd}~yYd}~yd}~wt"$rYywxYw)zFetch article detail contentrr r r rrz%]*id="Zoom"[^>]*>(.*?)rz2]*class="content"[^>]*>(.*?)\s*z.]*class="body_content"[^>]*>(.*?)z]*>.*?)flagsz]*>.*?z<[^>]+>iN)rrrrrrrr&searchr(r)sublenr*error HTTPErrorcode Exception)rrrr.contentr0text_lenes r fetch_detailr@-sr!nn$$S'$:~~%%c2%6yy{!!')!< II>bii P ggajG OQUWYW`W`aA''!*IIOQUWYW`W`aggajG ff92wbiiXGff7WBIIVGrvvj"g6<<>? b= << ! ! 66S= s$FFFG 8G G G ctjddtjv}|rdg}t dn&t dt dz}t dt dt dg}t}|D]} t|}t|}d}|D]4}|d |vs |j|d |j||dz }6t d |d t d t|d |dt|d tjdt|} t d| d| dk(r t dyt!dd5} t#j$|| ddddt dt dt'j(t*} | j-} | j/d| j1d}d}d}d}t3|D]%\}}| j/d|d f| j5ddkDr(|dz }|dzdk(rt d|d | d |d!|d"| [t7|d }|r; | j/d#|d$||d t8|d%f| j1|dz }n|dz }|d'zdk(s|| dz k(rt d|d | d |d!|d"| ||z dk\r4t!d(d5} | j;t=|dzddd|}tjd)(| j?t d*t d+| t d,|t d-|t d.|y#t$r} t d |d| Yd} ~ d} ~ wwxYw#1swY"xYw#t$r} t d&| Yd} ~ $d} ~ wwxYw#1swYxYw)/Nz/rootz --incrementalrzMode: INCREMENTAL (page 1)z Mode: FULL (z pages)z(=== Phase 1: Collecting article URLs ===rrz Page /z: z items, +z new, z totalz ERROR: g?z === z unique articles collected ===zNo articles found!z/tmp/nayong_articles.jsonwF) ensure_asciiz"Saved to /tmp/nayong_articles.jsonz" === Phase 2: Fetching details ===zCREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, content TEXT, source_url TEXT UNIQUE, site_name TEXT, publish_date TEXT )z1SELECT COUNT(*) FROM gov_raw WHERE source_url = ?dz [z] done=z skip=z empty=zjINSERT OR IGNORE INTO gov_raw (title, content, source_url, site_name, publish_date) VALUES (?, ?, ?, ?, ?)r$r%z DB error: 2z/tmp/nayong_progress.txtg333333?z === Done ===zTotal: z Inserted: z Skipped: zEmpty content: ) oschdirsysargvprintrange TOTAL_PAGESsetrr2addr-r8r<timesleepopenjsondumpsqlite3connectTEMP_DBcursorexecutecommit enumeratefetchoner@ SITE_NAMEwritestrclose) incrementalpages_to_fetch all_articles seen_urlspager.r/newartr?totalfconncinsertedskippedempty last_saveir=s rmainrrSsHHW!SXX-K *+q+/2 [M12 45LI  /"4(D!$'HC u:Y.MM#e*- '',1HC   GD6;-r#h- #fUXYeUfTggmn o 3   E F5'7 89 z "# )3 /71 ,67 ./ /0 ??7 #D AII   KKMHG EIL)3 EE }U ::<?Q  qLG3w!|A3awghZvgYgeWUV s5z*  * A\7CJ 3v;O A  QJE r6Q;!uqy. Cs!E7'(6''%Q R y=C 0#6 $!AE # $I 4=@ JJL N GE7  Jxj !" IgY   OE7 #$K / GD6!- . . /77L * QC()) * $ $sO<$M!AM N 9N N7 N&M==NN N4N//N47O __main__)__doc__rTrHrJrQr&urllib.requestrrVr^r,rrXrrN PAGE_SIZErr2r@rr__name__rrryss 8777  (Zz "  W X  9  #L`%F zFr