&j)8dZddlZddlZddlZddlZddlZddlmZddlm Z ejjddZ dZ dZd Zd Zd Zd d ddZej&Zej*j-eddZdZdZdZdZdZedk(reyy)u[ 茂名论坛-《茂名环保》专栏 (bbs.gdmm.com/forum-912) Discuz! forum, GBK charset N) BeautifulSoup)urljoinDB_PATHz/root/search.dbzhttps://bbs.gdmm.com/z*https://bbs.gdmm.com/forum-912-{page}.htmlu%茂名论坛-《茂名环保》专栏u论坛zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z`text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z User-AgentAcceptzAccept-Languagec tj|d}||_|jS#t$r}t d|Yd}~yd}~wwxYw)zFetch and decode HTML.timeoutz FETCH ERROR: N)sessiongetencodingtext Exceptionprint)urlrres /root/gov_crawler/crawl_gdmm.py fetch_htmlrsK KKRK ( vv  s#$s), A AA cg}d}tj||tjD]_}|jd}|jd}tjdd|j }|sM|j ||fa|S)z*Extract (tid, title) from forum list page.z(?:normalthread|stickthread)_(\d+).*?]*href="(?:https://bbs\.gdmm\.com/)?thread-\d+-\d+-\d+\.html"[^>]*class="s xst"[^>]*>(.*?)<[^>]+>r )refinditerDOTALLgroupsubstripappend)htmlthreadspatternmtid title_htmltitles rextract_threads_from_listr**s~GZG [[$ 2)ggajWWQZ z2z288:  NNC< ( ) Nc  ddddgd}tjd|}|r9tjdd|jdj }||d<tjd|tj }|r|jd}t |d }g}|jD]G}|jd k(r(|jd d } | s)|j| ;|jdk(r|jd} g} | D]T} | jddg} | Dcgc]}|jd }}| jdj|V| s|jdj| |jdk(r(|s|djdr|dxxdz cc<t|ts#|j }|s7|j|Jdj|}tjdd|}|j |d<|ds|jd d }||d<tjd|}|r"|jdj |d<tjd|}|r|jdj!dd}|j!dd j!dd j#}t%|d k\rm|d!|dj'd|dj'd|d"j'd|d j'df\}}}}}|d|d|d |d|d# |d$<n||d$<tj(d%|tj }|D]U\}}tjdd|j }t+t,|}|d&jd'|d(|d)Wtj(d*|tj }|D]U\}}tjdd|j }t+t,|}|d&jd'|d(|d)Wtj(d+|}|d,d-D]+}|t|d&vs|d&jd.|d)-|Scc}w)/zDExtract subject, content, date, attachments from thread detail page.r )subjectcontentdateauthor attachmentsz/]*id="thread_subject"[^>]*>(.*?)rrr-z.]*id="postmessage_(\d+)"[^>]*>(.*?)rz html.parserdiv T) separatorr!tabletrtdth)r!z |  brz z\n{3,}r.z!]*class="xw1"[^>]*>(.*?)r0u9发表于\s+(\d{4}[-/]\d{1,2}[-/]\d{1,2}\s+\d{1,2}:\d{2})/-:rz:00r/zC]*href="([^"]*forum\.php\?mod=attachment[^"]*)"[^>]*>(.*?)r1[z]()z9]*href="(attachment\.php\?aid=[^"]*)"[^>]*>(.*?)z!(https?://pan\.baidu\.com[^\s<]*)Nru[百度网盘]()rsearchr rr!rrchildrennameget_textr"find_alljoinendswith isinstancestrreplacesplitlenzfillfindallrBASE_URL) r#rresultr&r- post_htmlsoup content_partschilddiv_textrows table_linesrowcellsc cell_textsr full_textdate_strpartsymodhmi attch_linkshref clean_textfull_url attch_links2 pan_linkslinks rextract_detailrm8s F DdKA&&R4::<#y CT299UAGGAJ Y 6 ]] /EzzU" >>Ct>D!((2w&~~d+ ?CLL$6EBG!HQ!**4*"8!HJ!H&&uzz*'=>?!((;)?@t# r):)C)CD)I!"%-%E3'{{}!((.+ /.KK . FF9fi8 %OO-yi  4 @I )F9  6=A771:++-x NPTUA771:%%c3/  c*223<BBD u:?#Ahaq(958>>!;LeTUhnn]^N_afghaiaoaopqarrOAr1a !s!B4q1QCqC8F6N%F6N**ceikmktktuK!D dVVJD1779 8T*}$$q Bxj%BCD ::Z\`bdbkbklL"D dVVJD1779 8T*}$$q Bxj%BCD  ?FI"1 D s6-01 1 = ! ( (?4&)B CD Ms"Is&R<cddl}|s tdy|jtd}|j d|j }|j dd}|D]}|j d|d f|j r*|d r|d dd n|d }|j d |d |d ||jdt|jdt|d |jdd|jdr!dj|jdgndf|dz }|j|jtd|dy)zSave items to originals.db.rNzNo items to save.<r PRAGMA busy_timeout=30000a CREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, content TEXT, summary TEXT, site_name TEXT, group_name TEXT, page_url TEXT, publish_date TEXT, attachments TEXT, source_file TEXT, crawled_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) z)SELECT id FROM gov_raw WHERE page_url = ?page_urlr.r-z INSERT INTO gov_raw (title, content, summary, site_name, group_name, page_url, publish_date, attachments) VALUES (?, ?, ?, ?, ?, ?, ?, ?) site_name group_namer/r r1r9rz Inserted z new records.) sqlite3rconnectrexecutecursorfetchoner SITE_NAME GROUP_NAMErHcommitclose)itemsruconncur new_countitemsummarys r save_to_dbrsT  !" ??7B? /DLL,- ++-CKK I ?$zBRATU <<> +/ ?$y/$3'Y   O O  HH[) , HH\: .   HHVR 6:hh}6MDIIdhh}b1 2SU    Q ), KKMJJL Ii[ ./r+cddl}|jtd}|jd|j }|jd|j }t d|d|j|jy) zSync FTS indexes.rNror rpz INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r LEFT JOIN gov_search f ON f.rowid = r.id WHERE f.rowid IS NULL zSynced z to gov_search.) rurvrrwrxrowcountrr|r})rurr search_counts rsync_ftsrsp ??7B? /DLL,- ++-CKK < 12 KKMJJLr+c |ddl}|jd}|jdttd|jddd |jd tdd |jd dd |j }|j rdn |j}|j}g}d}td|dzD]}td|dtj|}t|} | std|Et| } tdt| d|| s tdx| D]*\} } |dkDr||k\rnd| d} td|dzd| d| ddt| }|s tdMt!|| }|dxs| }|d}t|j#d krK|j%d!r!d"j'|j%d!gnd#}|r d$| d%|d&|}n d$| d%|d'}||| |j%d(d#t(t*|j%d!gd)}|j-||dz }t/j0d*-|dkDs||k\sntd+t|d,|r!|j2st5|t7td-y).NruCrawl 茂名论坛-茂名环保) descriptionz --max-pageszMax pages to crawl)typedefaulthelpz--test store_truezTest mode: 1 page only)actionrz --max-itemszMax items to crawl (0=all)z --skip-dbzSkip DB save (debug)rz === Page z ===)pagez Failed to fetch page z Found z threads on page z No threads found, skippingzhttps://bbs.gdmm.com/thread-z -1-1.htmlz [z] TID=z: roz FAILED to fetch detailr-r.r1r9r z

u

附件： z

r/)r-r.rqr/rsrtr1g?z === Total: z items ===zDone.)argparseArgumentParser add_argumentint MAX_PAGES parse_argstest max_pages max_itemsranger FORUM_URLformatrr*rNrmr!rrHrzr{r"timesleepskip_dbrr)rparserargsrr all_items total_fetchedrrr#r$r' list_title detail_url detail_htmldetailr-r.attachments_textrs rmainrs  $ $1R $ SF  CI]^ ? 0 1 &$ OC1})!;7uIFJ C a(se2j"o5FG H$Z0K23#Krs  **..$5 6 " 8 3  Dp0  (   w ]@50p.M` zFr+