}j]ddlZ ddlZddlZddlZddlZddlZddlZddlmZdZejddZ dZ ddiZ dd Z d Zd Zd Zd ZdZdZedk(reyy)N)datetimez+https://www.qg.gov.cn/zwgk/zdxxgk/hjbh/xmhp SEARCH_DBz/root/search.dbu泉州市泉港区-项目环评z User-AgentzC''R'8 EDyy{))'))D E E  E E E 7Q; 1 1#b<= s0AB!B7 BB B C/CCcg}d}tj||D]W}|jd|jd}}td|}|dddz|ddzdzd z}|j |||fY|S) zExtract list items from a pagez7]*title="([^"]*)"r r/N-01)refinditergroupBASE_URLappend)htmlitemspatternmhreftitlepage_urlpub_dates r" extract_linksr7s EHG [[$ '2ggaj!''!*eZq'8c>D1I-3d: eXx01 2 LcFd}tjd|tj}|r|jdj }|sVdD]Q}tjd|zdz|tj}|s1|jdj }|S|S)z Extract content from detail pagez,]*class="TRS_Editor"[^>]*>(.*?)r )zarticle-contentz news-contentcontentmaintextz]*class="[^"]*z [^"]*"[^>]*>(.*?)\s*)r*searchDOTALLr,strip)r/r;r2clss r"extract_contentrA)sG A4SA''!*""$ M C 2S8;^^`dfhfofopA''!***, N   Nr8cXtjd|}|r|jdS|S)z%Extract publish date from detail pagez(\d{4}-\d{2}-\d{2})r )r*r=r,)r/ default_dater2s r" extract_daterD8s, ($/Awwqz r8ctjd|}|rT|jd}dD].}|j|}|dkDs|d|}|j S|j S|S)z$Extract clean title from detail pagez(.*?)r )u -项目环评u -生态环境u-重点信息公开u -政务公开u-泉州市泉港区人民政府rN)r*r=r,rfindr?)r/fallback_titler2r4suffixidxs r" extract_titlerJ@sx )40A D F++f%CQwds {{}   {{} r8ctdddD]h}td|d}tjj |t } tjj |d}|jd k(r|cSjy #YpxYw) z Determine total pages by probingrr /index_.htmr r r)rr-rrrrrstatus)nrrr s r"get_total_pagesrSNs2q"  '!D)nn$$S'$: >>))#r):D{{c!"    s 1A::A>cd}ttjD]\}}|jr t |}"|j dr t |j dd}R|dk(sX|dzttjksxtj|dzjst tj|dz}tdtjjdtd|d g}td ttd z}|r4t!|}td t|d |j#|d} |dkDr||kDrn{td|d}td|dt|}|snRt!|}|snDtd t|d |j#||dz }t%j&dtdt|ddl}|j+t,d} | j/d| j/d| j1} d} d} |D]\} }}| j/d|f| j3r| dz } 0t|}|s| dz } Ct5|}|std| dd| dz } gt7|| }t9||} | j/dtt|||||ddj;ddj=f| dz } | jA| jCtd!td"| td#| td$t|y#t$rYswxYw#t>$r!}td|d || dz } Yd}~]d}~wwxYw)%Nrz--pages==r z--pages[z] Starting crawl: z (max_pages=)zFetching page 1...r%z Found z itemsrrMrNzFetching page z...g?z Total items to process: rr zPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000z'SELECT id FROM gov_raw WHERE page_url=?z [SKIP] No content: zINSERT OR REPLACE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, status, summary) VALUES (?, ?, ?, ?, ?, ?, 'published', ?)rP  z [ERR] DB error for rz === Completed ===z Inserted: z Skipped: zTotal: )" enumeratesysargvisdigitint startswithsplit ValueErrorlenrrnow isoformat SITE_NAMEr#r-r7extendrrsqlite3connectDB_PATHexecutecursorfetchonerArJrDr r?rcommitclose) max_pagesra all_itemsr/r0pagerrgconncinsertedskippedr4r5rC detail_htmlr; clean_titler6r!s r"mainry]sI#((# +1 99;AI \\* %  Q0 )^ACHH 5#((1Q3-:O:O:QCHHQqSM*I + Ahlln&&( ));I;lS\R]]^ _`I  C D d# U F+, D  q=TI-  '$t, tfC()Sz d#  U F+,   3   &s9~&6 78 ??7B? /DLL*+LL,- AHG)2!%x ;h[I ::< qLG Ho  qLG "+. )%*6 7 qLG $K7  \:  II=Hh Xw#&&tS1779; < MH=!F KKMJJL ! Jxj !" IgY   GC N# $%s  ^  )(2aS9 : qLG s+NAN NN N=N88N=__main__))osr*rr[urllib.requestr urllib.errorrr-getenvrirerr#r7rArDrJrSry__name__r8r"rs| ?# 8 "))K!2 3 - P       b&H zFr8