uj dZddlZddlZddlZddlZddlZddlZdZdZdZ dZ ejjdejjejjejj!edZd d d d ZddZdZdZdZddZedk(r5dZeej8d kDreej8d Zeeyy)ui 常州环评网 (www.czeia.cn) 爬虫 - 环评公参公示 静态ASP站点，46页共901条环评公示 Nu常州环评网z www.czeia.cnzhttp://www.czeia.cnz&http://www.czeia.cn/list.asp?classid=5DB_PATHz search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z User-AgentAcceptzAccept-Languagec |dkDr td|nt} tj|td}d|_|j }g}d }tj||}|D]T\}} } } | j} | j} | s+|jt| | t|| d d d Vd} tjd |} | rt| jd} d}tjd|}|rt|jd}td|dt!|d| d|d ||| fS#t $r }td|d|gddfcYd }~Sd }~wwxYw)u获取分页列表rz&page=headerstimeoututf-8u [ERROR] 列表页第u页请求失败: rNzW]*?href="(/show\.asp\?id=(\d+))"[^>]*?title="([^"]*?)"[^>]*?>([^<]*?) )idtitleurl publish_dateu*(?:总数|总数)\s*(\d+)u'(?:页次|页次)\s+\d+/(\d+)u 第u 页: 找到 u 条 (总计 条, 共 u 页))LIST_URLrequestsgetHEADERSencodingtext ExceptionprintrefindallstripappendintBASE_URLsearchgrouplen)pagerresphtmleitemspatternmatcheshrefid_valrdate_strtotal total_match total_pagespage_range_matchs /root/gov_crawler/crawl_czeia.pyfetch_list_pager2s'+axXJfTF #XC||C"= yy EhGjj$'G)0  %feX >>#  f+Zv&$SbM     E))I4PKK%%a()Kyy!KTR*0034  E$|CJ<}UG:k]Z_ `a +u $$E  &tf,=aSAB1axs/E E1E,&E1,E1cx tj|td}d|_|j}i}tjd|}|r"|jdj|d <tjd |}|r2|jdjjd d |d <tjd|tj}|s%tjd|tj}|s%tjd|tj}|rb|jdj}tjddt|}tjddt|}||d<|Sd|d<|S#t $r}t d|d|Yd}~yd}~wwxYw)u$获取详情页完整标题和正文rrr u" [ERROR] 详情页请求失败 z: Nz

([^<]+)

rru日期[：:]\s*([\d/]+)/-rz:
(.*?)
\s*
z+
(.*?)
u.
(.*?)本文网址z src="(?!http)zsrc="zhref="(?!http)zhref="content)rrrrrrrrr r!rreplaceDOTALLsubr) rr$r%r&resulth1_match info_match content_match content_htmls r1 fetch_detailr@Fs||C"= yy Fyy,d3H"..+113w5ts/F F9F44F9ctjt}|j}|j d|j |S)u获取数据库连接a4CREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, content TEXT, publish_date TEXT, url TEXT UNIQUE, site_name TEXT, source TEXT, attachments TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ))sqlite3connectrcursorexecutecommit)conncs r1get_dbrIos< ??7 #D AII    KKM Kc |j} |jd|d|jdd|jdd|dttdf|jdkDrt d |dd d d nt d |jdd|j y #t$r}t d|Yd }~y d }~wwxYw)u保存单条数据z INSERT OR IGNORE INTO gov_raw (title, content, publish_date, url, site_name, source) VALUES (?, ?, ?, ?, ?, ?) rr6r7rru - 环评公参公示ru ✅ 新增: N(...u ⏭️ 已存在: ID=r ?u ❌ 写入失败: )rDrEr SITE_NAMErowcountrrFr)rGitemrHr&s r1 save_itemrRs A,   M HHY # HH^R ( K k. /    ::> $T']3B%7$8< = .txxc/B.CD E , &qc*++,sBB// C8C  Cc tddtdtdtdtddt}d}t |\}}}|std|j ytd |d |d |r||krtd |d |}td t |d|D]}td|ddddt|d}|rE|jd|d|d<|jd|d|d<|jdd|d<t||tjdd}d}||krt |\}} } |sntd|dt |dt|D]\} }td| dzdt |d|ddddt|d}|rE|jd|d|d<|jd|d|d<|jdd|d<t||tjd|dz }||kr|j tjt} | j!dtfj#d} | j td dtd!td"| d#ttddy)$u主运行函数=z2==================================================u 🌐 z (u) 爬虫ru(⚠️ 无法获取列表数据，退出Nu 📊 总计 ru 页u ⚠️ 限制只爬 u 📄 第1页 (u 条)u ▶️ r2rMrrr6r7g?u 📄 第u页 (u ▶️ [r4z] z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?rz =u ✅ 完成! u 共导入 u 条数据到 )rrO SITE_DOMAINrIr2closer"r@rrRtimesleep enumeraterBrCrrEfetchone) max_pagesrGr#r'r/r-rQdetailpbar_len_idxconn2counts r1runrds AfX, GI;b X 67 AfX, 8D D / 5E;  89  N5'K= =>Y, ( 489  SZL ./ 4="-.c23d5k* "JJwW >DM#)::nd>>R#SD $jjB7DO$ 3 DH + %d+ q!  4&c%j\78"5) IC KAwaE |2d7mCR6H5IM N!$u+.F & 7DM BW '-zz.$~BV'W^$"(**Y";Y dD ! JJsO   ! + $ JJL OOG $E MMJYL Y b b def gE KKM Cx. N9+[~gY OP AfX,rJ__main__)r)N)__doc__rrrBossysrYrOrWrrenvironrpathjoindirnameabspath__file__rrr2r@rIrRrd__name__r]r"argvrrJr1rrs     3 **..BGGLLQYAZ1[]h$i jDO0  )%X&R&,0?D zI 388}q $  N rJ