yzj 3 @ d Z ddlZddlZddlZddlZddlZddlZddlm Z dZ
dZ dZ dddZ
d Zej j! ej j# ej j% e d
Zd Zd Zd
Zd Zd Zd Zd Zd Z e dk( r e yy)uP
南充市人民政府 - 新闻动态/公示公告 (www.nanchong.gov.cn/xwdt/gsgg/)
列表: div.normal-list > ul.list-ul > li > a[href="./202608/t20260810_2353100.html"] + span 日期
静态 1 页 13 条 (createPage(1,0,"index","html") 证实无真实分页, "共40页" 是模板注释残留)
详情: meta ArticleTitle/PubDate + div.TRS_UEDITOR 正文 (保留 /
)
样本: https://www.nanchong.gov.cn/xwdt/gsgg/202608/t20260810_2353100.html
https://www.nanchong.gov.cn/xwdt/gsgg/t_2351041.html (短ID变体)
用法: python3 crawl_nanchong_gsgg.py [--pages=N | N] [--sync]
N)
BeautifulSoupu 南充市-公示公告zhttps://www.nanchong.gov.cn&https://www.nanchong.gov.cn/xwdt/gsgg/zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36)z
User-AgentRefererz
2023-01-01zquality_results.dbz/root/search.dbzwww.nanchong.gov.cnc 0 dd l }|j t }|j d |j }|j d d}| D ] } |j d|j dd |j dd |j dd |j d d |j d
d |j d d |j d d |j d
d |j dd |j dd f
||j z
} |j |j t d| dt | |z
|S # t $ r,}t d|j dd d| Y d }~@d }~ww xY w)Nr PRAGMA busy_timeout=300000a CREATE TABLE IF NOT EXISTS crawl_results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
site_id INTEGER DEFAULT 0,
title TEXT, url TEXT, content TEXT,
publish_date TEXT, summary TEXT,
domain TEXT, category TEXT DEFAULT '',
content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
industry TEXT DEFAULT '', group_name TEXT DEFAULT '',
script_name TEXT DEFAULT '', attachments TEXT DEFAULT '[]',
UNIQUE(domain, url)
)zINSERT OR IGNORE INTO crawl_results (domain, title, url, content, publish_date, summary, industry, group_name, script_name, attachments) VALUES (?,?,?,?,?,?,?,?,?,?)domain titleurlcontentpub_datesummaryindustry
group_name script_name attachments[]z
[push] ERR : z[push_to_searchdb] new=z skip=) sqlite3connect
QUALITY_DBexecutecursorgetrowcount Exceptionprintcommitcloselen)records batch_labelr conncurn_newres (/root/gov_crawler/crawl_nanchong_gsgg.pypush_to_searchdbr) " ss ??: &DLL-.
++-CKK
E
:
: KK xx$aeeGR&8!%%r:Jy"%quuZ'Rz2&lB(?}b)155 +EG
S\\ !E : KKMJJL #E7&W e1C0D
EF L : M!%%b/!2"QC8 9 9 :s C E F)!FFc dd l } | j t }|j d |j dt f j }|j
|s t d y | j t d }|j d |j d dx}} |D ] }|\ }} }
} } }
}}}|j d | f }|j } |rF|j d
||
xs d d d | xs d | xs d d d |
xs d |xs d |xs d |xs d
|d f |dz
}nH|j d|| |
xs d d d | xs d | xs d d d |
xs d |xs d |xs d |xs d
t | f |dz
} |j |j dt f |j |j dt f j d }|j
t d| d| d| d y # t $ r}t d| Y d }~y d }~ww xY w# t $ r }t d| d| Y d }~d }~ww xY w# |j
w xY w)Nr r zSELECT title, url, content, publish_date, summary, industry, group_name, script_name, attachments FROM crawl_results WHERE domain=? ORDER BY idz[sync] local read ERR: u [sync] 本地没有数据i, )timeoutzPRAGMA journal_mode=WALz'SELECT id FROM gov_raw WHERE page_url=?zUPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, industry=?, group_name=?, script_name=?, attachments=? WHERE id=?r i r zINSERT INTO gov_raw (title, page_url, content, publish_date, summary, industry, group_name, script_name, attachments, site_name, source_url) VALUES (?,?,?,?,?,?,?,?,?,?,?)z
[sync] ERR r zINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z[sync] OK updated=z
inserted=u DB共u 条)
r r r r DOMAINfetchallr r r SEARCH_DBfetchone SITE_NAMEr )r r# rowsr' dstupdinsr& r
r r r
r indgrpsnattr$ rowtotals r( sync_to_serverr< E s
z* 12||
^
I
(*
)*
//)S/
1CKK)*KK,-MC# 2AHI EE37Cb#++ G#PC,,.C
2KK ^ B 8(.b7=VXZ^[^J_ CI2rxRc!fNO 1HCKK F gmWf%=x~2PWP][]_c`cOd CI2rxRiQTVW 1HC! 2&
Q L
Lyl[ddfghi se:cU&s
CDQ
's+,8
2
cU"QC011
2 sP AG9 -4I "BH 2A!I 9 HHH I&H=7I =II Ic t j dd| xs d } t j dd| } t j dd| j S )Nz$·| | |\u200b|\ufeffr u ^[•·]\s*z\s+ )resubstrip)ts r( clean_titlerC z sG
6AGDA
~r1%A
66&#q ! ' ' )) c ~ t j t t d } | j d| _ t
| j d }g }|j d D ] }|j d D ] }|j d d
}|s|d j }|j d
s>t |j! d
}|s\|j d } | r| j! d
nd}
t# j$ d|
r%t# j$ d|
j' d nd}
d|dd z } |j) || |
d t+ } g }
|D ]/ }|d | v r | j- |d |
j) | 1 |
S # t $ r}t d| g cY d}~S d}~ww xY w)u 南充 gsgg 静态 1 页 13 条 headersr+ utf-8 html.parserz [list] ERR: Nz
ul.list-ulliaThrefrN ./rA spanr z(\d{4}-\d{2}-\d{2})r, r )r
r r
r )requestsr LIST_URLHEADERSraise_for_statusencodingr textr r selectfind_allfindrA
startswithrC get_textr? matchgroupappendsetadd)r& soupr' itemsulrK rL rN r
rQ r
r seenuniqits r(
fetch_listri s LL7B ?
QVV]3
Ekk,' M++d# MB$'AV9??$D??4(
67E776?D48t}}4}0bHNPhhWmowNxrxx 6AGGJ AH:T!"XEC LL5(K L MM" 5D
D
e9 E B
K9
qc"# s A
F F<"F71F<7F<c t j | t d }|j d|_ t |j d }d}|j dd d
i }|r |j d rt |d }|s,|j d
}|rt |j }|snt j d|j t j }|r=t t j dd|j! d j# }d}|j dd di }|r'|j d r|d j# d d }|j% d xs |j% d }|-|j' t j( d D ] } | } n ||d|g dS g }
|j' dd D ] } | d j# } t j d | t j* s<| j- d r| n4| j- d rd | z n t. | j- d r| nd | z z t | j d! xs | j1 d d" }
t3 fd#|
D r|
j5 |
d$ |j% d% }|r|j' dd D ] } | d j# } | s| j- d r| n4| j- d rd | z n t. | j- d r| nd | z z t | j d! xs | j1 d d" }
t3 fd&|
D r|
j5 |
d$ |j' dd D ]_ } | d j# } t j d | t j* s<| j7 d' }|sP|j9 a |j' dd D ] } | d j# } | j- d s5| j- d( s$| j- d rJ| j- d) r\| j- d rd | z n t. | j- d r| nd | z z | d< |j' d* D ]a }|j d+d }|s|j- d r*|j- d rd |z n t. |j- d r|nd |z z |d+< c |j; d, D ] } | j9 |j' t j( d- . D ]9 } | j7 d' xs | j7 d/ }|s*|j9 ; t= |j? t j d0dt j t j* z 1 t j d2dt j t j* z 1 t j d3d4 t j d5d t j d6dt j 1 t j d7dt j 1 |
rDt3 fd8|
D }|s.djA d9 |
D }jC d:z |z tE t j d;d j# }d<v }|d=k r |s |
s|d||
dS |||
dS # t $ r}t d| d| Y d }~y d }~ww xY w)>NrF rG rI rJ z [detail] z ERR: r metaname ArticleTitle)attrsr h1z(.*?)u -\\s*南充市人民政府$r, PubDate
zdiv.TRS_UEDITORzdiv.view TRS_UEDITOR)class_r
r r
r rL TrM rN z)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$httpz//zhttps:/rP c 3 . K | ] }|d k( ywr N .0xabs_urls r( z parse_detail.. s 9qqx7*9 )rl r zdiv.attachmentc 3 . K | ] }|d k( ywry rz r{ s r( r z parse_detail.. s =11U8w.=r prO
javascriptimgsrczDdiv.share, div.ewm, div.qrcode, div.jzgov-qrcode-div, div.dzzw-shareu 扫一扫|分享到|打印本页)stringdivz]*>.*?)flagsz]*>.*?z]*>zz
\s*
z\s*.
s :ah)+:s c 3 ` K | ]& }d |d dt j |d d ( yw)z rl z
N)html_modescape)r| rL s r( r z parse_detail.. s3 mcd ahZr(//!F):T9UU] ^ ms ,.
<[^>]+>z![]()
Q===KKg >?
@
c
- y
99A4 N
c"A
c
- By ??3 4??4#8 QWAXaeapapq}a~,0OOD,AD x[_[j[jkn[oSWux{u HAAfIB & ~ggeR s~~f-,/NN4,@(S.hY\YgYghkYlRUrux{r{F|CJ~
nn cd
bjj1S&T U NN3 82>>%#8
KKM
G++-.I3R"$$QSQUQU+VI12yrtt TI{E95I}b)4I5r9BDDQI2B NI :T::ww mhl mmH!((*T1H r
( r z [fail] 2 r rt r r uE 风电|风力发电|海上风电|燃煤发电|光伏|太阳能|电力poweru 四川r r F) ensure_asciir ) r site_namer
r r r
r r r r r z[total] valid=z--sync)sysargvr\ intr
ValueErrorisdigitr ri r CUTOFFr r- r1 r? r@ r ospathbasenamejsondumpsr` r) r< ) max_pagesrL rd validrh detrecs r( mainr
sm I
XXab\ <<
#
Q 2 3 YY[AI
{
+, LE M#e*
&' E
j>bnv 5 K: /qGSb1A0BC D 2e9% ; Ibk#2 . /0 1 wKBBzNcefC "\0R[e99~J92j>vvj"c)n=dsC#%99-uwz |C xD xS HJ KR HS $T Z\" 77++CHHQK8QTUbQc4::c-&8uMim
S+,
N3u:,
'( 0 0! => sxx
A
s H<< I I __main__) __doc__rS r? r r r htmlr bs4r r1 r rT rU r r r dirnameabspath__file__r r/ r- r) r< rC ri r r __name__rz rD r( r s
$ ( 3 D7
WW\\"''//"''//(*CDFZ
[
F2Ej* #La]H) X zF rD