j3& d dl Z d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dlm Z d dl
m Z e j dd Z
dZdZd Z ej" ed
z
j% d Zd
ddddZh dZd Zd dZd Zd Zd Zd Zd Z e dk( r7 e ej> dkD rej> d d k( r e y e yy) N)datetime timedelta)
BeautifulSoup)urljoin SEARCH_DBz/root/search.dbu 大观区-通知公告zhttps://www.aqdgq.gov.cnz-https://www.aqdgq.gov.cn/zwdt/tzgg/index.htmliG )daysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36zUtext/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8zzh-CN,zh;q=0.9zhttps://www.aqdgq.gov.cn/)z
User-AgentAcceptzAccept-LanguageReferer>) pdddldth1h2h3h4h5h6hrlioltdthtruldivnavpreformmainasidetabletbodytfoottheadcenterdialogfigurefooterheaderhgroupaddressarticlecaptiondetailssectionfieldset
blockquote
figcaptionc | yddl }ddlm g fd | dj }|j dd| }|j dd | }|j S )
ul 块级边界出换行、行内节点直接拼接、
出换行（≈ 浏览器看到的换行结构）。N r )NavigableStringc F | j D ] }t | rj t | *t |dd dk( rj d Lt |dd t
v r+j d | j d | y )Nnamebr
)children
isinstanceappendstrgetattr _BLOCK_TAGS)nodechr6 outwalks /root/gov_crawler/crawl_aqdgq.pyrD zbody_text..walk0 s --
B"o.
3r7#VT*d2
4 VT*k9
4 R
4 R
z[ \t\r\f\v]*\n[ \t\r\f\v]*r: z\n{3,}z
)rebs4r6 joinsubstrip)el_retr6 rC rD s @@@rE body_textrO ( sb z#
C H
A -tQ7A 61%A 779rF c dd l }t | D ]? } |j | t d }d|_ |j
dk( r|j c S A y # ||dz
k rt j d Y dxY w)Nr )headerstimeoutzutf-8 ) requestsrangegetHEADERSencoding status_codetexttimesleep)urlretriesrW irs rE fetchrd C st
7^ S'2 >A AJ}}#vv
$ 7Q;
1
s 9A A4c F g }t | d }|j dt j d }|s|S |j d D ] }|j d }|s|j dd j
}|s|j d
}|j d d j
}|r|j d rt t | }|j d
d } | r| j d
nd}
|s|s|
s|j |||
f |S )z3Parse list page, return list of (title, url, date). html.parserr doc_listclass_r atitler5 TrK href
javascriptspandate) r findrG compilefind_allrY rK get_text
startswithr BASE_URLr= ) htmlitemssoupr r rj rk rm r` span_daterp s rE
parse_listr{ P s
E } -D
4
; 7 8B
kk$ - GGCL gr"((*JJTJ*EuuVR &&(t|4 h%GGF6G2 1:y!!!- ST LL%d+ ,!-$ LrF c t | d }|j dd }|r t | nd}t j dd| j }d}d}|j dd }|r}|j
d
d D ]g }|j d
} t j d| }
|
r|
j d }d| v sCt j dd| j }i g } g } |j dd }
|
s|j dd }
|
rN|
j D ] }t |t r%|j } | s&| j | 8|j dv r t | } | sT| j | f|j dk( sv| j d |
j
dt j dt j D ] }t# t$ |j' dd }|j d
xs1 t( j* j- | j/ d d }|sk|sn| j ||d d j1 | j }||||| fS ) zParse detail page.rf r newstitlerh r5 z\s+ r newsinforo spTrl z!(\d{4}-\d{2}-\d{2}\s*\d{2}:\d{2})rU u 来源u ^来源[：:]?\s*z
j-fontContent newscontnet)r r r0 r9 rj z$\.(doc|docx|pdf|xls|xlsx|xlsm)(\?|$))rm rm ?r )r` r8 r: )r rq rO rG rJ rK rs rt searchgroupr; r< r> r= r8 rr Ir rv rY ospathbasenamesplitrI )rw r` ry title_elrk rp sourceinfor txtdm
content_parts attachmentscontrL a_tagatt_urlatt_name content_texts rE parse_detailr l sO } -D yyky2H#+Ih E FF63 & , , .E D
F 99U:9 .D --t-4 GB++D+)C?EBxx{3 4b#>DDF
G MK 99U?9 3D yy}y5 --
)B"c"hhj!((- 33 m!((-D $$R(
) ]]3RZZ8_acaeae-f]g GE h &"(=>G ~~D~1\RWW5E5Eg5N5T5TUX5YZ[5\H8""7H#EF G 99]+113L ,fk 99rF c
t j t }|j d |j }d}d}| D ]b \ }}}}} }
|s
|r|d d n|} |j d|f |j r|dz
}?|j d|||t || |
d| f |dz
}d |j |j ||fS )NzPRAGMA busy_timeout=5000r rT 'SELECT id FROM gov_raw WHERE page_url=?rU INSERT INTO gov_raw (title, page_url, publish_date, site_name, content, source_url, attachments, category, summary) VALUES (?,?,?,?,?,?,?,?,?) 政府公告) sqlite3connectDB_PATHexecutecursorfetchone SITE_NAMEcommitclose) rx connc new_count
skip_countrk r` rp contentr atts_jsonsummarys rE
push_to_dbr s ??7 #DLL+,
AIJ8= 4sD'69 #*'$3- ;cVD ::< !OJ
]
Cy'69nV] ^
Q KKMJJL j rF c t dt d t t } | s t d y t | }t dt | d g }|D ]$ \ }}}|t k\ s|j |||f & t dt | d g }t | D ] \ }\ }}}|dz dz d k( r t d
|dz d t | t | } | rLt | | \ }} }
} } |s|}|
s|}
| rt j | d
nd}
|j |||
| | |
f t j d |j d d |D cg c] \ }}}}}} |||||| f }}}}}}} t | \ }}t dt d t dt | t d| t d| y c c} }}}}}w )N[u ] 开始爬取 ⚠️ 获取列表失败
列表: 条u 近3年: rU
r u
详情: /F ensure_ascii[] ?c | d S )NrV )xs rE zmain.. s
ad rF T)keyreversez
[u ] 完成u
处理:
新增: u
跳过: )printr rd LIST_URLr{ len THRESHOLDr= enumerater jsondumpsr^ r_ sortr )rw rx all_itemsrk r` rp detail_resultsrb d_titler d_dater attsr r r r src
push_itemsr r s rE r r s Ai[
'( ?D
+, t E Js5zl$
'(I ! 1sD 9 eS$ / 01
KI't
,- N!*9!5 E3
aC2:? Jqse1S^$45 6Sz 5A$5L 2GWffdeGD6@D
4e<$I ! !7C&)"T U
3 ND9Ucdd5QRb$T2sBc40dJd &z 2Iz C {(
#$ Js>*+
,- Jyk
"# Jzl
#$
es #Gc
t dt d t t } | s t d yt | }t dt | d t
j t }|j }d}d}|D ] \ }}}|t k r|j d|f |j r|d z
}:t | } | sHt | | \ } }
} } }
| s|} | s|} |
rt j |
d
nd }|
r|
dd
n| }|j d| || t |
| |d|f |d z
}t! j" d |j% |j' t d| d| y)u 增量 - 只爬第1页r u ] 增量爬取r Nr r r r rU Fr r rT r r r r u
, 跳过: )r r rd r r{ r r r r r r r r r r r r^ r_ r r )rw rx r r newskiprk r` rp r r r r r r r s rE incrementalr ss Ai[
'( ?D
+, t E Js5zl$
'( ??7 #D
A
C D ! sD ) ;cVD ::< AID Sz 1=dC1H.&&$%v<@DJJt%8d #*'$3-
]
c69gvy.Za b
q
3/2 KKMJJL Jse:dV
,-rF __main__rU z
--incremental) ) r sysrG r^ r r r r rH r urllib.parser getenvr r rv r nowstrftimer rZ r@ rO rd r{ r r r r __name__r argvr rF rE r s ( ' ' (
"))K!2
3 $ % :
X\\^iU 3
3 = =j I De'* * 6 84:l!0+%Z+.Z z
388}qSXXa[O ;
rF