e|j' d Z ddlZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl Z ddl m
Z
mZ dZedz ZdZdZdZej( j+ d d
Zej( j+ d d Zd
ddedZdZdadZ e ej: D ]k \ Z Z e jA d r e!e jE dd d Z4e dk( s:e dz e$ej: k sS e!ej: e dz Zm edk\ rendZ%d dZ&d Z'd Z(d Z)d Z*d Z+d Z,d Z-e.d k( r e- yy# e#$ r Y w xY w# e#$ r Y w xY w)!u
crawl_huizhoucn_qwgg.py - 今日惠州网-城市公告
https://www.huizhou.cn/gov/polity_qwgg/
TRS WCM (gbk编码), 列表 ul.list.m_t15 > li > span.datetime + a[href=./YYYYMM/tYYYYMMDD_N.htm][title]
分页: index.shtml + index_N.shtml (N=1..49, 共50页 ~1100条; index_50 空)
详情: /gov/polity_qwgg/YYYYMM/tYYYYMMDD_N.htm
→ h1.article_title(标题) + div.info(来源/时间) + div#ajax_text(正文 p.Custom_UnionStyle)
N)
BeautifulSoupCommentzhttp://www.huizhou.cnz/gov/polity_qwgg/u 今日惠州网-城市公告u 广东zcrawl_huizhoucn_qwgg.pyDB_PATHz/root/search.db
JSONL_PATH zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-LanguageRefererg333333?g z--pages==z--pagesc t j t z
}|t k r t j t |z
t j at | D ]U } t j | t | }|j dk( r d|_ |c S t j d|dz z W y # t $ r Y )w xY w)N)headerstimeout gbk ) time _last_reqREQUEST_INTERVALsleeprangerequestsgetHEADERS status_codeencoding Exception)urlretriesr dtirs )/root/gov_crawler/crawl_huizhoucn_qwgg.pyhttp_getr# 9 s
y B
#b() I
7^ S'7CA}}# "
$
1q1u9 s $3B99 CCc | syt j | } | j dd j dd j dd j dd } t j dd| } | j S )Nr u u u u \s+ )html_libunescapereplaceresubstrip)ts r" clean_titler. K so ! A (B ''"5==hKSST\^`aA
vsA A 779 c b | dk( rt }nt d| dz
dz }t | }|g S |j }g }t j d| D ] }|j d |j d t
|j d }}}|sdt j d|j d r=t
t j d|j d j d nd }t j j t | } |j | ||f |S )
u0 抓一页列表, 返回 [(abs_url, title, date)]r
index_z.shtmlzr\s*(\d{4}-\d{2}-\d{2})\s*\s*]*href="([^"]*t20\d+_\d+\.htm)"[^>]*title="([^"]*)"r z >([^<]+)r r ) LIST_URLr# textr* finditergroupr. searchurllibparseurljoinappend)
pager r! html_textitemsmdatehreftitleabs_urls
r"
fetch_listrD T s qy6$q&00
Ay I E
[[ O QZ [ - GGAJ
K
4KEdTVT]T]^mopovovwxoyTzK /1771: F L LQ OP ACE,,&&x6
gud+, - Lr/ c t | }|ddddg fS d|j v st |j dk rddddg fS t |j d }d}|j dd }|rt |j
}|s>t j d
|j }|rt |j d nd}d}|j d d
}|r7t j d|j
}|r|j d }|j d } | ||ddg fS | j g d D ] }
|
j | j d D ] } | j | j d D ] }
t |
j
} | j d so| j d s^| j d sM| j d s<| j d s+| j d sd| v s| j d sd | v s|
j t | j d }
g }| j d d ! D ] }|d" j }|j d# s|j d$ r't j" j% t& | |d"< |j) j+ d% s%d&|j) v sd'|j) v st |j
} | s|j- |d" | f | j d(d ) D ]_ }|d* j }|j d# s|j d$ s9t j" j% t& | |d*< a t/ | }||||
|fS )+uC 详情页: 返回 (title, pubdate, content, has_table, attachments)NFu 页面找不到了i@ z html.parserr h1
article_title)class_u (.*?)_今日惠州网r
divinfoz(\d{4}-\d{2}-\d{2}) ajax_text)id)scriptstyleiframeobjectmetalinkc " t | t S )N)
isinstancer )ss r" <lambda>z parse_detail.<locals>.<lambda> s
Jq',B r/ )stringpu （编辑u (编辑u 编辑：u 责任编辑u （来源u (来源u 责任编辑：u 扫码u 分享到tableaT)rA rA /http)z.pdfz.docz.docxz.xlsz.xlsxz.zipz.rarz.wpsupload
attachmentimg)srcr` )r# r4 lenr findr. get_textr* r7 r6 find_all decomposeextract
startswithboolr, r8 r9 r: BASE_URLlowerendswithr; str)r r! souprB rF tmpubdaterJ r? bodytagctxt has_table attachmentsrZ hr_ rU contents r" parse_detailrx h sh
AyT4**qvv%QVVt);T4** /D E
4 0B BKKM*
YY6
?,. BHHQK(BG 99U69 *D II,dmmo > ggajG 99 9 $D |gtUB..}} TU
]]"B]
C }}S! #,,.) NN; '3>>)+DWbHc>>.1S^^K5PTWTbTbclTm$+s~~h/G;Z]K] MMO TYYw'(IK
]]3T]
* 5
fIOO << V 4 ,,Xq9AfI 779 ` aemqrqxqxqzez K OP OV OV OX X ajjl+C""AfIs#345 }}U}- ; J << V 4 --h:CJ; $iG '7I{ ::r/ c |s| S | j r| gng }|D ] \ }}|j d| d| d dj | S )Nz <p><a href="z" target="_blank">z</a></p>z
)r, r; join)rw ru partsrC rs s r" append_attachmentsr| s[ WIbE# N
|G9,>se8LMN ;;u r/ c | j dt f }|j D ch c] }|d c}S c c}w # t $ r
t cY S w xY w)Nz.SELECT page_url FROM gov_raw WHERE site_name=?r )execute SITE_NAMEfetchallr set)conncurrows r" load_existing_urlsr sN llKi\Z"%,,.13A111 u s )? :? ? AAc
| j d|t f }|j ry d} ddlm} || }| j d|||||t t t ||f
}|j dkD ryt j dd| } t j | } t j d d
| j d d } | j d |j |t | f y
y# t
$ r Y w xY w# t j $ r=}
dt# |
v r*t% j& d t) | ||||| cY d }
~
S d }
~
ww xY w)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?Fotherr )classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)<[^>]+>r r% r& r zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)Tlockedr2 )r~ r fetchone crawler_libr r
GROUP_NAME SCRIPT_NAMErowcountr* r+ r' r( r, lastrowidsqlite3OperationalErrorrl r r store_record) r r rB rw r@ rt r industryr summaryes r" r r sF
,,PSVXaRb
cC
||~ 5(/H ll
+ #ugtY
KQZ\d e
<<! ffZW5G ''0GffVS'288:4C@G LLf y':
# $ # # s1v JJqM c5'4K K
sF C< C- B/C< - C96C< 8C99C< <E 1E E EE c
z d } t sbt j t d } | j d | j d t | }t
dt | dd d }d }g }t d
t d
z D ] }t | }t
d | d t | d |st
d
d nG|D ]> \ }}} t s
|v r|d
z
}t | \ }
} } }
}| t
d|d d d| d |d
z
}G|
s|}
| s| } t | | } t t j dd| j }|dk r"d| vr |s t
d|
d d d| d |d
z
}t rC|j! ||
| | |
t" t$ t& d t
d|
d d d| dd |d
z
}t) | ||
| | |
}|r/|d
z
}j+ | t
d|
d d d| dd :|d
z
}A t rmt- t dd 5 }|D ]+ }|j/ t1 j2 |d d z - d d d t
d!t dt | d"d | | j5 | j7 t
d#| d$| d y # 1 sw Y `xY w)%N< )r zPRAGMA busy_timeout=60000zPRAGMA journal_mode=WALu
已入库 u 条, 用于列表级预查重T)flushr r
z [list] page=z items=zEmpty page, stop paginationz skip empty: z | r r
z<img)page_urlrB rw publish_datert site_name
group_name script_namez
JSONL + ( z ()z + wzutf-8)r F) ensure_ascii
zJSONL written: z rows)z [done] new=z skip=) r r connectr r~ r printra r _PAGESrD rx r| r* r+ r, r; r r r r addopenwritejsondumpscommitclose)r existing new_count
skip_count
jsonl_rowsr< r> rC rB r@ dtitleddaterw rt atts plain_lenokfr s r" mainr s D w3 01 ./%d+
3x=/)GHPTUIJJa!$ ) 4
TF'#e* 6dC /t < $) # GUD 'X"5a
6B76K 3FE7ItuSbzl#gY?tLa
($7GBFF:r7 ; A A CDI2~&"7vcr{m3wi@Ma
!! '&W$) !**#. #
6#2;-r%:$GQ dGVWeYOBQ W%VCR[ME7!4DAa
G#
) V
*cG
4 D!
D
3U;dBC
D D
|2c*o->fETR
K {&
5TB D Ds 1J11J:__main__)r2 r )/__doc__sysosr* r htmlr' urllib.parser8 r r r bs4r r ri r3 r r r environr r r r r r
_MAX_PAGES enumerateargvr rZ rg intsplit
ValueErrorra r r# r. rD rx r| r r r __name__ r/ r" <module>r s & # ) ) *
'
**..$5
6
ZZ^^L"
-
@O'
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA$(6;r DACH zF S
s$ +D1'D<1D98D9<EE