9Jj2 d Z ddlZddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
ddl m Z dZ
dZdZd Zd
Zd Zd Zd
dddZd Zd Zd Zd Zd Zd Zd Zd Z d Z d Z e dk( r e yy) uJ
Crawler for 广信区人民政府 - 通知公告
CMS: PowerCMS + UCAPCONTENT
List: ul.infoList > li > span.date + a[href]
Pagination: list.shtml (page1), list_{N}.shtml (page N, N=2..41), 10/page, ~41 pages
Usage:
python3 crawl_srx_tzgg.py # Full crawl
python3 crawl_srx_tzgg.py --max-pages 1 # Incremental
N)datetime)
BeautifulSoup)urljoinz/root/search.dbu" 广信区人民政府-通知公告u 通知公告u 江西zhttp://www.srx.gov.cnz)http://www.srx.gov.cn/srx/tzgg/list.shtml) zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentAcceptzAccept-Languagec t | S )u' 保留 HTML 表格结构（不转 md）)str)tableargskwargss #/root/gov_crawler/crawl_srx_tzgg.py table_to_mdr $ s u: c g }t }| j D ]\ }t |t r%|j }|s'|j | 9|j dk( rp|j dd j }|j d }|r'|r%t || }|j d| d| d |s|j | |j d k( r`|j d
d } | s| |v r|j | |j d d }
t || } |j d| d 't |d
s5|j dd }|sL|j | _ dj | S )Nahref Tstrip zimgsrcalt ">查看图片get_text )
setcontents
isinstancer r appendnamegetr r addhasattrjoin) p_tag
detail_urlparts seen_imgselemtr textfullr r full_srcs r
render_paragraphr0 ( sV EI dC
A Q
YY#
88FB'--/D==t=,D z40 ybd ;< T"
YY%
((5"%C)#
c"hhub)":s3 y
2D EF
T:
&
c
.A Q3 4 88E?r c g }t }t }| j ddg D ] }|j | j D ]J }|j |j j }|dk( r!t
|| }|s?|j | Q|dk( rA|j d } t | dk rwt | }
|
s|j |
|dk( rt | |v r|j t | |j d d
} | s| |v r|j | |j d d
} t || }
|j d |
d
$|dv r|j g dd D ] }t | |v r|j t | |j dk( r3|j d rMt
|| }|s\|j | n|j dk( rA|j d } t | dk rt | }
|
s|j |
|j dk( r`|j d d
} | s| |v r|j | |j d d
} t || }
|j d |
d
-|j dk( s>|j d s|j d rb|j dd }|syd}t! | D ]I \ }}||v rt | t | k rd} n'||v s*t | t | k sB|||< d} n |r|j | *t
|| }|s:|j | M t }g }|D ] }t# j$ d| D ]W }|j' d |v r"|j) |j' d d
}8|j |j' d Y |sv|j | |}dj+ | S )Nscriptstylepr
Tr r r r r r r )divsectioncenter ucapcontent)r4 r
r span) recursiver: r Fu" 查看图片 r
)r find_all decomposechildrenr" lowerr0 r! r lenr idr$ r# r find_parent enumeraterefinditergroupreplacer& ) content_divr( r) processedseen_imgs_globalschildtntxtall_textmdr r r/ subis_dupiexistingseen_imgs_finaldedupedpartms r
extract_contentr[ H s EIu
!
!8W"5
6
%% R" ::
ZZ
9"5*5C S!
7] ~~D~1H8}r! U#B R
5[%yI% MM"U) $))E2&C** $$S)iir*":s3 y
2D EF
>
> ~~&Ct~T ,
.c7i'
bg&88s?w/ *3
;C S)XX( "||$| 7H8}r) $S)B R(XX& ''%,C "22 $(,,S1 !ggeR 0#*:s#; y
:L%MNXX's+sw/G ,,s$,7C!&+4U+; &KAx "h3s8s8}3L)- % '33x=3s83K+.a)- %& & !LL -Y,
.` #5*5C S!eR"j eOG ! BDI 0Awwqz_,||AGGAJ3 ##AGGAJ/ 0
NN4 !
E ;;u r c g }t }| j dd D ] }|d }t j d|t j s.t || }|j
d xs& |j d d j d d
}||vsy|j | |j ||d |S ) Nr T)r r z&\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$r /?r )r" url)
r r> rF searchIr r splitr$ r! )soupr( attsseenr r full_urlr" s r
extract_attachmentsrh s
D 5D
]]3T]
* =y
99>bdd K z40H::D:)NTZZ_R-@-F-Fs-KA-NDt#" T( ;<= Kr c t j | t d }d|_ t |j d }d}|j d d
d i }|r$|j d
r|d
j }|sL|j d }|r9|j d }t j dd| j }|}d}|j d d
di }|r'|j d
r|d
j d d }|j dd } | r| j d }
|
r|
} t || } | r
t! | | } nd} t# | j dk rd| d| d} |j% d }
g }|
D ]J }|j d d }|st' | | }|j d d }|j) d| d L |r| d d j+ | z z
} | r| D ] }| d |d! d|d
dz
} ||| | fS # t $ r1}t d| d| t j d d d g fcY d }~S d }~ww xY w)"N headerstimeoututf-8z fetch_detail error : file html.parserr metar" ArticleTitle)attrscontenttitleTr u6 \s*[-_—|].*(?:通知公告|广信区人民政府).*$PubDate
r6 conTxtclass_ UCAPCONTENT 查看图片
r= z
r r r r! )r r` r r rd r itemsr r r: r rw page_urlr s r
extract_list_pager
s}
x
C LLgr :
}}
z 8szzJ /D
4
+B
Ekk$ 6 GGCLwwvfw-f
y gr"((*Dajjtj.D3%044==t=," X LL%84 5 6 L9
!#b,3::> s #E. . F$7"F F$ F$c t j t d } | j d | j d | S )N< )rm zPRAGMA journal_mode=WALzPRAGMA busy_timeout=5000)sqlite3connectDB_PATHexecute)conns r
init_dbr 3 s1 ??7B /DLL*+LL+, Kr c
|r|n|}t j dd| j }|rt j |d nd} | j dt |||j |||t |t f
}|j dkD S # t $ r(} t d| t j Y d } ~ yd } ~ ww xY w)
Nz\s+r F) ensure_asciiz[]zINSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r z DB error: rp )rF rS r jsondumpsr SITE_NAMECATEGORYGROUProwcountr r r r )
r rw r publish_daterv r summaryatt_jsoncurr s
r
save_articler : s geGffVS'*002G>Itzz+E:tH ll
8 (EKKM<
gx5
:
||a
QC szz2s AB
B> B99B>c
t } t t j dkD r t t j d } t
dt d| t j t }d}d}t d| dz D ]K }t | }|s"t
d| d t j n |t | z
}|D ] \ }}}|r*|d
k r%t
d | d |d d
t j 3|j d|f } | j rWt | \ }
} } }
|
xs |}| xs |}| xs d| d| d}|
xs g }t |||||| r.|dz
}|dk r$t
d| d |d d t j t! j" d t
d| dt | d| dt j N |j% |j' t
d| d| t j y # t
$ rb t j d dv rIt t j dkD r- t t j d } n# t
$ r Y nw xY wY =w xY w)Nr< )z --max-pages z Scraping z
, max pages: rp r r z: 0 articles (end)z
2020-01-01z Skip old: r ( z)SELECT id FROM gov_raw WHERE page_url = ?r~ r r z + 2 g333333?ro z found, z newz
Done! Found: z, New: ) MAX_PAGESrB r argvint
ValueErrorr r r r ranger r fetchoner r timesleepcommitclose) max_pagesr total_new total_foundpnarticlesrw r r r det_titledet_date det_contentdet_atts final_title
final_date
final_content
final_attss r
mainr N s` I
388}q CHHQK(I
Ii[
i[
9
K 9DIKAy1}% W$R( GB4 12 D s8}$ %- !E8T|+ TF!E#2J<8szzJ,, JXKXC||~9Eh9O 6Ixh#,uK!)TJ ' [\(2k]RZ+[M! RJD+x]T^_Q >D Ak#2.>-?@szzR JJsO) , t2c(m_HYKtD3::V= W@ KKMJJL OK= {
;#**M] xx{ ..3sxx=13D #CHHQK 0I! s5 H :I> I+*I>+ I74I>6I77I>=I>__main__) __doc__r rF r r r osr r bs4r urllib.parser r r r r BASE_URLr r r r r0 r[ rh r r r r r r __name__ r r
r s 2 1 1 1
0 " 6 DO0 @i \ <1BC !L(3Nl zF r