j}- d Z ddlZddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
ddl m Z dZ
dZdZd Zd
Zd Zd Zd
dddZd Zd Zd Zd Zd Zd Zd Zd Z d Z e dk( r e yy)ut
Crawler for 苏锡通科技产业园区 - 环保公示
CMS: TrueCMS jPage (embedded initData)
List: #initData div with 27 items (single page, no AJAX access)
Detail: #zoom.test-1 content, div.container-main-title title, Meta PubDate
Usage:
python3 crawl_stpac_hbgs.py # Full crawl (all items)
python3 crawl_stpac_hbgs.py --max-pages 1 # Incremental
N)datetime)
BeautifulSoup)urljoinz/root/search.dbu( 苏锡通科技产业园区-环保公示u 项目环保公示u 江苏zhttp://www.stpac.gov.cnz.http://www.stpac.gov.cn/stkjcyy/hbgs/hbgs.html zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentAcceptzAccept-Languagec t | S )u' 保留 HTML 表格结构（不转 md）)str)tableargskwargss %/root/gov_crawler/crawl_stpac_hbgs.py table_to_mdr $ s u: c g }| j D ]Q }t |t r%|j }|s'|j | 9|j
dk( rp|j
dd j }|j d }|r'|r%t || }|j d| d| d |s|j | |j
d k( rU|j
d
d }|s|j
d d } t || }
|j | r d | d|
dnd
|
d t |d s*|j dd }|sA|j | T dj | S )Nahref Tstrip[]()imgsrcalt contents
isinstancer r appendnamegetr r hasattrjoin) p_tag
detail_urlpartselemtr textfullr r full_srcs r
render_paragraphr/ ( sK E dC
A Q
YY#
88FB'--/D==t=,D z40 qba 01 T"
YY%
((5"%Chhub)":s3 r#b
! 44zQRAST
T:
&
c
.A Q- . 88E?r c 6 g }t }| j ddg D ] }|j | j D ]E }|j |j j }|dk( r!t
|| }|s?|j | Q|dk( rA|j d }t | dk rwt | } | s|j | |dk( r}t | |v r|j t | |j d d
}
|
s|j d d
} t ||
} |j | r d | d
| dnd| d |dv s |j g dd D ] }
t |
|v r|j t |
|
j dk( r!t
|
| }|sJ|j | \|
j dk( rA|
j d }t | dk rt |
} | s|j | |
j dk( s|
j d d
}
|
s|
j d d
} t ||
} |j | r d | d
| dnd| d H dj | S )Nscriptstylepr
Tr r r r r r r r r )divsectioncenter)r3 r
r ) recursive
)setfind_all decomposechildrenr# lowerr/ r" r lenr idaddr$ r r& ) content_divr( r) processedschildtntxtall_textmdr r r. subs r
extract_contentrK D su EI
!
!8W"5
6
%% 1] ::
ZZ
9"5*5C S!
7] ~~D~1H8}r! U#B R
5[%yI% MM"U) $))E2&Ciir*":s3 r#b
! 44zQRAST
/
/ ~~&;t~L
]c7i'
bg&88s?*3
;C S)XX( "||$| 7H8}r) $S)B R(XX& ''%,C !ggeR 0#*:s#; r#b
!%<4PXzYZI[\)
];1]f ;;u r c g }t }| j dd D ] }|d }t j d|t j s.t || }|j
d xs& |j d d j d d
}||vsy|j | |j ||d |S ) Nr T)r r z&\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$r /?r )r# url)
r: r; researchIr r splitrA r" )soupr( attsseenr r full_urlr# s r
extract_attachmentsrY s
D 5D
]]3T]
* =y
99>bdd K z40H::D:)NTZZ_R-@-F-Fs-KA-NDt#" T( ;<= Kr c . t j | t d }d|_ t |j d }d}|j d d
d i }|r$|j d
r|d
j }|s|j d d
d i }|s'|j dd }|r|j d }|sL|j d }|r9|j d }t j dd| j }|}d} |j d d
di }|r|j d
ryt j d|d
j }
|
rPdt! |
j# d t! |
j# d t! |
j# d fz } | s3t j$ d|j } | r| j# d } |j dd } t' || }
| r
t) | | }nd}t+ |j dk rd | d | d }|j- d! }g }|D ]U }|j d"d }|st/ | | }|j d#d }|j1 |r d$| d%| d&nd'| d& W |r|d(d(j3 | z z
}|
r|
D ] }|d)|d
d%|d* d&z
} || ||
fS # t $ r1}t d| d| t j d d d g fcY d }~S d }~ww xY w)+N headerstimeoututf-8z fetch_detail error z: file html.parserr metar# ArticleTitle)attrscontentr5 zcontainer-main-titleclass_Tr titleu> \s*[-_—|]+\s*(?:环保公示|苏锡通科技产业园区).*$PubDate) (\d{4})[-/年.](\d{1,2})[-/月.](\d{1,2})%04d-%02d-%02dr u) 发布时间[：:]?\s*(\d{4}-\d{2}-\d{2})zoomr@ z z
r r r r r r r r9 u
附件：[rP )requestsr$ HEADERSencoding Exceptionprintsysstderrr r, findr r rQ rJ matchintgrouprR rY rK r? r; r r" r& )rP rerU ri mtct title_tagr+ date_strdm0dmrB attachmentsrf imgs img_linesr r r. r r s r
fetch_detailr s, $ LLgr :
/D
E
6&.!9 :B bffY 9
##%
YYvfn%=Y
>
YYu%;Y
<
KKdK+E IIg& """.AXZ\^_`ffhAE H
6&)!4 5B bffY hhCR ]EXEXEZ[ '3syy|+OQTUXU^U^_`UaQb*ccH
YYCQVV
L
xx{H ))Ef)-K%dC0K!+s3 7==?b Rwh7}}U# YC''%$C"3,ggeR( C2cU"XJa!8tH:UVEWX Y v I 66 6G AA & {"QuXJa@ @G A (G[ 00A $
%cU"QC0szzBT4##$s #K L#&L LLc | dkD rg S t j t t d }d|_ t |j d }|j d d
}|s t
d t j g S g }|j d
d D ] }|j d }|j d }|s)|j d s;|d j } t t | }
|j d } |r|j d nd} | rx| j } t! j" d| }
|
rPdt% |
j' d t% |
j' d t% |
j' d fz } | s|
s|j) | |
| f t
dt+ | dt j |S # t
$ r*}t
d| t j g cY d}~S d}~ww xY w) z|Extract items from #initData embedded in page 1.
Page 1 has all 27 items. Page 2+ not accessible (no working AJAX).
r r[ r\ r_ z Error fetching list: r` Nrb r5 initDatarp z No #initData found!ulzlist-ulrg r spanr Tr r rk rl rm rn z Found z items in embedded data)rq r$ LIST_URLrr rs rt ru rv rw r r, rx r; r r r rQ ry rz r{ r" r? )page_numr| r} rU init_divitemsr r r r rP ri r dmls r
extract_list_pager s !| LL7B ?
/Dyy:y.H
%CJJ7 E Y 7 5 GGCLwwv v V9??$D(D)CJJTJ*E48t}}4}0bH#>>+hh KXV /3syy|3Dc#))TU,FWY\]`]f]fgh]iYj2k kH eS( 34 5"
HSZL 7
8szzJ L;
's+#**= s 'G H H>H H c t j t d } | j d | j d | S )N< )r^ zPRAGMA journal_mode=WALzPRAGMA busy_timeout=5000)sqlite3connectDB_PATHexecute)conns r
init_dbr s1 ??7B /DLL*+LL+, Kr c
|r|n|}t j dd| j }|rt j |d nd} | j dt |||j |||t |t f
}|j dkD S # t $ r(} t d| t j Y d } ~ yd } ~ ww xY w)
Nz\s+r F) ensure_asciiz[]zINSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r z DB error: r` )rQ rJ r jsondumpsr SITE_NAMECATEGORYGROUProwcountrt ru rv rw )
r ri page_url publish_daterf r summaryatt_jsoncurr} s
r
save_articler s geGffVS'*002G>Itzz+E:tH ll
8 (EKKM<
gx5
:
||a
QC szz2s AB
B> B99B>c
t } t t j dkD r t t j d } t
dt t j t }d}d}t d| dz D ]: }t | }|s"t
d| dt j n |t | z
}|D ] \ }}}|r*|d k r%t
d
| d |d d t j 3|j d
|f } | j rWt | \ }
} } }
|
xs |}| xs |}| xs d| d| d}|
xs g }t |||||| r*|dz
}t
d| d |d d t j n!t
d|d d t j t! j" d = |j% |j' t
d| d| t j y # t
$ rb t j d dv rIt t j dkD r- t t j d } n# t
$ r Y nw xY wY )w xY w)Nr )z --max-pagesrm z Scraping r` r z Page z: 0 articles (end)z
2020-01-01z Skip old: r ( z)SELECT id FROM gov_raw WHERE page_url = ?r r r z + 2 z - skip dup: g333333?z
Done! Found: z, New: ) MAX_PAGESr? rv argvrz
ValueErrorru r rw r ranger r fetchoner r timesleepcommitclose) max_pagesr total_new total_foundpnarticlesri r dater det_titledet_date det_contentdet_atts final_title
final_date
final_content
final_attss r
mainr # sF I
388}q CHHQK(I
Ii[
!
3 9DIKAy1}% $R( GB4 12 D s8}$ %- !E8T|+ TF!E#2J<8szzJ ,, JXKXC||~9Eh9O 6Ixh#,uK!)TJ ' IQ{m2hZq+IM! RJD+x]T^_Q ZL+cr*:);<3::N{3B'7&89
K JJsO- @ KKMJJL OK= {
;#**M] xx{ ..3sxx=13D #CHHQK 0I! s5 G? ?:I*: II* I# I*"I##I*)I*__main__) __doc__rq rQ r r r osrv r bs4r urllib.parser r r r r BASE_URLr r rr r r/ rK rY r r r r r __name__ r r
r s 2 1 1 1
6 $ ;
DO0 8: ~ D1R'X(3Nl zF r