V_je d Z ddlZddlZddlZddlZddlZddlmZ ddlm Z dZ
dZ dZ dZ
d Zd
Zd d d
ddZdZdZd Zd dZd Zd Zd Zd ZdefdZedk( rxdej6 v Z e rd neZ d ej6 v rKej6 j= d Z e d z e ej6 k r e!ej6 e d z Z ee e yy)!u
大连市生态环境局 - 清洁生产审核
https://epb.dl.gov.cn/col/col4737/index.html
TRS/Hanweb CMS - AJAX pagination via dataproxy.jsp
N)
BeautifulSoup)urljoinz/root/search.dbu+ 大连市生态环境局-清洁生产审核u 大连z,https://epb.dl.gov.cn/col/col4737/index.html zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8https://epb.dl.gov.cn/)z
User-AgentAcceptzAccept-LanguageRefererzINSERT OR IGNORE INTO gov_raw
(site_name, page_url, title, publish_date, summary, content, category, attachments)
VALUES (?, ?, ?, ?, '', ?, ?, ?)zhttps://epb.dl.gov.cn/module/web/jpage/dataproxy.jsp?page=1&webid=33&path=https://epb.dl.gov.cn/&columnid=4737&unitid=32863&webname=%E5%A4%A7%E8%BF%9E%E5%B8%82%E7%94%9F%E6%80%81%E7%8E%AF%E5%A2%83%E5%B1%80&permissiontype=0c | dd l } | j t }|j d |j d |S )Nr zPRAGMA journal_mode=WALzPRAGMA busy_timeout=5000)sqlite3connectDB_PATHexecute)r conns !/root/gov_crawler/crawl_epb_dl.pyinit_dbr # s2 ??7 #DLL*+LL+, K c X |xs t }t j | || }d|_ |S )Nheaderstimeoututf-8)HEADERSrequestsgetencoding)urlr r hrs r fetchr + s* 7A S!W5AAJ Hr c t | S )u' 保留 HTML 表格结构（不转 md）)str)tableargskwargss r table_to_markdownr% 2 s u:r c t | d }g }t |j D ]9 }|j |j j }|dk( r(|j dd }|sF|j
| X|dk( r t | }|sk|j
| }|dv r|j dd D ]) }|j dd }|s|j
| + |j dd D ]! }t | }|s|j
| # |j dd } |d
v r|j dd }|s)|j
| < d j | S ) z6Parse article HTML content (between ContentStart/End). html.parserp Tstripr" )divcentersectionF) recursive)scriptstylemetabrz
)
r listchildrennamelowerget_textappendr% find_alljoin)
textsouppartschildtntxtmdsub_psub_t remainings
r extract_contentrF 6 sf } -D E dmm$ " ::
ZZ
9..D.1C S!
7]"5)B R
/
/ u =
&nnSn5LL%
& 5 A
%&u-LL$
% s$7I 4
4 ..D.1C S!? "B ;;u r c t j t t d } d| _ | j dk7 r t
d | j d g S t j d
| j t j }g }|D ] }t j d | }|s |j d }|j d
j }t d| }t j d| } | r| j d nd}
|j! |||
d |S # t
$ r }t
d| d g cY d}~S d}~ww xY w)z3Get all list items from the proxy (returns all 89). r r z [ERROR] Proxy: TflushN z [ERROR] Proxy HTTP z&z,]*href="([^"]*)"[^>]*title="([^"]*)" r z\[(\d{4}-\d{2}-\d{2})\] )titler date)r r PROXY_URLr r Exceptionprint status_coderefindallr< DOTALLsearchgroupr+ r r9 ) r erecordsitemsreca_matchhrefrO full_urldate_mdate_strs r
get_all_itemsrc ` s( LLGR @
}}
%amm_5TB jjBAFFBIIVG E J)) KSQ }}Q
a &&(3T:5s;&,6<<?"
uXxHI J L%
!!%T2 s 'D# # E,E=EEc t | }|j dk7 ryt |j
d }|j
}d}|j
d d
d i }|r$|j d
r|d
j }d}|j
d d
di }|r$|j d
r|d
j }|s;|j
d d
di }|r$|j d
r|d
j }d} |j
d }
|j
d } |
dkD r*| dkD r%|j
d|
} || dz | }
t |
} g }|j d }|r|j dd D ]u }|d }t j d|t j s.|j d xs |j! d d }t# | | }|j% ||d w t' | j d k r d | d | d!} || ||rt) j* |d"# d$S dd$S # t $ r }t d| d| d Y d}~yd}~ww xY w)%z+Fetch detail page and extract full content.z
[ERROR] z: TrI NrK r' rN r2 r6 ArticleTitle)attrscontentPubDatepubdate ContentStart
ContentEndr >rL z div.art_cona)r_ r_ z"\.(pdf|doc|docx|xls|xlsx|rar|zip)$r* /)r6 r z u!
（本文为PDF附件）F) ensure_ascii)rO rg pub_date attachments)r rR rS rT r r< findr r+ rF
select_oner: rU rX
IGNORECASEr8 splitr r9 lenjsondumps)r r rZ r= htmlrO mtrr rB content_textcsceafterrawattachments_listart_cona_tagr atfus r extract_detailr z ss #J }} /D 66D
E
6&.!9 :B bffY 9
##% H
6&)!4 5B bffY i=&&(
YYvfi%8Y
9
"&&#)}**,H L
> "B
< B Av"q& #r"519R &s+ oom,G %%c%5 AEf
Ayy>2==Q^^$^/C1773<3C S!_ ''B(?@ A < 2%%cU"UG3UV K[tzz"2G bd g
3%r!%T2s H0 0 I9IIFc ^ t dt d| d t }|j }t }t dt | d |s t dd |j
y|t z }|d | }t dt | d | d
d dx}}|D ] } | d | d | d
} } }
|j d| f |j r|dz
};t | }
|
|dz
}N|
d r t |
d t |
kD r|
d }
|
d }|
d }|j t t | |
| xs |
d |t |f |dz
}| r|dk\ r nt j d |j |j
t dt d| d| d |S )N[z] Starting, max_pages=TrI z Total items from proxy: z [ERROR] No items foundr z Taking first z items (z pages)rO r rP z)SELECT id FROM gov_raw WHERE page_url = ?rL rg rs rr
g333333?z] Done. Inserted=z
, Skipped=)rS SITE_NAMEr cursorrc rx closePER_PAGEr fetchoner
INSERT_SQLCATEGORYtimesleepcommit) test_mode max_pagesr cur all_items max_itemsr\ total_inserted
total_skippeditemrO r rP detailrg rs s r crawlr s Ai[.yk
:$G 9D
++-C I &s9~&6
7tD
(5
H$I jy !E OCJ Q M $ > Q M '?s6'? 3c%j @7OE# ]+ J sE4#=6*+= X{!
! 2-
358 KKMJJL Ai[).)9M?
S[_` r __main__z--test z --max-pagesrL )r r )rH N)"__doc__ry rU sysr r bs4r urllib.parser r
r r BASE_URL MAX_PAGESr r r rQ r r r% rF rc r r __name__argvr mpindexidxrx int r r r s
% $ $
9 9
DO0' (
!
' T4<~ Y 2j zCHH$IYB hhnn]+ 7S] "SXXcAg&'B I, r