j2 d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
dZ dZ dZ
dZd Zd
d d dd
ZdZh dZd Zd Zd Zd Zd ZdefdZedk( rxdej4 v ZerdneZ dej4 v rKej4 j; d Z e dz e ej4 k r e ej4 e dz Z eee yy) u
怀宁县人民政府 - 建设项目环评文件（报告书、报告表）审批
https://www.ahhn.gov.cn/grassroots/column/2000000951?catId=1002621
蓝汛(Lonsun) CMS - 基层政务公开目录
N)
BeautifulSoup)urljoinz/root/search.dbu( 怀宁县-建设项目环评文件审批u 安徽zBhttps://www.ahhn.gov.cn/grassroots/column/2000000951?catId=1002621 zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentAcceptzAccept-LanguageRefererz
INSERT OR IGNORE INTO gov_raw
(site_name, page_url, title, publish_date, summary, content, category, attachments)
VALUES (?, ?, ?, ?, '', ?, ?, ?)
>) pdddldth1h2h3h4h5h6hrlioltdthtruldivnavpreformmainasidetabletbodytfoottheadcenterdialogfigurefooterheaderhgroupaddressarticlecaptiondetailssectionfieldset
blockquote
figcaptionc | yddl }ddlm g fd | dj }|j dd| }|j dd | }|j S )
ul 块级边界出换行、行内节点直接拼接、
出换行（≈ 浏览器看到的换行结构）。N r )NavigableStringc F | j D ] }t | rj t | *t |dd dk( rj d Lt |dd t
v r+j d | j d | y )Nnamebr
)children
isinstanceappendstrgetattr _BLOCK_TAGS)nodechr3 outwalks "/root/gov_crawler/crawl_ahhn_hn.pyrA zbody_text..walk7 s --
B"o.
3r7#VT*d2
4 VT*k9
4 R
4 R
z[ \t\r\f\v]*\n[ \t\r\f\v]*r7 z\n{3,}z
)rebs4r3 joinsubstrip)el_retr3 r@ rA s @@@rB body_textrL / sb z#
C H
A -tQ7A 61%A 779rC c | dd l } | j t }|j d |j d |S )Nr zPRAGMA journal_mode=WALzPRAGMA busy_timeout=5000)sqlite3connectDB_PATHexecute)rN conns rB init_dbrS J s2 ??7 #DLL*+LL+, KrC c , | y| j d S )z,Extract text from an element, handling None.r2 TrH )get_text)elems rB safe_textrX R s | ==t= $$rC c b | dk( rt }n t d| } t j |t d }d|_ |j d k7 r t
d |j d
| d g S t |j d }g }|j d D ] }|j dd }|s|j dd j }|s|j d }|j dd j } | r| j d rt t | }
|j dd } t! | j } |j# ||
| d t
d| dt% | dd |S # t
$ r }t
d| d| d g cY d
}~S d
}~ww xY w) z s.|)j- d% xs |*j# d5 d6 }+tA | |* },|(j1 |+|,d7 w ||||(rtC jD |(d89 d:S d
d:S # t $ r }t d| d| d Y d}~yd}~ww xY wc c}!w );z-Fetch a detail page and extract full content.r[ r\ r_ z [ERROR] Failed to fetch r` Tra Nrc rd z for re r2 metar5 PubDate)attrscontentz%div.j-fontContent.newscontnet.minh500zdiv.xxgk_contentzdiv.content_mainr z
__TABLE_z__
rZ r7 r __TABLE___r r r u | 项 目 | 内 容 |z|------|------| rU rowspanz| z | z |z| | ( -z|-z-|-z-|rf )ri ri z"\.(pdf|doc|docx|xls|xlsx|rar|zip)$/)r5 rm F) ensure_ascii)r content_htmlpub_date attachments)#rp rq rr rs rt ru rv r rw ry rH
select_oner; find_all enumeratereplacerL splitrz endswithr{ maxrV intr: rF minrangeljustrD search
IGNORECASEr jsondumps)-rm r} r~ r r meta_date content_divr content_textcontent_html_str
all_tablestir marker
clean_soup
clean_textpartssegments table_idxsegrowsmax_colsrowcells table_lineslabelvaluer num_cols
col_widthsicell cell_textwri
cell_textscitxtsepsep_liner r ri att_titler s- rB extract_detailr s LLgr :
}}
eC59F /D H &(; "C))+C~~j)cll4.@ Q J' &rNE >>$/D#$#' AC$'LL$ $>E'*8SZ'@HA $q=+BDU*VK'+ M(+ dD\(B#&u:?,1!H,=,=c,=,NE,1!H,=,=c,=,NE',25eAhll9a6P2Q+2Q;,7,>,>E7#eWTV?W,X,7,>,>E7#eWTV?W,X(3(:(:T%;K(L M + % TDIIk4J-JT-Q R (0H*+x J'+ W(+ dD\(B/8/? !WGAt04
c
0NI47
1
s9~4VJqM!W W
KU)UQ#aQUB *@)UJ)U*,K+4T? [C(+ dD\(B-/
*// !QBQSVYZ_V`Q`%)*<*R7RUY7Y$Z [ + % TDIIk4J-JT-Q R S!}> "@ yy' K9D%%c%5" EV}
99:D"-- P T2Idjjob6II sD)H ( C D E $ FQtzz+EB XZ u
*3%r!5TBv *Vs #V ' V* V'
V""V'Fc H t dt d| d t }|j }d}d}d}t d|dz D ]* }t | }|s|dkD rt d| dd n+|dz
}|D ] } | d }
| d
} | d } |j
d | f |j r|dz
};t | }
|
|dz
}N|
d
}t |j dk r d| d|
d}|
d }|j
t t | |
| xs |
d |t |f |dz
}| s|dk\ s n |j | r|dk\ rt d| dd nt j d - |j! t dt d| d| d| d |S )zMain crawl function.[z ] Starting crawl, max_pages=Tra r rZ rn z empty, stoppingrg rm rl z)SELECT id FROM gov_raw WHERE page_url = ?r z u!
（本文为PDF附件）r r
z Test mode: stopping after z insertsz] Done. Inserted=z
, Skipped=z, Pages=)ru SITE_NAMErS cursorr r rQ fetchoner r{ rH
INSERT_SQLCATEGORYcommittimesleepclose) test_mode max_pagesrR curtotal_inserted
total_skipped
pages_crawledr| r itemrg rm rl detailr r s rB crawlr s Ai[4YK
@M 9D
++-CNMM!Y]+ 2!(+!|z)9:$G
DMEu+Cr s
6 O DO0S
* 6%/dDN Y @F zCHH$IYB hhnn]+ 7S] "SXXcAg&'B I, rC