uj d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
ddl Z dZ e dz Z
dZdZd Zd
Zd d d
e dz ez dZd ZddZd Zd Zd Zedk( r e yy)u:
康定市人民政府(www.kangding.gov.cn) - 公示公告栏目 (column 1840)
URL: https://www.kangding.gov.cn/wzlb/c/1840 (旧入口 /kdsrmzf/c100353/common_list.shtml 301 至此)
CMS: 自建 Vue.js SPA + SSR
列表: POST /site-service/c/manuscript/getManuscriptListByColumnId
参数 {"columnId":"1840","current":N,"size":15,"type":false} -> data.total/data.records[]
1667 条, 112 页, 15条/页
详情: https://www.kangding.gov.cn/gsgg/article/{id} (SSR HTML)
标题 div.txt-title, 日期 span[title], 正文 div#contentBox
生产库 schema: gov_raw(id, site_name, source_url, page_url, title, publish_date, date_rank,
summary, status, category, visits, content, tags, industry, attachments, group_name, has_table, script_name)
gov_search: fts5(title, site_name, summary, tokenize=trigram) rowid 对齐 gov_raw.id
N)unescape)urljoinzhttps://www.kangding.gov.cnz6/site-service/c/manuscript/getManuscriptListByColumnId1840u 康定市-公示公告zcrawl_kangding.pyu 四川zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z!application/json, text/plain, */*zapplication/json/gsgg/c/)z
User-AgentAccept Content-TypeRefererc t | xs d } t j dd| } t j dd| } t j dd| j } | S )uD 清洗标题: strip · 实体前缀 和站点后缀 u ^[\s\xa0·\u00b7]+z
\s*\.{3,}\s*$z\s+ )r resubstrip)ts #/root/gov_crawler/crawl_kangding.py clean_titler * sV bA
$b!,A
Q'A
vsA $$&A H c 6 t ||dd}| j t |t dd }|j dk7 ry|j }|j
d dk7 ry|j
d xs i }|j
d
xs g |j
d d fS ) u& POST 列表 API, 返回 records 列表F)columnIdcurrentsizetype )jsonheaderstimeoutverify )Nr coder datarecordstotal) COLUMN_IDpostLIST_APIHEADERS status_coder get)sessionr r payloadrr ds r
fetch_listr- 3 s $$PUVG XGWbQV WA}} 668D xx1 BA 55 !r155!#4 44r c d}t j d| t j }|r/t t j dd|j d }d}t j d| }|r|j d }t j d| t j }|s%t j d| t j }|s||dg fS |j d }g }t j d|t j D ] } | j d t j dd| j d j } }
t j d
|
t j s| j d s|j t ||
| xs |
j d d
f ||||fS )u@ 解析详情页, 返回 (title, date, content_html, attachments)r z']*>(.*?)
z<[^>]+> z$]*title="(\d{4}-\d{2}-\d{2})z.]*>(.*?)
\s*z ]*>(.*)z$
]+href="([^"]+)"[^>]*>(.*?) z0\.(?:pdf|docx?|xlsx?|zip|rar|wps|et|dps)(?:\?|$)u 附件/)
r
searchSr r groupfinditerr I
startswithappendr split) html_texturltitlemdatedmcmbodyattsahreftxts r parse_detailrG @ st
E
?
D :I FB xx{ DiQSQUQU VB
YY8)RTT
J
dB""
88A;D
D
[[@$
M JGGAJz2qwwqz B H H Jc
99H$PRPTPT UY\YgYghpYq KKd+S-GDJJsOB4GH IJ $d ""r c g fd}t j d|| } t j dd| } t j dd| } | j dd j d d } t j d
d | } | j d D cg c] }|j }}|D cg c] }|s| }}d j | }t
D ] \ }}|j d | d
| } t j d
d | }|D ] \ } }
| |vs |d| d|
dz
} |j S c c}w c c}w )u9 HTML -> 文本, 保留表格、\n\n 分段、附件嵌入c h j | j d dt dz
dS )Nr @@TABLEr/ @@)r9 r5 len)r> tabless r _save_tblz html_to_text.._save_tbl` s/
aggaj!Vq ),,r zz"]*>
z.]*>r z r z\n{3,}
rJ rK z
z
)r
r replacer: r join enumerate) r; attachmentsrN lnlinesouttextir rE rF rM s @r html_to_textr[ \ sN
F- 0)YGIse8L LDM ::<
9
$s E7E?Ec ^ t j } | j dt dd | j ddd | j d t
| j }|j }|j }|j d s t j j | }t j | }|j }d }d }d }t j } | j# t$ d
z t& z t( dd t- | dd \ }
} | r| dz dz nd} t/ |j0 | }
t3 d| d| d|
d t5 d|
dz D ]W }t- | |d \ }}
|t3 d| d |dz
},t3 d| dt7 | d |sJ|D ] } |j# d }t9 |j# d d }|j# d xs |j# d xs d d d! }|r|sat$ d"| }|j; d#||f |j= d d kD r|dz
}| j# |i t( d$d%idd }d&|_ tA |jB | \ }}}}|jE st3 d'|d d |dz
}|r|}|r|}tG || }|rd(jI d) |D nd }tK jL ||d d* z jO d& jQ } |r|d d* nd } |j; d+tR ||||d | d d d |d d,|tT d-|v rdnd |f |j; d.|jV |tR | f |jY |dz
}t3 d/| d0|d d1 t[ j\ d4 Z |j_ t3 d5| d6| d7| y # t* $ r Y w xY w# t* $ r$} |dz
}t3 d2d d d3| Y d } ~ ld } ~ ww xY w)8Nz--pagesr/ u 抓取页数)r defaulthelpz--dbz/mnt/data/search.dbu SQLite 数据库路径)r] r^ z
--script-name)r] z/mnt/r r r F)r r r )r r u 共 u 条 / u 页, 本次爬 u 页[u ] 列表请求失败u ] 列表 u 条idr= r releaseTimetime
z/gsgg/article/z?SELECT COUNT(*) FROM gov_raw WHERE page_url=? AND script_name=?r z text/htmlzutf-8u 跳过空正文: rQ c 3 & K | ] \ }}| y w)N ).0h_s r zmain.. s %9DAqa%9s i zINSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, status, category, visits, content, tags, industry, attachments, group_name, has_table, script_name) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)otherz!$q( NI [[2ZW2Znk2Zdfot[u % 6B277G6T3 t#))+ /cr
|<= NI #E %H&|T:=A&++%9D%99r kk574C=#8"@"@"IJTTV+2'$3- A %1grSUWXZace gzG8K1QRT_a `]]E9g>
Q XJacr
|45O* 2V
3i4l JJL OI;j :i[
QRA
t
2Q uSbzl"QC011
2s? 1'O/ AO? :O?A'O? C/O?/ O<;O<? P,P''P,__main__)r/ r_ )__doc__rn r rv r
ry rd htmlr urllib.parser r| r~ r% r# r rr r r& r r- rG r[ r __name__rg r r r s ( N N $ !
@1&*$y0
5#84VSr zF r