正文 + meta ArticleTitle/PubDate
- 附件: 相对路径 {hash}/files/xxx.pdf → urljoin(详情页URL) 绝对化
- total=2076, pageSize=15 → 138 页；CUTOFF 3年窗口
用法:
python3 crawl_santai_gsgg.py --pages=138 # 全量
python3 crawl_santai_gsgg.py --pages=1 # 测试
N)urljoin)push_to_searchdbu" 三台县人民政府-公示公告zhttps://www.santai.gov.cnz/stxrmzf/c111285/list.shtml 8027b3f0b2554c65aa1958ff5634aa37zU/common/search/{ch}?_isAgg=true&_isJson=true&_pageSize=15&_template=index&page={page}z
2023-08-10u 政府公开zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z.application/json, text/javascript, */*; q=0.01)z
User-AgentRefererAcceptc t j | xs d } t j dd| } | j dd j dd } | j S )N u ^(?:·| |\s|•|·)+u u )htmlunescaperesubreplacestrip)ts &/root/gov_crawler/crawl_santai_gsgg.py clean_titler . sP
ag2 A
12q9A (B ''"5A 779 c t j t | }t j |t
dd }|j dk7 rg dfS |j xs d|_ t j |j }|j di j d d }g }|j di j d
g xs g D ]r }t |j d d }|s!|j d
d }|s6t t | }|j d xs d d d } |j! ||| d t ||fS # t $ r g dfcY S w xY w)N)chpage Fheaderstimeoutverify r utf-8datatotalresultstitler urlpublishedTimeStr
)r! r" date)API_TPLformat
CHANNEL_IDrequestsgetHEADERS status_codeapparent_encodingencodingjsonloadstext Exceptionr r LIST_URLappend)
r r" rjr itemsrowr!
detail_urlpubs
r fetch_list_pager; 4 sI
..JT.
2C S'2eDA}}1u $$/AJ JJqvv
EE&" ! !'1 -E EuuVR $$Y39r GCGGGR 01 ggeR Xs+
ww)*0b#26
uZEF G %< 1u s " E
EEc " t j t dd }|j |j xs d|_ |j }|}t j d| }|r t |j d xs |}d}t j d| }|r|j d }g }t j d |t j D ] } | j d | j d
} }
|
j d r|
n t |
} t t j d d
| xs | j# d d } |j% | | d d
}
t j d|t j }|s%t j d|t j }|r|j d }
|
rt j dd
|
t j t j& z }
t j d fd|
}
t j dd
|
}
t j dd
|
}
t j dd|
}
t j dd
|
}
t j d d
|
}
t j d d
|
}
nd| d }
t) }|D ]8 }|d |vs |j+ |d |d |
vs'|
d |d d!|d" d#z
}
: |
||fS # t $ r dg dfcY S w xY w)$u, 返回 (content_html, attachments, pub_date)r Fr r Nz+
]+href="([^"]+\.(?:pdf|doc|docx|xls|xlsx|zip|rar|wps|7z))"[^>]*>(.*?) httpz<[^>]+>r /)namer" zT
]*class="article_content"[^>]*id="zoomcon"[^>]*>(.*?)