ujdZddlZddlZddlZddlZddlZddlmZmZmZddl Z ddl Z ejjdejjejjeddlmZe j$e j&j(dZdZdZdZej2ej4ed z j7d Zd d iZedzZdddZdZ dZ!dZ"dZ#dZ$dZ%dZ&dZ'e(dk(r8ejZ)e'e*deje)z ddyy)uq 丰城市人民政府 - 公示公告 (jxfc.gov.cn) 列表使用 JSON API 分页 URL: /fcsrmzf/gsgg/pc/list.html N)datetimetimezone timedelta)push_to_searchdbu"丰城市人民政府-公示公告zhttp://www.jxfc.gov.cnz/fcsrmzf/gsgg/pc/list.htmliG)daysz%Y-%m-%dz User-Agentz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36z/fcsrmzf/gsgg/pc/list)pageNumpageSizecr tj|tdd}d|_|jS#YyxYw)Nr F)headerstimeoutverifyzutf-8)requestsgetHEADERSencodingtext)urlrs "/root/gov_crawler/crawl_fc_gsgg.pyfetchrs5 LLgr% H vv s/26cn tj||tdd}|jS#YyxYw)Nr F)paramsrrr)rrrjson)api_urlrrs r fetch_jsonr!s3 LL"UZ [vvxs-04cg}|jdijd|jd|jdg}t|tr"|D]}|jd|jdd}|jd|jd|jdd}|jd |jd |jd |jd d}|s*|jd |jdd}|rd|d}|s|s|jds t|z}|rt |dk\r|dd}|j |j||f|S)u"从 JSON API 返回中提取文章datalisttitlenamerhreflinkUrl publishDate publish_date createDate create_dateid articleIdz/fcsrmzf/gsgg/pc/content_.htmlhttp N)r isinstancer! startswithBASE_URLlenappendstrip)r itemsrecordsrecr"r%date_str article_ids rparse_list_from_jsonr;(sM Ehhvr"&&vtxxQS@T/UVG'4  >CGGGSWWVR%89E775#''&#'')R2H"IJDww}cggncggl\_\c\cdqsu\vFw.xyH WWT377;+CD 6zl%HDv.#d?DH  3'}H ekkmT8<= > Lcg}gd}|D]4}tj||tjD]}|jdj }t |j dk\r|jdj nd}t |j dk\r |jdj }nd}|s%tjdd|j }|rt |dkr|jds t|z}|j|||f|s4|S|S) u#兜底: 从 HTML 页面提取文章)zD]*href="([^"]*gsgg[^"]*content[^"]*\.html)"[^>]*title="([^"]*)"zY]*href="([^"]*)"[^>]*target="_blank"[^>]*>(.*?)\s*]*>(\d{4}-\d{2}-\d{2})zY]*>.*?]*href="([^"]+)"[^>]*>(.*?).*?(?:]*>|\[)?(\d{4}-\d{2}-\d{2})r r$<[^>]+>r.) refinditerDOTALLgroupr5r3groupssubr1r2r4)htmlr6patternspatmr%r"r9s rparse_list_from_htmlrL>s EH S$ 2 2A771:##%D*-ahhj/Q*>AGGAJ$$&BE188:!#771:++-z2t4::<CJN??6*$ LL%x0 1 2   L#" Lr<c>|dk(r ttzStd|dS)Nr z/fcsrmzf/gsgg/pc/list_r-)r2 LIST_PATH)pages r get_page_urlrPYs( qy)##Z-dV5 99r<c.tt}||d<|S)Nr )dict API_PARAMS)rOrs rget_api_paramsrT^s * FF9 Mr<ct|}|syi}tjd|tj}|r7tjdd|j dj |d<|jds^tjd|tj}|r7tjdd|j dj |d<tjd|}|r|j d|d <|jd s,tjd |}|r|j d|d <|jd s,tjd |}|r|j d|d <d}d D]n}tjd |zdz|tj}|s1t|j ddkDsO|j dj }n|rt|dkrFtjd|tj}|r|j dj }|rutjdd|tjtjz}tjdd|tjtjz}||d<|jd|jd|jd fS)N)NNNz]*>(.*?)r@r$r r"z(.*?)u(发布日期[：:]\s*(\d{4}-\d{2}-\d{2})r(u(发布时间[：:]\s*(\d{4}-\d{2}-\d{2})z(\d{4}-\d{2}-\d{2})) TRS_Editorcontentzarticle-contentarticle_contentz xxgk-contentCustom_UnionStylecon_textz main-contentz info-contentrz]*class="[^"]*z0[^"]*"[^>]*>(.*?)\s*(?:|