ujdZddlZddlZddlZddlZddlZddlmZmZmZddl Z ddl Z ejjdejjejjeddlmZe j$e j&j(dZdZdZdZej2ej4ed z j7d Zd d iZedzZdddZdZ dZ!dZ"dZ#dZ$dZ%dZ&dZ'e(dk(r8ejZ)e'e*deje)z ddyy)uq 丰城市人民政府 - 行政执法 (jxfc.gov.cn) 列表使用 JSON API 分页 URL: /fcsrmzf/xzhy/pc/list.html N)datetimetimezone timedelta)push_to_searchdbu"丰城市人民政府-行政执法zhttp://www.jxfc.gov.cnz/fcsrmzf/xzhy/pc/list.htmliG)daysz%Y-%m-%dz User-Agentz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36z/fcsrmzf/xzhy/pc/list)pageNumpageSizecr tj|tdd}d|_|jS#YyxYw)Nr F)headerstimeoutverifyzutf-8)requestsgetHEADERSencodingtext)urlrs "/root/gov_crawler/crawl_fc_xzhy.pyfetchrs5 LLgr% H vv s/26cn tj||tdd}|jS#YyxYw)Nr F)paramsrrr)rrrjson)api_urlrrs r fetch_jsonr s3 LL"UZ [vvxs-04c g}|jdijd|jd|jdg}t|tr|D] }|jd|jdd}|jd|jd|jdd}|jd |jd |jd d}|s*|jd |jd d}|rd|d}|s|s|jds t|z}|rt |dk\r|dd}|j |j||f|S)NdatalisttitlenamerhreflinkUrl publishDate publish_date createDateid articleIdz/fcsrmzf/xzhy/pc/content_.htmlhttp )r isinstancer! startswithBASE_URLlenappendstrip)r itemsrecordsrecr"r%date_str article_ids rparse_list_from_jsonr:'s> Ehhvr"&&vtxxQS@T/UVG'4  >CGGGSWWVR%89E775#''&#'')R2H"IJDww}cggncggl\^F_.`aH WWT377;+CD 6zl%HDv.#d?DH  3'}H ekkmT8<= > Lcg}gd}|D]3}tj||tjD]}|jdj }t |j dk\r|jdj nd}t |j dk\r|jdj nd}|s%tjdd|j }|rt |dkr|jds t|z}|j|||f|s3|S|S) N)zD]*href="([^"]*xzhy[^"]*content[^"]*\.html)"[^>]*title="([^"]*)"zY]*href="([^"]*)"[^>]*target="_blank"[^>]*>(.*?)\s*]*>(\d{4}-\d{2}-\d{2})zY]*>.*?]*href="([^"]+)"[^>]*>(.*?).*?(?:]*>|\[)?(\d{4}-\d{2}-\d{2})r r$<[^>]+>r-) refinditerDOTALLgroupr4r2groupssubr0r1r3)htmlr5patternspatmr%r"r8s rparse_list_from_htmlrK;s EH  S$ 2 2A771:##%D*-ahhj/Q*>AGGAJ$$&BE-0_-Aqwwqz'')rHz2t4::<CJN??6*$ LL%x0 1 2   L  Lr;c>|dk(r ttzStd|dS)Nr z/fcsrmzf/xzhy/pc/list_r,)r1 LIST_PATH)pages r get_page_urlrORs( qy)##Z-dV5 99r;c.tt}||d<|S)Nr )dict API_PARAMS)rNrs rget_api_paramsrSWs * FF9 Mr;ct|}|syi}tjd|tj}|r7tjdd|j dj |d<|jds^tjd|tj}|r7tjdd|j dj |d<tjd|}|r|j d|d <|jd s,tjd |}|r|j d|d <|jd s,tjd |}|r|j d|d <d}d D]n}tjd |zdz|tj}|s1t|j ddkDsO|j dj }n|rt|dkrFtjd|tj}|r|j dj }|rutjdd|tjtjz}tjdd|tjtjz}||d<|jd|jd|jd fS)N)NNNz]*>(.*?)r?r$r r"z(.*?)u(发布日期[：:]\s*(\d{4}-\d{2}-\d{2})r(u(发布时间[：:]\s*(\d{4}-\d{2}-\d{2})z(\d{4}-\d{2}-\d{2})) TRS_Editorcontentzarticle-contentarticle_contentz xxgk-contentCustom_UnionStylecon_textz main-contentz info-contentrz]*class="[^"]*z0[^"]*"[^>]*>(.*?)\s*(?:|