~j"dZddlZddlZddlZddlZddlZddlmZddl Z ddl Z dZ dZ dZ dZdZd Zd Zej$j'd d Zd e dZdZdZddZdZdZdZdZddZedk(rkddlZdZ ejBddD]IZ"e"jGdr e$e"jKdddZ 1e"jOsBe$e"Z Kee yy#e&$rY^wxYw)u 怀化市洪江区 - 通知公告 站点: hjq.huaihua.gov.cn/hjq/c115285/list.shtml CMS: 湖南 TRS (createPageHTML 分页 list_N.shtml) 列表: a[title=完整标题] href=202608/uuid.shtml (日期在 URL 202608) 详情: meta ArticleTitle/PubDate + div#fontzoom 正文 HTML 分页: createPageHTML('paging',25,1,'list','shtml',500) -> list_N.shtml 用法: python3 crawl_hjq_huaihua_tzgg.py [--pages=N | N] N)urljoinu洪江区-通知公告zhttp://hjq.huaihua.gov.cnz0http://hjq.huaihua.gov.cn/hjq/c115285/list.shtmlz&http://hjq.huaihua.gov.cn/hjq/c115285/zhjq.huaihua.gov.cnu湖南z 2023-01-01DB_PATHz/root/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36)z User-AgentRefererctjdd|xsd}tjdd|}tjdd|jS)Nz$·| | |\u200b|\ufeffu ^[•·]\s*z\s+ )resubstrip)ts +/root/gov_crawler/crawl_hjq_huaihua_tzgg.py clean_titlersG 6AGDA ~r1%A 66&#q ! ' ' ))ctjd|xsd}|r[t|jdt|jdt|jd}}}|dd|dd|dSy) Nu-(\d{4})[年\-/. ](\d{1,2})[月\-/. ](\d{1,2})r04d-02dr searchintgroup)smymods r normalize_dater #sk BAGLAqwwqz?C OS_q2C"S1S'** rct|D]D} tj|dtd}d|_|j dk(r|j cSFy#t$r!||dz krddl}|jdYqwxYw) NF)timeoutheadersverifyzutf-8rrr) rangerequestsgetHEADERSencoding status_codetext Exceptiontimesleep)urlretriesirespr/s r fetchr5*s 7^+ +<<ROD#DM3&yy ' +  +7Q;ZTZZ] +s>A'A>=A>cptjd|xsd}|rt|jdSy)NzKcreatePageHTML\s*\([^,]*,\s*\d+,\s*\d+,\s*'(\w+)'\s*,\s*'(\w+)'\s*,\s*(\d+)rrrr) html_textrs r get_page_countr86s2 `bkbqoqrA1771: rc g}|s|St}tjd|tjD],}|j dj |j dj }}t |xs/t tjdd|j d}t|dkr||vr|j|tjd|stjd |}d} |r$|j d} | ddd | dd d | d d } |jd s t||n|} |j|| | |d/|S)Nz8]*href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)rr<[^>]+>rrz/c115285/\d{6}/z /(\d{6})/rhttp)titler1pub_datehref_raw)setr finditerDOTALLrr rr lenaddr startswithrappend) r7base_urlitemsseenrhref title_attrr?dmr@ymd detail_urls r extract_list_itemsrQ<sL E  5D [[C299b771:++-qwwqz/?/?/AjJ'Z;rvvj"aggVWj7Y+Z u:>  4<  yy+T2  YY|T * ((1+Cbq'!C!H:Qs1Qxj9H48OOF4KWXt,QU  uZX[_`a%b& Lrc Rt|}|syd}tjd|}|rt|j d}|sdtjd|tj }|r=ttj dd|j dj}d}tjd|}|rt|j d}dtjd|tj }|s%tjd|tj }|r`|j d}tjd |}|r|d|jd zn |jd }|d k7r|d|d zn|tj d dtjtjztj ddtjtjzg}tjdD]}|j djt|j d} } tjd| tjsb| jdr| n)| jdr t| znt!t"| t%fd|Dr|j'| xs| j)dd dd} tj d| tj d| tj ddtjtjztj ddtj dd|rDt%fd|D} | s.dj+d|D} j-d z| zt/tj d!dj}d"v}d#vxsd$v}|d%kr |s |s |s|d|gd&S|||d&S)'Nrz/(.*?)u [-_—]\s*.*$z*]+id="fontzoom"[^>]*>(.*)z]+id="conTxt"[^>]*>(.*)zs\s*