uj/ dZddlZddlZddlZddlZddlZddlZddlmZmZddl m Z ddl m Z ejddZdZd Zd Zd Zed Zd dddddddddddd ZddddZdZdZdZd Zd!Zd"Zd.d#Zd/d$Zd%Zed&k(rqe ejBd'kDrejBd'nd(Z"e"d(k(reye"d)vr ed*+ye"dk(r*ed'Z#e#re#jId,dndZ%ed-e%yyy)0u Crawler: 芜湖市弋江区生态环境分局 - 行政审批 (Epoint 基层政务公开, catId=1141240) Fix: content is in yjq.gov.cn page (j-fontContent gkwz_contnet), preserving tables as HTML. N)datetime timedelta) BeautifulSoup)urljoin SEARCH_DBz/root/search.dbu'弋江区生态环境分局行政审批u 行政审批zhttps://www.yjq.gov.cnzhttps://sthjj.wuhu.gov.cnz/wuhu/site/label/8888grassrootsContentPageList67878916603431151truez yyyy-MM-dd504list1141240) labelNamesiteIdorganIdpageSize pageIndexisDate dateFormatlengthtypeactionisJsoncatIdzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z!application/json, text/plain, */*z0application/x-www-form-urlencoded; charset=UTF-8)z User-AgentAcceptz Content-Typec2tdtd|dy)N[z] T)flush)printSITE)msgs #/root/gov_crawler/crawl_yjq_xzsp.pylogr&'s AdV2cU 4(citdt|i} tjt|t dd}|j S#t$r}td|Yd}~yd}~wwxYw)NrF)dataheaderstimeoutverifyz API error: ) API_DATAstrrequestspostAPI_URLHEADERSjson Exceptionr&) page_indexr*res r%fetch_list_pager9+s` 5h 5 S_ 5D MM'grRW Xvvx  k! s1A A*A%%A*c tj|tdd}d|_|j|fS#t $r}t d|d|d|fcYd}~Sd}~wwxYw)Nr)F)r+r,r-zutf-8zFetch error for : )r0getr3encodingtextr5r&) detail_urlr7r8s r% fetch_detailr@5sb LLWb O vvz!!   zl"QC 01Z s14 AAAAcpt|d}|jdd}|rtt|Sy)z\Extract content from j-fontContent div. Works for both gkwz_contnet and newscontnet minh500. html.parserdivc.|xrd|jvS)Nz j-fontContent)split)cs r%z&get_content_by_class..CsA4V/QWWY:Vr')class_)rfind_clean_contentr/)htmlsoup content_divs r%get_content_by_classrO?s7 } -D))E*V)WKc+.// r'c|syt|d}|jddD]R}|jdd}|jdxsd}t t |}|j d |d |d T|jgd D]}|jg}|jd dgD]u}|jd k(r|jt|-|jdk(s=|jd rO|jdd} | se|j| w|rdj|S|jS)zhConvert content HTML to text. Preserve tables as HTML in original position. separator='' for paragraphs.rIrBaT)hrefrR)stripu附件r z]())spanbstrongfontemiustablep) separatorrSz )rfind_allr<get_textrBASE_URL replace_withunwrapnameappendr/ find_parentjoinrS) content_htmlrMa_tagrRr>full_urltagpartselts r%rKrKIs:  } 5Ds.4yy$~~D~)5X8T* QtfBxj23 4}}QR  EmmWcN+  77g  LLR ! WW^BNN7$; b 5A Q  "'6;;u @L,>,>,@@r'cdddd}tjd|}|rn|jdj}tjdd|j}tjdd|j}||d<tjd|}|r%|jdjd d |d <t ||d <|S) z.Extract title, date, content from detail page.rItitle publish_datecontentz(.*?)u'[_\-—]\s*芜湖市生态环境局\s*$u*[_\-—]\s*芜湖市政务公开平台\s*$rrzPubDate.*?content="([^"]+)"N rsrt)researchgrouprSsubrO)rLresultmrrs r% parse_detailr}es2" =F )40A   "A2uMSSUDb%PVVXw 0$7A!"!1!1!3CR!8~,T2F9 Mr'c tjt|dzz jd}t d|dt d}|sy|j dd}ttd }||zdz |z}t d |d |g}tdt|dzd D]}t |}|sn|j d g}|D]} | j dxsddd} | |kr!| j ddxs#| j ddxstd| dd} |j| d| j dd| | dt d|dt|dtjdt dt|dg} t!|dD]\} } t#| d\}}|r.t%|}|dr|d| d<|dr|d| d<|d | d <nd| d <| j| d| d| d| j d dt&d!t(| dd"| dzdk(rt d#| d$t|tjdd}d}ddl}|j-t.}|j1d%|j1d&|j3}| D]W} |j1d'| d(| d)| d)| d| d| d | d xsddd*t(f|j4dkDr|dz }n|dz }Y|j9|j;t d,|d-|d.y#t6$r}t d+||dz }Yd}~d}~wwxYw)/Nr)days%Y-%m-%dzFull crawl (cutoff: rTrutotalrrTotal: z , pages: ir* publishDaterIrv urlDomainlink/openness/grassroots/ contentId.htmlrr content_idrrdateurl Page r; items?zAfter filter: rrsrrt-rrpage_urlrsrt site_name source_url Detail: /PRAGMA journal_mode=WALPRAGMA busy_timeout=10000INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category) VALUES (?,?,?,?,?,?,?,?)rr DB error: Result:  new,  skipped)rnowrstrftimer&r9r<intr.rangeminrbrflentimesleep enumerater@r}r#COLUMNsqlite3connectDB_PATHexecutecursorrowcountr5commitclose) months_backcutoffr*r page_size total_pages all_itemspgitemsitempub_dater?enrichedidxrL_detailstoredskippedrconnrFr8s r% crawl_allrusllnykB.>?? I I* UF vha() 1 D  HHWa EHZ()I9$q(Y6K'% +/0IAs;?C01r" $ D/52s;H& +r2Idhhvr6JIQYPZZoptvAqBpCCHOIJ   ";/'2. !    gbTCJ"23 4 3-0FG ??7 #DLL*+LL,- A  II,k"D$4d:6Fg^ 4d9oy/'R#.8  9zzA~! 1   KKMJJL(6(& 23   *QC ! qLG sAM M7M22M7c td|dtd}|sy|jdd}ttd}t ||zdz |z|}td|d |d g}t d|dzD]}t|}|sn|jd g}|D]z}|jd d xs#|jdd xstd|dd} |jdxsd dd} |j|d|jdd | | d|td|dt|dtjdtdt|g} t|dD]\} }t|d\} }| r.t| }|dr|d|d<|dr|d|d<|d|d<nd |d<| j|d|d|d|jdd tdt |dd | dzdk(rtd!| d"t|tjdd}d}ddl}|j%t&}|j)d#|j)d$|j+}| D]W} |j)d%|d&|d'|d'|d|d|d|dxsd dd(t f|j,dkDr|dz }n|dz }Y|j1|j3td*|d+|d,y#t.$r}td)||dz }Yd}~d}~wwxYw)-z0Full crawl: fetch first N pages, no date cutoff.z Full first z pages (no date cutoff)ruNrrrrz, crawling first z pagesr*rrIrrrrrrvrrrrr;rrz Total items: rrsrrtrrrrrrrrrrrrrr)r&r9r<rr.rrrbrfrrrrr@r}r#rrrrrrrr5rr)pagesr*rrrrrrrr?rrrrLrrrrrrrFr8s r%crawl_first_pagesrs+eW345 1 D  HHWa EHZ()Iuy(1,:EBK'%)+f=>IA{Q'r" $ D+r2Idhhvr6JIQYPZZoptvAqBpCCHOIJ/52s;H   ";/'2. !    gbTCJ"23 4 3-0FG ??7 #DLL*+LL,- A  II,k"D$4d:6Fg^ 4d9oy/'R#.8  9zzA~! 1   KKMJJL(6(& 23   *QC ! qLG sAL L>!L99L>c tjtdz jd}t d|dt d}|sy|j dg}|Dcgc] }|j dxsd dd |k\s|"}}g}|D]}|j d d xs#|j d d xstd |dd}t|\}} |r t|nd d d d} |j| dxs|j dd ||j dxsd dd | dtdt|dtjdd} d} ddl} | j!t"}|j%d|j%d|j'}|D]W} |j%d|d|d|d|d|d|d|dxsd ddtf|j(dkDr| dz } n| dz } Y|j-|j/t d| d | d!ycc}w#t*$r}t d|| dz } Yd}~d}~wwxYw)"NrrzIncremental (since rTrur*rrIrvrrrrrrqrrrtrrrrrrrrrrsrrrrr)rrrrr&r9r<rbr@r}rfr#rrrrrrrrrr5rr)rr*ritfilteredrrr?rLrrrrrrrFr8s r%crawl_incrementalrsllnya00 : :: FF fXQ'( 1 D  HHVR E"Srrvvm'<'BCR&HF&RSHSH XXk2.E$((622FEXJVklpq|l}k~DKE z*a'+d#2r^`1aG_="(="!XXm4:CR@i( 66(+$    3 FG ??7 #DLL*+LL,- A  II,k"D$4d:6Fg^ 4d9oy/'R#.8  9zzA~! 1   KKMJJL(6(& 23UTJ  *QC ! qLG s%$ H4H4+AH99 III__main__ru incremental)fullalli)rrr)$))&__doc__sysosr4rwrr0rrbs4r urllib.parsergetenvrr#rrb STHJJ_URLr2r.r3r&r9r@rOrKr}rrr__name__rargvmoder*r<rr'r%rsG )((( "))K!2 30  # ' J+ ,,   D1F ) A8  S4lQ4h14h zchh-!+388A;D }  $ q!(,!$! geW   r'