uj( dZddlZddlZddlZddlZddlZddlZddlmZmZddl m Z ddl m Z ejddZdZd Zd Zd d iZd ZdZdZdZdZdZdZdZd"dZdZedk(reej>dkDrej>dndZ e dk(reye dk(reye dk(rFedZ!e!ree!ngZ"edee"de"ddD]\Z#Z$Z%e&de%d e#dd!yyy)#u Crawler: 鞍山市行政审批局 - 通知公告 (TRS WCM static HTML) List: glist.html (page 1), glistN.html (page N+1), 101 pages x 20 = ~2015 items Detail: /html/ASSPJ/YYYYMM/017xxxxxxxxxxxxx.html Content div: hwq-info-article-center N)datetime timedelta) BeautifulSoup)urljoin SEARCH_DBz/root/search.dbu鞍山市行政审批局u 通知公告zhttp://xzspj.anshan.gov.cnz User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36) z.docz.docxz.pdfz.xlsz.xlsxz.pptz.pptxz.zipz.rarz.7zec2tdtd|dy)N[z] T)flush)printSITE)msgs '/root/gov_crawler/crawl_anshan_xzspj.pylogrs AdV2cU 4(c|dk(r td}ntd|dz d} tj|td}d|_|j S#t $r}td|d |Yd }~y d }~wwxYw) zG LLgr : vv  j (1# ./s.A A3A..A3cg}t|d}|jdd}|s'|jdd}|r|jd}|r|jdd }|D]}|jdd }|jdd }|s.|s1|jd } |jd} | sW| j dsi| jd } | d} |j | | | f|S)z*Extract (title, url, date) from list page. html.parserulc|r|xrd|vSdS)NdisplayF)vs rz!parse_list_page..3s)=yA~r)styledivc.|xrd|jvS)Ninfosplitcs rr,z!parse_list_page..6sQ5N6QWWY;Nrclass_liF) recursivec6|r|xrd|jvSdS)NtimeFr1r3s rr,z!parse_list_page..=TUq7PVqwwy=P[`rc6|r|xrd|jvSdS)NnameFr1r3s rr,z!parse_list_page..>r;rTstripahref)rfindfind_allget_textrappend) htmlitemssoupr'info_divlisr7time_divname_divdatea_tagtitler!s rparse_list_pagerP+s E } -D 4M NB 99U+N9O t$B kk$%k0 5Bwwu-`waHwwu-`waHH((t(4 c*UYYv.!NNN6E-CLL%d!34 5 Lrc tj|td}d|_|jS#t $r}t d|d|Yd}~yd}~wwxYw)zFetch detail page.rrrz Detail error : N)rrrrrrr) detail_urlr"r#s r fetch_detailrTIsQ LLWb A vv  mJ.Zs#ab!2]8PTUT[T[T]8]hmrr5Tr>rOz(.*?)rc6|r|xrd|jvSdS)Nzhwq-info-article-timeFr1r3s rr,zparse_detail..cs#_`1[7NRSRYRYR[7[fkrz(\d{4}-\d{2}-\d{2})rXc6|r|xrd|jvSdS)Nzhwq-info-article-centerFr1r3s rr,zparse_detail..ks#deA4`:SWXW^W^W`:`kprrYbody  separatorr?) rrBrDresearchgroupr?extract_clean_textstr) rFrSresultrH title_divmrK time_text content_divr]s r parse_detailrkTs,2" =F } -D %(m nI#,,4,8w '? II-t 4 ggaj..0F7Oyy'kylH%%' II,i 8 %&WWQZF> "))E*p)qK.s;/?@y M yy   $ D IF9  Mrc|syt|d}|jddD]R}|jdd}|jdxsd}t t |}|j d |d |d T|jgd D]}|jg}|jd dgD]u}|jd k(r|jt|-|jdk(s=|jd rO|jdd} | se|j| w|rdj|S|jS)zDConvert content HTML to text with markdown links, preserving tables.rVr&r@T)rArAr>u附件r z]())spanbstrongfontemiustablepr_z )rrCrrDrr replace_withunwrapr=rEre find_parentjoinr?) content_htmlrHrNrArfull_urltagpartselts rrdrdws:  } 5Ds.4yy$~~D~)5X8T* QtfBxj23 4}}QR  EmmWcN+  77g  LLR ! WW^BNN7$; b 5A Q  "'6;;u @L,>,>,@@rc tjt|dzz jd}t d|dg}t dt dzD]}t|}|snt|}|sn|D];\}}}||kr |rd|jvs|dk(r'|j|||d =t d |d t|d t|dtjd t|dksnt dt|dg} t|dD]\} } t| d}|r2t!|| d} | dr| d| d<| dr| d| d<| d| d<nd| d<| j| d| d| d| j#ddt$dt&t&d| dzdk(rt d| dt|tjd d} d}ddl}|j+t,}|j/d|j/d|j1}| D]V} |j/d| d| d | d | d| d| d| dxsddd!| d"f|j2dkDr| dz } n|dz }X|j7|j9t d$| d%|d&y#t4$r}t d#||dz }Yd}~d}~wwxYw)'Nrdays%Y-%m-%dzFull crawl (cutoff: rmrglist#)rOr!rMz Page rRz items (accumulated: ?zAfter filter:  itemsr!rOrXrMrYrV-rOpage_urlrXrY site_namecolumnrz Detail: /PRAGMA journal_mode=WALPRAGMA busy_timeout=10000INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category) VALUES (?,?,?,?,?,?,?,?)rrr DB error: Result:  new,  skipped)rnowrstrftimerrange MAX_PAGESr$rPlowerrElenr:sleep enumeraterTrkrr COLUMNsqlite3connectDB_PATHexecutecursorrowcountrcommitclose) months_backcutoff all_itemspgrFrGrOr!rMenrichedidxitemdetailstoredskippedrconnr4r#s r crawl_allrsIllnykB.>?? I I* UF vha()IAy1}%r" %  %  E3f}'SYY[0C3J      gbTCJ<'"23 4 3-2FG ??7 #DLL*+LL,- A  II,k"D$4d:6Fg^ 4d9oy/'R#.X@  AzzA~! 1   KKMJJL(6(& 23   *QC ! qLG s AK K8K33K8c "tjtdz jd}t d|dt d}|syt |}|Dcgc](\}}}||k\s |jdrd|vs$|||f*}}}}g}|D]m\}} } t| }|r t|| nd d d d } |j| d xs|| | | d td ttdtjdod} d} ddl}|j!t"}|j%d|j%d|j'}|D]V} |j%d|d|d|d|d |d|d |d xsd dd|df|j(dkDr| dz } n| dz } X|j-|j/t d| d| dycc}}}w#t*$r}t d|| dz } Yd}~d}~wwxYw)NrrzIncremental (since rmrhttprrVrWrOrYrrrrrrrrrrXrrrrrr)rrrrrr$rP startswithrTrkrEr rr:rrrrrrrrrr)rrFrGrrtdfilteredrrOr!rMrrrrrr4rr#s rcrawl_incrementalrsDllnya00 : :: FF fXQ'( 1 D  D !E).ppgaA!v+allSYFZ_fno_oAq pHpH$ sDC ,0dC(TVce6fG_- i( 66(+    3 FG ??7 #DLL*+LL,- A  II,k"D$4d:6Fg^ 4d9oy/'R#.X@  AzzA~! 1   KKMJJL(6(& 23SqH  *QC ! qLG s1G!.G!G!G!AG(( H1H  H__main__r incrementalfulllistzPage 1: rz z | <)$)'__doc__sysosjsonrar:rrrbs4r urllib.parsergetenvrr rrr ATTACH_EXTSrrr$rPrTrkrdrr__name__rargvmoderFrGrrtrr r*rrrs@ )((( "))K!2 3!  'D a  ) < FA8S4l24j zchh-!+388A;D }  q!)-%2 hs5zl& )*Ray 'GAq! BqcQsVH% & '  r