uj*dZddlZddlZddlZddlZddlZddlZddlmZdZ dZ dZ dZ dZ d e d zd Zd Zd ZdZdZdZdZdZdZedk(reyy)u:Crawl tzw.nantong.gov.cn - 通州湾示范区 通知公告N)datetimezhttps://tzw.nantong.gov.cntzwnantong_tzggz/root/search.db z> ! !#r ! 2D 99;  gi  88cttdz}tjd|}|r|j dStjd|}|r|j dSdS)z*Extract columnId from list page JS config.rzcolumnId:'([^']+)'zcolumnId:"([^"]+)"N)rBASEresearchgroup)htmlms r get_column_idr#sZ 66 7D '.Awwqz '.A1771:$$rc td|d|d|dt }t|}tjd|}|sgSt j |j}|jdd}g}tjd|} | D]} tjd | } tjd | } tjd | } | sH| sK| jd }|jd s t|z}|j| jd || r| jd nddtjd|}|rt|jd nd}||fS)zLGet article list from TrueCMS API. Returns list of {title, url, date, href}.z>/truecms/messageController/getMessage.do?callback=jq&columnId=z &startrecord=z &endrecord=z &perpage=z\{.*\}resultz&zhref="([^"]+)"ztitle="([^"]*)"z]*>(\d{4}-\d{2}-\d{2})rhttp)titlerdatez (\d+)r) rPER_PAGErrrjsonloadsr getfindall startswithappendint) column_idstartendrrr"data result_xmlarticlesrecordsrechref_mtitle_mdate_mhreftotal_mtotals rget_articles_from_apir@ sWV$$-; 5'SE8* FC :D )T"A  ::aggi D(B'JHjjBJOG ,c2)).4 )flagsz  z*]*>r&z ]*>z\n{3,})rsubIsplitstripjoin) html_chunktextlliness rextract_clean_textrOFs 66'6:RTT :D 66,d"$$ 7D 66)VT 6D 66?TQSQUQU VD 66-T 6D 66)VT *D $ 4 0 11QWWY 1E 1 99U D 66)VT *D ::< 2s.D?cg}tjd|tj}|D]4\}}t|z}|j d|j d|d6dj |S)z4Extract attachment links, return as markdown string.zR]*href="(/truecms/attachmentController/download\.do[^"]*)"[^>]*>([^<]+)[z]()rD)rr.rGrr0rIrJ)r!partslinksr=namefull_urls rextract_attachmentsrWTsq E JJ] bdd E6 d$; qb !456 99U rc~t|}d}tjd|}|r|jdj }|sLtjd|}|r4tj dd|jdj }d}tjd|}|r|jd}|s)tjd|}|r|jd}d}tjd|} | r| jdj }d} |j d } | d kr|j d } | d kDr|jd d | } | d kr| } |j d | dz} d}| }|d kDr{|t|krm|j d |}|j d|}|d krnC|d k\r ||kr|dz }|j d |dz}n |dz}|dz}|d kDr|t|krm|d k(r|| |dz n|| d}t|} t|}|r | r| dz } | |z } ||| |fS)z4Get (title, date, content, source) from detail page.r&z

([^<]+)

rz([^<]+)u\s*[-–—|_].*$u(发布时间[：:]\s*(\d{4}-\d{2}-\d{2})z$(\d{4}-\d{2}-\d{2})u来源[：:]\s*([^<]+)z id="zoom"rz class="pp3"zrENrB) rrrr rIrFfindrfindlenrOrW)rr!r(tmtm2r)dmdm2sourcesmcontentidxr3posdepthr4noncchunk attachmentss r get_detailrlasK :D E & -B  !!# ii148 FF/SYYq\BHHJE D > EB xx{ ii?F 99Q|dxsd dd }|jd t|d|d|d|d |d r0ttj|d d jnd|dd|f|j|dz }t d|dddd|d |j|S)NrzPRAGMA busy_timeout=30000z)SELECT id FROM gov_raw WHERE source_url=?rz SKIP: r((rdr&izINSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, content) VALUES (?,?,?,?,?,?,?,?)r)z%Y-%m-%drz OK: 2z | )sqlite3connectDB_PATHexecutecursorfetchoneprint SITE_NAMEr1rstrptime timestampcommitclose)r8connccountrexistingrds r save_to_dbrsF  ??7 #DLL,- A E  899H1U8+V__a  HQwZ_-. / Y<%2u- ( %!E(AgJ& JKF)S""1V9j9CCE FYZ Tc]G % &    qz#2's1V9+67 8 JJL Lrc Ltdtdt}|s tdytd|t|dt\}}|s tdytd|d|tzdzd t |}t tdzt|ttzdztD]}t|tzdz |}|tzdz}td |d |d |d t|||\}}|s tdn+tdt|d|j|tdt|dg} t|D]h\} } td| dzdt|d| dddd t| d\} } }}| j| xs| d| d| xs| d||djt#| }td|dt| dy#t$r,}td|tj Yd}~d}~wwxYw)Nz === Crawling z ===zERROR: Cannot find columnIdz Column ID: rzERROR: No articles foundzTotal: z articles (z pages)z Page z (records -z)...z No more items, stoppingz Got z articlesz === Total: z articles ===z [/z] r(rpz...rr))r(rr)rdrbz ERROR: z === Complete: z new records ===)rwrxr#r@r*listrangemin MAX_PAGESr]extend enumeraterlr0 Exception traceback print_excr)r2r7r? all_articlesr3r4page_numitems_r8iar(r)rdrbesaveds rmainrsQ M)D )*I  +, K { #$,Iq(COHe  () GE7+ex&7!&;%Lx!|S H0D%E%I8T #%("Q&.H$q( zE7!C5=>(E3?q - /  s5zl),-E" # M#l+,M :;G,' "1 AaC5#l+,Bqz#2.?sCD "+5ah+? (E4& NN,!G*x)& "   " w E UG1S\N2B CD  " IaS/ "    ! ! "s ;G.. H#7"HH#__main__)__doc__sysrr+rqurllib.requestrrrrrxrsr*rr rr#r@rOrWrlrr__name__rrrs@888#     Q44  9 %#L  A(H23El zFr