ztjhdZddlZddlZddlZddlZddlZddlmZmZejjdejjejje ddl mZejedz j!dZdd iZd d d d dd dZdZdZdZdZddZedk(rsej4Zej9dedej9ddej=Zeej@ejBZ"e#de"dyy) u[crawl_ahtlyaq_hjsp.py - 铜陵义安区-环评审批(OpennessContent, --col支持双栏目)N)datetime timedelta)push_to_searchdbiG)daysz%Y-%m-%dz User-Agentz6Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0u义安区-环评审批决定z-/openness/OpennessContent/showList/1599/53929)namepathu义安区-环评受理公示z-/openness/OpennessContent/showList/1599/53865)5392953865zhttps://www.ahtlyaq.gov.cnc<tj|tdS)Nr )SITESget)cols '/root/gov_crawler/crawl_ahtlyaq_hjsp.pyget_siters 99S%. ))ct|dd|dzd} tj|dtd}d|_t j d |j}g}|D]{\}}t jd d |}t|d kr,|jd r t|z}n |jdr|}n td z|z}|j|d |d}|S#gcYSxYw)Nrz/page_z.htmlFtimeoutheadersverifyutf-8zG]+href="([^"]*OpennessContent/show/\d+\.html)"[^>]*title="([^"]*)"\s+/http)urlpub_datetitle) BASErequestsr HEADERSencodingrefindalltextsublen startswithappend) sitepage_numr!ritemsresultshrefr#full_urls rfetch_urls_from_pager6s F4<.x!|nE :C LLgwu M  JJR  EG J evr5) u:>  ??3 d{H __V $HczD(HxR%HI J N' s $CC$cVtdD](} tj|dtd}d|_nyd}t j d|jt j}|rKt jd d|jd j}t jd d|}t jd d|j}d}t j d |j}|r|jd }|s6t j d|jdd}|r|jd }d}t j d|jt j}|r|jd j}|||fS#d}YxYw)NrFrr)rrrz([^<]*)z<[^>]+>ru$\s*[-_|]\s*义安区人民政府\s*$rz1]*>(.*?)) ranger%r r&r'r(searchr*DOTALLr+groupstrip) r!_attemptr1r#mr"mpmdcontents r fetch_detailrC.s`!H  S'75QA AJ   y E +QVVRYY?Az2qwwqz288:>EJ FF62u % + + -EH G PB 88A;  YY-qvvet} = xx{HG 7KA''!*""$ '8 ##/ As $F!!F(rr c pt|}g}t}t|D]}t||}|snt d|dzdt |d|D]}|dr |dt kr|d|vr|j|dt|d\}} } |s|d}| xs|d} | jsd|j||d| |d| d d t d t |d|D cgc]} | d js| } } | rt| d |t | Scc} w)Nz Page rz: z itemsr"r!r#rr)r#r!r" site_namerBsummaryz Total: rBahtlyaq_) rsetr9r6printr,CUTOFFaddrCr=r.r) max_pagesrr/recordsseenpgr2itr#rBr"pdr1valids rrunrSMs] C=DG 5DI$T2.  1vRE |623 B*~"Z.6"9%yD  HHRY '3BuI'> $E7H7 +R ^B==? NNr%yb!&\g"   * Ic'l^6 *+ 811Y<#5#5#7Q 8E 8 (3% 01 u: 9s 8D3D3__main__z--pages)typedefaultz--col)rV)rLrzDone: z records)rr )$__doc__sysosr%r(argparserrrinsertdirnameabspath__file__ crawler_librnowstrftimerJr&r r$rr6rCrS__name__ArgumentParserap add_argumentint parse_argsargspagesrcntrIrrrlsa&&&(277??277??8#<=>( (,,.9%0 0 : :: F Q R4>m n4>m n $*4$>> z  "BOOICO3OOGWO- ==?D   1C F3%x ! r