&sj:dZddlZddlZddlZddlZddlZddlZddl Z ddl Z dZ dZ dZ dZdZddd Z dd lmZej(j+d d Zej(j+d dZdZddddZdZdZeej:D]h\ZZejAdr e!ejEdddZ4ejAdsFejEdddjIZjedk\rendZ%eevreeZ dZ&d dZ'd!dZ(dZ)dZ*dZ+e,dk(re+yy#e$rdZYwxYw#e#$rYwxYw)"u 阜康市人民政府 (fk.gov.cn) — API 模式 列表: GET /content/page?channelIds={CH}&page=N&size=15&orderBy=27 → JSON {code,data:{content:[{title,urlWhole,releaseTime}]}} CH=11699 通知公告 / CH=12341 生态环境 详情: /p262/tzgg/20260731/452995.html 或 /c/fkrmzf/tzgg/xxx.jhtml, 正文 div.article-content 用法: python3 crawl_fk_tzgg.py --channel=11699 --pages=5 Nzhttps://www.fk.gov.cnu阜康市-通知公告u新疆zcrawl_fk_tzgg.pyu 环评公示u阜康市-生态环境)1169912341) BeautifulSoupDB_PATHz/mnt/data/search.db JSONL_PATHutf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagerz--pages==z --channel=c2|sytj|}|jddjddjddjdd}tjdd|}tjdd|}|j S) Nru​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts "/root/gov_crawler/crawl_fk_tzgg.py clean_titler1s !A (B''"5==hKSST\^`aA Er1MA vsAA 779cddl}ddl}|j|jjt t }|r||d<d}tdD]d} |j|||d}|jdk(rt|_ |jcSd|jz}tj d ft#d |dd d |d y#t$r} t| dd}Yd} ~ Jd} ~ wwxYw)NrRefererFheaderstimeoutverifyHTTP %sPz [fetch fail] Z -> Tflush)requestsurllib3disable_warnings exceptionsInsecureRequestWarningdictHEADERSrangeget status_codeENCencodingtext Exceptionstrtimesleepprint) urlrefererrr)r*hlast_err_res rfetchrB;s W//FFG W A) H 1X  # S!WU KA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #s:CC C0C++C0cddl}ddl}|j|jjdt |||fz}t t}d|d<d}tdD]]} |j||dd} | jd k(r| jcSd | jz}tjd _t!d |dd d|dy#t$r} t| dd }Yd} ~ Jd} ~ wwxYw)Nrz8%s/content/page?channelIds=%s&page=%d&size=%d&orderBy=27zapplication/jsonr rFrr!r"r#r$z [api fail] r&Tr')r)r*r+r,r-BASE_URLr.r/r0r1r2jsonr6r7r8r9r:) channelpagesizer)r*r;r=r>r?r@rAs r fetch_apirJPs W//FFG DRY[_aeGf fC W A$AhKH 1X # S!R FA}}#vvx 1==0H 1  S"Xx 8E   #1vcr{H #s 3CC C4C//C4c 4 !"|sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}g i!tjdd|}tjd d|}tr t |d }|j d d D]>}|j dsG|j ds6tjd|j ddtjs\|d}|j dxs|}|jd xsW|j dxsD|j dxs1tjj|jdd}tjj|} j!||fdt#!z}d|d|d!|<|j%|A|j dD]_} | j dxs| j dxsd} d| j'vsd| j'vsP| j)a|j dD]@} | j d} | s| jd r+g} | D]} | j dxs| j dxsd} | s.tjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}| j!d|d|d| sd"t#!z}d#d$j-| zd%z!|<| j%|C|j dD]8} | j dxs| j dxsd} | rd| j'vrAtjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}d"t#!z}d&}| j.}|!|j0dk(rd }n|j.}|!|r d|d|d!|<n d'|d|d(!|<| j%|;t3|}tjd)d*|tjtjz} !fd+}tjd,||tjtjz}trQ t |d }|j d D]%}d-D]}|j6j9|d 't3|}g"!"fd.}tjd/|tjrd0nd}tjd1||tjtjz}tjd%d2|tj}tjd3d4|tj}tjd5d|tj}g}|jd6D]}}|j+}|stjd7d|}tjd8d|}|j;d9dj;d:d;}tjd|}|j;d>d;j;d?d;}tjd@d|}|stjdAd|tj}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}|j!|j+d6j-|}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}tjdCd|}tjdDd|}tjdEd|}tjdFd|}t=j>|}tjdGd4|}tjdHd6|}tjdflagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】u'发布(?:时间|日期)[:：][^<]{0,30} html.parseraT)hrefappendixz data-appendixz\.(pdf|docx?|xlsx?|zip|rar|wps)rPoldsrc)r_titletitle?z__ATTACH__%d__ imgsrczfileTypeImages/icon_filetypeimagespaltu图片z __IMG__%d__

z


Fz

]*>(.*?)z\1c6|jd}|jdry|jd}tjdd|}t j |j }tjdd|j }|s5tjj|jddxsd }tjj|}j||fd tz}d |d |d |<|S)Nr z javascript:rr$<[^>]+>u^附件[:：]\s*rUru附件z __LINK__%d__rVrWrX)group startswithrrrrrospathbasenamespliturllibparseurljoinappendlen) mrPinnertxtabs_urlkey attachments link_protectpage_urls r _link_replz html_to_text.._link_replswwqz ??= ) ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*s<00ELcR S rz&]*href="([^"]+)"[^>]*>(.*?))styleclasslangdiralignvalignwidthheightborder cellpadding cellspacing indenttextnoextractcontentwzadestabindexcZ|jd}jD]I\}}|jds|j||jddjdd}Kt j dd|tj }t j dd|tj }t j d d|tj }t j d d|tj }j|d td z zS) Nr__IMG__r^rr_(]*>rLz

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

__TBL__%d__r ) rbitemsrcrrrIrkrl)rmtblkvrs table_protects r _tbl_replzhtml_to_text.._tbl_replsggaj &&( ODAq||I&kk!QYYub%9%A%A&"%MN Off@"cQSQUQUVff2E3bddKff4fcNff3RBDDIS!M 2Q 677rz ]*>r z]*>.*?z

z  rz z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  rz [ \t]{2,}ra z z[\s\u200b\u200c\u200d\ufeff]+z

\s*

rz __LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r)$rrSrrfind_allr1searchget_textrdrerfrgrhrirjrkrl replace_withlower decomposerjoinparentnamer7r6attrspoprrrr enumeratesetadd)#contentrt_souprOrP real_hrefrorprqrYrZr\imgssrcshas_p_ancestor_parrutagattrr has_tablepartsblockbplainrriroutseenfinalrrrsrs# ` @@@r html_to_textrdsR 1byff]Brtt"))Lnpqpupuv|AqBDFDHDHCIV9D !h 74I**4*0CAEE(OCquuW~CY[Y`Y`YiYijsjyjyz}j~@AkBZCC$ll228YGG&&~6*S->>CQXZ](^L%NN3' (~~e, $wwu~@):@b)SYY[8#KSWWW5E#K"R"R"T#HXZX_X_XhXhilirirsviwxyizX{#H@HC KKQXZ](^_ ` +c,.??,1GLL4F,F,O S)s+ ,~~e, &ggen?(9?R4 C ,,..x=wwu~?)9?RFFH{BGGL\L\]`]f]fgj]klm]nLo{s{#c,&77!&zz&yyC')-;;D & "QXZ](^L%"Y`ad(eL%  %% &&%jGff>wVXVZVZ]_]a]aVabGff> G[][_[_bdbfbf[fgG !'=9E~~d+ .D.DIIMM$-. .%jGM 8YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  FF="artt 4 &&( DAq !QA  . 2FAs -!+S1A 2 QWWY' * ++e C""$ 1kk!Q M*23kk-!+S12 &&!2s +C &&"c *C &&"c *C &&#R -C   C C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&K   :   sSA:i:?Ei::i:i:1i:B)i:1Ei: :i:Aj : jj jjcd}d}d}tjd|}|rt|jd}|sRtjd|tjtj z}|rt|jd}tjd|}|rU|jdd|jdj dd|jdj d}trgt|d }|jd xs$|jd xs|jd }|r"djd |jD}|||fS)Nrz)(.*?)u'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2})-r$rrNzdiv.article-contentz div.contentzdiv#zoomc32K|]}t|yw)N)r7).0cs r zparse_detail..s"?a3q6"?s) rrrrbrrzfillr select_onercontents) html_textrtrTdate content_htmlrmdmsoupels r parse_detailr s# E DL > JAAGGAJ'  II-y"$$+ F  +E =y IB XXa["((1+*;*;A*> @Q@QRS@TUY 6 __2 3 tt}7U tY]YhYhisYt 77"?2;;"??L $ $$rcd}d}t}g}g}d}|tkrtt|}|r|j ddk7rt d|zdn|j di}|j d xsg}|st d |zdn|j d xs|j d xsd} t d |t || fzd|D]6} | j dxs| j dxsd} t| j dd} | j dxsddd} | r| sd| jds$tjjt| } | |vr|j| t| }|s|dz }t|| \}}}|s| }|s| }t!|| \}}}t t#j$dd|j'}t#j(d|t"j* xr+t#j(d|t"j* xr| }|dks|rt d|ddzd|dz }t,| | ||d|t.dt0|rdj3d|Dndd }t4r/|j7|t d|ddd|dd|dz }|j7||dz }t d |ddd|ddt9j:d!9|dz }t9j:d"|tkrt4rlt=t4d#d$%5}|D]+}|j?tAjB|d&'d(z- dddt d)t4t |fzdnt|rrtDjFjIdtJjFjMtJjFjOtPdd*l)m*}||tV+t d,|zdt d-|zdt d.t,d/td0dy#1swYxYw)1Nrr coder!zPage %d: api fail, stopTr'datarzPage %d: empty, stop totalElementstotalzPage %d: %d items (total %s)urlWholer;rrT releaseTime httpraz]z]z skip empty/placeholder: %s(z; c30K|]\}}|d|yw)|N)rurs rrzmain..Vs(KdaAq)9(Ks) site_name source_urlr;rTpub_datesummaryrcategorytags group_namerrz JSONL + z ()z + g?g?wr )r4F) ensure_asciirzJSONL written: %s (%d rows))push_to_searchdb) batch_labelu 新增: %du 跳过: %dz=== z done (channel=z) ===),r_PAGESrJ_CHANNELr1r:rlrrcrhrirjrErrBrrrrrrr SITE_NAMECATEGORY GROUP_NAMErrrkr8r9openwriterFdumpssysreinsertrddirnameabspath__file__ crawler_libr SCRIPT_NAME) new_count skip_count seen_urls jsonl_rowsbatchrHjrrritr;rTrdhtmldtitleddaterrratts plain_lenno_paraitemfrowrs rmainr#sIJIJ E D &. h %AEE&MS( +d2$ ? uuVR #)r (4/t < )CTXXg->C! ,c%j%/HHPTU* B&&$;u ;Cw 34EFF=)/R"5De>>&)ll**8S9i MM# #JEa *6uc*B 'FE<'3L#'F $GYBFF:r7;AACDI))Iw=={biiPY[bdfdhdhFiBi{r{n{G2~4vcr{B$Oa &c#Ur""(OStyy(Kd(KKY[ D!!$'VCR[%@MQ  LL  NI 6#2;6d C JJt U* V   3q &.t *cG 4 D! D 3U;dBC D D +z3z?.KKSWX  HHOOArwwrwwx/HI J 4 U < , "$/ , #40 Ix @M D Ds 1QQ#__main__)NrD))-__doc__rrdrr8htmlr urllib.parserhsqlite3rFrErrrr CHANNEL_IDSbs4r ImportErrorenvironr1rrr3r/ _MAX_PAGESrrargvrrOrcintrg ValueErrorrrrrBrJrrr__name__rrrrsHGGG " $   0;ST ! **..$9 : ZZ^^L" -  @O'    chh .DAq||J QWWS!_Q/0J l #773?1%++-."QA {H%I*(f'R%,NNb zF M&   s#D.'D;.D87D8;EE