3sj;dZddlZddlZddlZddlZddlZddlZddl Z ddl Z dZ dZ dZ dZdZdZd Z dd lmZej*j-d d Zej*j-d dZdZddde dZdZeej:D]k\ZZejAdr e!ejEdddZ4edk(s:edze$ej:ksS e!ej:edzZmedk\rendZ%dZ&dZ'd!dZ(dZ)dZ*dZ+dZ,dZ-e.d k(re*yy#e$rdZYwxYw#e#$rYwxYw#e#$rYwxYw)"u^ 广宁县人民政府-通知公告 (gdgn) 列表: /xwzx/tzgg/ + index_{N}.html (19页 276条, 广东统一发布平台) 详情: /xwzx/tzgg/content/post_{数字}.html 正文: div.conTxt > div#fontzoom 注意: 服务器IP被屏蔽, 本地Mac直连IP+Host头 (降级闭环模式) 用法: JSONL_PATH=/tmp/gdgn.jsonl python3 crawl_gdgn_tzgg.py --pages=1 Nzwww.gdgn.gov.cnz119.125.233.69u广宁县-通知公告u广东zcrawl_gdgn_tzgg.pyu 环评公示z /xwzx/tzgg/) BeautifulSoupDB_PATHz/mnt/data/search.db JSONL_PATHutf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-LanguageHostz--pages==z--pagesc~|jdtzdtzjdtzdtzS)uIhttps://www.gdgn.gov.cn/xxx -> https://119.125.233.69/xxx (保留Host头)https://zhttp://)replace BASE_DOMAINBASE_IP)urls $/root/gov_crawler/crawl_gdgn_tzgg.py to_ip_urlr5s7 ;;zK/g1E F N Ny[fOfhqt{h{ ||c2|sytj|}|jddjddjddjdd}tjdd|}tjdd|}|j S) Nru​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescaperresubstrip)ts r clean_titler:s !A (B''"5==hKSST\^`aA Er1MA vsAA 779rcddl}ddl}|j|jjt t }|r||d<d}tdD]m} |jt|||d}|jdk(rt|_ |jcSd|jz}t!j"d ot%d |dd d |d y#t$r} t| dd}Yd} ~ Jd} ~ wwxYw)NrRefererF)headerstimeoutverifyzHTTP %sPz [fetch fail] Zz -> Tflush)requestsurllib3disable_warnings exceptionsInsecureRequestWarningdictHEADERSrangegetr status_codeENCencodingtext Exceptionstrtimesleepprint) rrefererr#r+r,hlast_err_res rfetchrCDs W//FFG W A) H 1X  # Ys^QPU VA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #sACC C9!C44C9c 4 !"|sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}g i!tjdd|}tjd d|}tr t |d }|j d d D]>}|j dsG|j ds6tjd|j ddtjs\|d}|j dxs|}|jd xsW|j dxsD|j dxs1tjj|jdd}tjj|} j!||fdt#!z}d|d|d!|<|j%|A|j dD]_} | j dxs| j dxsd} d| j'vsd| j'vsP| j)a|j dD]@} | j d} | s| jd r+g} | D]} | j dxs| j dxsd} | s.tjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}| j!d|d|d| sd"t#!z}d#d$j-| zd%z!|<| j%|C|j dD]8} | j dxs| j dxsd} | rd| j'vrAtjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}d"t#!z}d&}| j.}|!|j0dk(rd }n|j.}|!|r d|d|d!|<n d'|d|d(!|<| j%|;t3|}tjd)d*|tjtjz} !fd+}tjd,||tjtjz}trQ t |d }|j d D]%}d-D]}|j6j9|d 't3|}g"!"fd.}tjd/|tjrd0nd}tjd1||tjtjz}tjd%d2|tj}tjd3d4|tj}tjd5d|tj}g}|jd6D]}}|j+}|stjd7d|}tjd8d|}|j;d9dj;d:d;}tjd|}|j;d>d;j;d?d;}tjd@d|}|stjdAd|tj}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}|j!|j+d6j-|}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}tjdCd|}tjdDd|}tjdEd|}tjdFd|}t=j>|}tjdGd4|}tjdHd6|}tjdflagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】u'发布(?:时间|日期)[:：][^<]{0,30} html.parseraT)hrefappendixz data-appendixz\.(pdf|docx?|xlsx?|zip|rar|wps)rIoldsrc)r_titletitle?z__ATTACH__%d__ imgsrczfileTypeImages/icon_filetypeimagespaltu图片z __IMG__%d__

z


Fz

]*>(.*?)z\1c6|jd}|jdry|jd}tjdd|}t j |j }tjdd|j }|s5tjj|jddxsd }tjj|}j||fd tz}d |d |d |<|S)Nr z javascript:rr'<[^>]+>u^附件[:：]\s*rNru附件z __LINK__%d__rOrPrQ)group startswithrrrrrospathbasenamespliturllibparseurljoinappendlen) mrIinnertxtabs_urlkey attachments link_protectpage_urls r _link_replz html_to_text.._link_replswwqz ??= ) ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*s<00ELcR S rz&]*href="([^"]+)"[^>]*>(.*?))styleclasslangdiralignvalignwidthheightborder cellpadding cellspacing indenttextnoextractcontentwzadestabindexcZ|jd}jD]I\}}|jds|j||jddjdd}Kt j dd|tj }t j dd|tj }t j d d|tj }t j d d|tj }j|d td z zS) Nr__IMG__rWrrX(]*>rEz

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

__TBL__%d__r ) r[itemsr\rrrIrdre)rftblkvrl table_protects r _tbl_replzhtml_to_text.._tbl_replsggaj &&( ODAq||I&kk!QYYub%9%A%A&"%MN Off@"cQSQUQUVff2E3bddKff4fcNff3RBDDIS!M 2Q 677rz ]*>r z]*>.*?z

z  rz z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  rz [ \t]{2,}rZ z z[\s\u200b\u200c\u200d\ufeff]+z

\s*

rz __LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r)$rrSrrfind_allr3searchget_textr]r^r_r`rarbrcrdre replace_withlower decomposerjoinparentnamer9r8attrspoprrrr enumeratesetadd)#contentrm_souprHrI real_hrefrhrirjrRrSrUimgssrcshas_p_ancestor_parrntagattrr has_tablepartsblockbplainrriroutseenfinalrkrlrs# ` @@@r html_to_textrYsR 1byff]Brtt"))Lnpqpupuv|AqBDFDHDHCIV9D !h 74I**4*0CAEE(OCquuW~CY[Y`Y`YiYijsjyjyz}j~@AkBZCC$ll228YGG&&~6*S->>CQXZ](^L%NN3' (~~e, $wwu~@):@b)SYY[8#KSWWW5E#K"R"R"T#HXZX_X_XhXhilirirsviwxyizX{#H@HC KKQXZ](^_ ` +c,.??,1GLL4F,F,O S)s+ ,~~e, &ggen?(9?R4 C ,,..x=wwu~?)9?RFFH{BGGL\L\]`]f]fgj]klm]nLo{s{#c,&77!&zz&yyC')-;;D & "QXZ](^L%"Y`ad(eL%  %% &&%jG ff>wVXVZVZ]_]a]aVabGff> G[][_[_bdbfbf[fgG !'=9E~~d+ .D.DIIMM$-. .%jGM 8YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  FF="artt 4 &&( DAq !QA  . 2FAs -!+S1A 2 QWWY' * ++e C""$ 1kk!Q M*23kk-!+S12 &&!2s +C &&"c *C &&"c *C &&#R -C   C C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&M   <   sSA:i:?Ei::i:i:1i:B)i:1Ei: :i:Aj : jj jjcddl}|j}|jjtd|_d}d}t }g}g}tdtdzD] } t||}t|} | std|zdntd|t| fzd| D]\} } } | |vr |j| t| }|s|dz }1t!|| \}}}|s| }|s| }t#|| \}}}tt%j&d d |j)}t%j*d |t$j, xr+t%j*d |t$j, xr| }|d ks|rtd|ddzd|dz }t.| | ||d |t0d t2|rdj5d|Dnd d }t6r/|j9|td|ddd|dd|dz }c|j9||dz }td|ddd|ddt;j<dt;j<d t6rlt?t6dd5}|D]+}|jAtCjD|ddz- dddtdt6t|fzdn|rddl#m$}||tJ td!|zdtd"|zdtd#t.zdy#t$r} td|| fzdYd} ~ d} ~ wwxYw#1swYxYw)$NrFr zPage %d fetch error: %sTr)zPage %d: empty, stopzPage %d: found %d itemsrZrz]z] z skip empty/placeholder: %s(z; c30K|]\}}|d|yw)|N).0urs r zmain..3s(KdaAq)9(Ks) site_name source_urlrrMpub_datesummaryrcategorytags group_namerkz JSONL + z ()z + g?g?wr)r6) ensure_asciirzJSONL written: %s (%d rows))push_to_searchdb) batch_labelu 新增: %du 跳过: %dz=== %s done ===)&r+Sessionr"updater1r$rr2_PAGES fetch_list parse_listr8r<rerrC parse_detailrrrrrr SITE_NAMECATEGORY GROUP_NAMErrrdr:r;openwritejsondumps crawler_libr SCRIPT_NAME)r+s new_count skip_count seen_urls jsonl_rowsbatchpage html_textrrBrirMdatedhtmldtitleddate content_htmlrratts plain_lenno_paraitemfrowrs rmainrsBAIIWAHIJIJ Ea!$/ "1d+Iy)E (4/t <  '4U*<]*href="(https?://www\.gdgn\.gov\.cn/xwzx/tzgg/content/post_\d+\.html)"[^>]*>(.*?)r rZrr'uJ]*href="%s"[^>]*title="[^"]*?发表时间[：:]\s*(\d{4}-\d{2}-\d{2}))rfinditerrrrrr[rrrrerescaperd)rrrfrIrhrMrtms rrrYs E [[wzCEGEIEILNLPLPEPQ *  ,224z2qwwqz BCIIKC E Q  YYdgigpgpqugvvyBC 88A;D dE4() * Lrcd}d}d}tjd|}|rt|jd}|sRtjd|tjtj z}|rt|jd}tjd|}|rU|jdd|jdj dd|jdj d}trgt|d }|jd xs$|jd xs|jd }|r"djd |jD}|||fS)Nrz)(.*?)u'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2})-r'r!rGzdiv.conTxt div#fontzoomz div.conTxtzdiv#zoomc32K|]}t|yw)N)r9)rcs rrzparse_detail..}s"?a3q6"?s) rrrr[rrzfillr select_onercontents) rrmrMrrrfdmsoupels rrrks# E DL > JAAGGAJ'  II-y"$$+ F  +E =y IB XXa["((1+*;*;A*> @Q@QRS@TUY 6 __6 7 w4??<;X w\`\k\klv\w 77"?2;;"??L $ $$r__main__)Nr)/__doc__sysr]rr:htmlr urllib.parserasqlite3rrrrrrrrbs4r ImportErrorenvironr3rrr5r1 _MAX_PAGESrargvrrHr\intr` ValueErrorrerrrrCrrrrr__name__rrrrsHGGG  $  "   ! **..$9 : ZZ^^L" -  @O'    chh  DAq||J QWWS!_Q/0J iAECM1 SXXa!e_-J "QA} *g'TI5X$%, zFU M&       s5D+%D8!E+D54D58E?EE  E