$j 标题 2026-09-11… 分页: 同一 URL 追加 ?open=fdzdgknr&nr_page=N&nr_per_page=10（服务端渲染第 N 页） 共 fdzdgknr_total=2517 条，10 条/页 详情: .../doc_.html 标题: meta ArticleTitle / h1.details-title 日期: meta PubDate / span.details-release-date（发布日期：YYYY-MM-DD） 正文: div.details-content#details-content（保留

/） N)datetime timedelta) BeautifulSoup JSONL_PATHutf-8u1淄博市生态环境局博山分局-环评信息u山东zcrawl_boshan.pyu 环评信息zhttp://www.boshan.gov.cnzL/gongkai/site_bsqsthjj/channel_c_5f9f6c5d03741cded2c00eeb_n_1605681925.7632/ iG)daysz%Y-%m-%dzkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-LanguageRefererz--pages==z--pagesc|sytj|}|jddjddjddjdd}tjdd|}tjdd|}tjdd|}tjd d |}|j S) Nru​u‌u‍u<[^>]+>u3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*u ^[•·]\s*\s+ )html_libunescapereplaceresubstrip)ts crawl_boshan.py clean_titler?s !A (B''"5==hKSST\^`aA z2q!A Er1MA ~r1%A vsAA 779ctt}d}tdD]} ddl}ddl}|j |j j|j|||d}|jdk(rt|_ |jcSd|jz}tj dt#d |dd d |d y#t$r}t|dd}Yd}~Jd}~wwxYw)NrF)headerstimeoutverifyzHTTP %sPz [fetch fail] Zz -> Tflush)dictHEADERSrangerequestsurllib3disable_warnings exceptionsInsecureRequestWarningget status_codeENCencodingtext Exceptionstrtimesleepprint) urlr!hlast_errattemptr,r-res rfetchrAKs W AH8  # $  $ $W%7%7%N%N O S!WU KA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #sA'C C C)C$$C)c .&'(|sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}g&i'tjdd|}tr t |d }|j d d D]}|d }|j ds;|j ds*tjd|tjsd|vsU|xsdj}|j dxs|j dxsdj}|r(tjd|tjs|}n|xs|}|jd xs[|j dxsH|j dxs5tjj|jddxsd}tj j#|} &j%| |fdt''z} d| d|d'| <|j)| |j dD]`} | j dxs| j dxsd} tjd | tjsQ| j+b|j d!D]@} | j d}|s| jd r+g}|D]} | j dxs| j dxsd} | s.tj j#| } | j d"xs| j dxsdjxs5tjj| jddxsd#}|j%d| d|d|sd$t''z} d%d&j-|zd'z'| <| j)| C|j dD]} | j dxs| j dxsd} | s.tj j#| } | j d"xs| j dxsdjxs5tjj| jddxsd#}d$t''z} d(| d|d)'| <| j)| t/|}&'fd*}tjd+||tjtjz}tr t |d }|j d D]%}d,D]}|j2j5|d- '|j d.D]h}|jd sD|j7d/s3|j7ds"|j7d s|j+Y|j9jt/|}g((fd0}tjd1|tjrd2nd}d-}||k7r@|}tjd3||tjtjz}||k7r@tjd'd4|tj}tjd5d6|tj}tjd7d|tj}g}|jd8D]}|j}|stjd9d|}tjd:d|}|j;d;dj;dd=|}tjd?d|}t=j>|}|j;d@d=j;dAd=}tjdBd|}|stjdC|r'jAD]\}}|j;||}tC(D]\}}|j;dD|z|}tjdE|tjs*tjdF|tjrdG|z}|j%|jd8j-|}'jAD]\}}|j;||}tC(D]\}}|j;dD|z|}tjdHd|}t=j>|}tjdId6|}tjdJd8|}tjd>d=|}tEg}!} |jd8D]W} tjdKdtjd?d| } | r| | vr6| jG| |!j%| Yd8j-|!j}gtE}#}"&D].\}$}%|$|#vr |#jG|$|"j%|$|%f0|||"fS#t0$rYwxYw#t0$rYwxYw)LuR正文 HTML 清洗：保留

/

，附件转

绝对URL

段落rrz )flagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】 html.parseraThrefrGappendixz data-appendixz&\.(pdf|docx?|xlsx?|zip|rar|wps|et|ofd)z/module/download/oldsrcOLDSRCz^(?:javascript:|mailto:|tel:|#)r_titletitle?附件z__ATTACH__%d__ imgsrcz(icons?/|fileTypeImages/icon_)paltu图片z __IMG__%d__z

z
z

z

*s<00ELcR S rz&]*href="([^"]+)"[^>]*>(.*?)) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingNdivtablecfj|jddtdz zS)Nr __TBL__%d__r)rdrXre)rf table_protects r _tbl_replzhtml_to_text.._tbl_repls.QWWQZ(M 2Q 677rz ]*>rzz

z  z8]*>z z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  rz [ \t]{2,}r z z[\s\u200b\u200c\u200d\ufeff]+u^[【\[（(]?\s*(?:返回顶部|回到顶部|返回列表|打印(?:本页)?|关闭(?:窗口)?|分享|扫一扫|上一篇|下一篇)\s*[】\]）)]?$r}]]z

%s

z __(?:LINK|TBL|IMG|ATTACH)__\d+__z[ \t]+\nz\n{3,}r)$rrSrZrfind_allr1searchrrYget_textr]r^r_r`rarbrcrdre replace_with decomposejoinr7r6attrspopfindunwraprrritems enumeratesetadd))contentrm_souprErG_href_oldsrc real_hrefrhrirjrSrTrUimgssrcsrntagattrdr has_tableprevpartsblockbplainkvitbloutseenfinaldedupseen_uurrkrlr~s) ` @@@r html_to_textr^s 1byff]Brtt>CQXZ](^L%NN3'% (&~~e, $wwu~@):@b99>RTTJMMO $^^C( ,zz%(  6D#\!ggenG0AGR"$"(,,"6"6x"E"wwu~G1AGRNNPO!ww// #q0ABOFN WVY$Z[\+c,.??,1GLL4F,F,O S)s+ , ~~e, &ggen?(9?R ,,..x=wwu~?)9?RFFHGww'' #q(9:G>F#c,&77T[]`$a S!  % &%jGff> BDD244K1G !'=9E~~d+ .O.DIIMM$-. .^^E* zzz-affWo !u affSkKKMHHJ   %jGM8YY>AI D '/&&;Y rtt 5 '/ ffWj'>Gff\4=GffPRTBDD*G Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  88kmr s  &&( DAq !QA  . 2FAs -!+S1A 2yyArtt,9a1NaA QWWY/ 2 ++e C""$ 1kk!Q M*23kk-!+S12 &&4b# >C   C C &&dC (C &&FC (C &&sC (C%%D YYv ffVR B!:;cTk    Q  ++e  " " $C6E1 ;  1  aV   5  ]   F   sG-A3k7!F+k7 :k7k7Ck78Ek7>C l7 ll llcg}ts|St|d}|jdd}|s|S|jdD]}|jdtjd}|s-|jdd }|sC|j d xsd j }t|j d xs|jdd } | rt| dkrtjd|s|jdr t|z}n1|jds tjj||}d } |jdtjd} | r}tjd| jd } | rU| j!dd| j!dj#dd| j!dj#d} |j%|| | f|S)NrDr{z zfxxgk-listclass_trtdzzfxxgk-list-titlerETrFrGrrMrrKr%z/doc_[0-9a-f]+\.html/httpzzfxxgk-list-time(\d{4})-(\d{1,2})-(\d{1,2})r-r)rrrrcompiler1rrrrerr\BASE_URLrarbrcrXzfillrd) html_textrmrsouprrtd_titlerErGrMdatetd_timedms r parse_listrs E  M 2D ))GM) 2C  ll4 *774 3G(H7I  MM#DM ) f #**,AEE'NIajjDj.IJE Q yy0$7  ??3 d?D(<<''$7D''$rzz2E'F'G 97;K;KRV;K;WXB%'XXa["((1+2C2CA2FQR HYHYZ[H\] dE4()/*0 Lrcdx}x}}d}tr= t|d}|jddtjdtji}|r|j drt |d}|jddtjdtji}|r|j drptjd |d}|rU|jd d |jd jd d |jd jd }|s.|jd} | rt | jd}|s|jdtjd} | r{tjd| j}|rU|jd d |jd jd d |jd jd }|jdxs|jd} | r"djd| jD}|stjd|tjtjz} | ret | jd } tj d| Dcgc]}|j#s|}}|r|dj#n| }|smtjd|}|rU|jd d |jd jd d |jd jd }|||fS#t$rYwxYwcc}w)NrrDmetaname ArticleTitle)rrPubDatez!(\d{4})[-/](\d{1,2})[-/](\d{1,2})rrr%rh1TrKspanzdetails-release-daterrzdetails-content)idzdiv.details-contentc32K|]}t|yw)N)r7).0cs r zparse_detail..Os&C!s1v&Csz(.*?)z[-|]ru'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2}))rrrrrZr1rrrXrr select_onercontentsr6rr`r)rrmrMr content_htmlrmtmdrrspelrfrrUrs r parse_detailr5s"$$E$D< D  M:D6&"**_bdd2S)TUBbffY'#ByM26&"**Z2N)OPBbffY'YYCR ]S)+!bhhqk6G6G6JBHHUVKL]L]^_L`aDYYt_' $ (?@EYYvbjj9P.QYR#A2;;=QB-/XXa["((1+:K:KA:NPRPXPXYZP[PaPabcPde/0ZDOODY4ZB!ww&Cr{{&CC   II-y"$$+ F AGGAJ'A "! 4B1 QBEB(-E!HNN$1E  YYA9 M !#!bhhqk.?.?.BBHHQKDUDUVWDXYD $ $$    CsHd t@|r-tCjD|Dcgc] \}}||d c}}dnd d }tFr|jI|n|jI||dz }td|dd|||rdnd fzdtKjLdtFs|r||tg}tKjLddtFrltOtFdd5}|D]+}|jQtCjD|dd z- dddtd!tFt-|fzdn|r||ttd"|||fzdtd#tzdy#t($r} td|| fzdYd} ~ d} ~ wwxYwcc}}w#1swYxYw)$Nr)push_to_searchdbz"=== %s (%s) pages=%d cutoff=%s ===Tr'rz*%s?open=fdzdgknr&nr_page=%d&nr_per_page=%dzPage %d fetch error: %szPage %d: empty, stopzPage %d: no items, stopzPage %d: found %d itemsrrrrr z skip empty/placeholder: %s()rMr;F) ensure_ascii) site_name source_urlr;rMpub_datesummaryrcategorytags group_namerku + %s (%s, %d字%s)&u, 表格g?) batch_labelg333333?wr)r4rzJSONL written: %s (%d rows)u*新增: %d | 跳过: %d | CUTOFF截断: %dz=== %s done ===))r,sysr^insertr]dirnameabspath__file__ crawler_librr: SITE_NAME SCRIPT_NAME_PAGESCUTOFFrr+LIST_URLPER_PAGErAr6rrerrrrrrrrZCATEGORY GROUP_NAMEjsondumpsrrdr8r9openwrite)r,r new_count skip_count cutoff_count seen_urls jsonl_rowsbatchpager;rr@rrirMrdhtmldtitleddaterrratts plain_lenno_pararritemfrows rmainr_sHHOOArwwrwwx'@AB, .)[&RX1Y Y-.-I- \IJ Ea!$> 19C>(DRZA[[C c I (4/t < 9c* +d2$ ?  '4U*<>U!yyGRTTBBUKT} 1}4vcr{B$Oa &ggUr""(VZ $zz8<=1q+=E S`b D!!$' T" NI (F3BK 9B:,LLSW Y JJsOO' Pe U <E 3}>@ *cG 4 D! D 3U;dBC D D +z3z?.KKSWX  U < 6)ZQ]9^ ^fjk i 't4G  +tQi7t D  T>  D Ds* O+O,"11O2 O) O$$O)2O;__main__))1__doc__rr]rr8rhtmlr urllib.parserarrbs4r ImportErrorenvironr1rr3rrrrr CHANNEL_PATHrrnowstrftimerr* _MAX_PAGESrargvrrEr\intr` ValueErrorrerrrArrrr__name__rrrs ?>>>(!ZZ^^L" -   ?    %H l "  (,,.9'2 2 <        s5D<<E 8E<EE EEEE