uj1<dZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z dZdZdZdZdZd Zej(j+d d Zej(j+d d Z ddlZ e j2j4j7ddddZdZeej@D]k\Z!Z"e"jGdr e$e"jKdddZ4e"dk(s:e!dze'ej@ksS e$ej@e!dzZmedk\rendZ(eej@D]6\Z!Z"e"dk(s e!dze'ej@ks%ej@e!dzZ8d dZ)dZ*dZ+dZ,dZ-dZ.dZ/dZ0e1dk(re0yy#e$rY wxYw#e&$rYwxYw#e&$rYwxYw)!u= crawl_hejian_sthj.py - 河间市-生态环境 https://www.hejian.gov.cn/hejian/c107617/list.shtml 列表: table.zb_tab > tr > td.name_detail > a (标题CSS截断,须详情页取完整) 每行4td: 标题 | 来源 | 分类 | 日期 分页: list_{N}.shtml (createPageHTML('page_tag',50,1,'list','shtml',1000) → 20页) 详情: meta ArticleTitle(完整标题) + meta PubDate(真实日期) + div.file_detail 正文 排除: div.location_ breadcrumb / ul.category 元数据 / div.operation 打印按钮 附件: a href 相对路径 {hash}/files/xxx.zip → urljoin 转绝对 N) BeautifulSoupzhttps://www.hejian.gov.cnz3https://www.hejian.gov.cn/hejian/c107617/list.shtmlz:https://www.hejian.gov.cn/hejian/c107617/list_{page}.shtmlu河间市-生态环境u河北zcrawl_hejian_sthj.pyDB_PATHz/mnt/data/search.db JSONL_PATHzkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagez--pages==z--pagesz--jsonlc<t|D]4} tj|tdd}d|_|j cSy #t $rN}||dz k(r'td|d|tjYd}~y tjd Yd}~d}~wwxYw) NF)headerstimeoutverifyutf-8rz [WARN] z: )filerg@) rangerequestsgetHEADERSencodingtext Exceptionprintsysstderrtimesleep)urlretriesires &/root/gov_crawler/crawl_hejian_sthj.py fetch_pager#Bs 7^   S'2eLA AJ66M    GaK #b,3::> JJsOO  s/A B )B<BBc2|sytj|}|jddjddjddjdd}tjdd|}tjdd|}|j S) Nru​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts r" clean_titler.Ps !A (B''"5==hKSST\^`aA Er1MA vsAA 779ctjd|}|rHt|jd}t|jd}|dzdz}t d|Sy)u从 createPageHTML('page_tag', 每页条数, 当前页, 'list', 'shtml', 总数) 提取总页数 ⚠️ 河间实际每页 20 条(非参数值50), 总数1000 → 50页zzcreatePageHTML\s*\(\s*['\"]page_tag['\"]\s*,\s*(\d+)\s*,\s*\d+\s*,\s*['\"][^'\"]*['\"]\s*,\s*['\"][^'\"]*['\"]\s*,\s*(\d+)r)r*searchintgroupmax) html_textmper_page total_itemspagess r"get_total_pagesr=Zsc PR[ \Aqwwqz?!''!*o r!b(1e} r/cg}t|d}|jdd}|s|S|jdD]}|jd}t|dkr$|djd d }|s=|j d d }t |j d }t |dj d } |r|stjd|r|}tjjt|} tjd| } | r| jdnd } |j| || f |S)N html.parsertablezb_tabclass_trtdraThrefrIrr,z \.{3,}\s*$(\d{4}-\d{2}-\d{2})r)rfindfind_alllenrr.get_textr*r4urllibparseurljoin LIST_URL_P1r6append) r8itemssouptbrDtdsrGrItitledateabs_urldms r" parse_listr^gs( E M 2D 78 ,B  kk$-kk$ s8a<  FKK$K ' uuVR AJJTJ233q6???675  99]E *E,,&&{D9 YY-t 4 rxx{b gud+,#-$ Lr/cd}d}d}t|d}|jddtjdtji}|r|j drt |d}|jddtjdtji}|rK|j dr:tjd |dj}|r|jd }|jd d } | rZdjd| jD}tjdd|tjtjz}|||fS)Nrr?metaname ArticleTitle)attrscontentzPubDate|publishdaterLrdiv file_detailrBc32K|]}t|yw)N)str).0cs r" zparse_detail..s;!s1v;sz]*>.*?flags)rrMr*compileIrr.matchr,r6joincontentsr+S) r8page_urlrZr[ content_htmlrWmtmdr]els r" parse_detailrys  E DL M 2D 6&"**_bdd*K!L MB bffYByM* 6&"**5KRTT*R!S TB bffY XX,bm.A.A.C D 88A;D 5 /B ww;r{{;; vv2B BDDSUSWSWKX $ $$r/c r |sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}gitjdd|}tjd d|}t |d }|j d d D]h}|j dsG|j ds6tjd|j ddtjs\|d}|j dxs|}|jd xsW|j dxsD|j dxs1tjj|jdd}tjj|}|j!ds|j!drd|zn|}j#||fdt%d}d|d|d|<|j'|k|j dD]} | j d } | s| jd r+g} | D]} | j d!xs| j dxsd} | s.tjj| }| j#d|dtjj| jddxsd"d| sd#t%d}d$d%j)| zd&z|<| j'| t+|}fd'}tjd(||tjtjz} t |d }|j d D]%}d)D]}|j,j/|d 't+|}g fd*}tjd+|tjrd,nd}tjd-||tjtjz}tjd&d.|tj}tjd/d0|tj}tjd1d|tj}g}|jd2D]W}|j3}|stjd3d|}tjd4d|}|j5d5dj5d6d7}tjd8d7|}tjd9d|}t7j8|}|j5d:d7j5d;d7}tjdd|}tjd?d|}tjd@d|}tjdAd|}tjdBd0|}tjdCd2|}tjd8d7|}t?}g}|jd2D]X} tjdDdtjd9d| }|s2||vr7|jA||j#| Zd2j)|j3}||fS#t0$rYwxYw)ENrrz rlz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】u'发布(?:时间|日期)[:：][^<]{0,30}r?rGTrHappendixz data-appendixz&\.(pdf|docx?|xlsx?|zip|rar|wps|et|ofd)rIoldsrcrJ_titlerZ?zhttp://zhttps:////http: __ATTACH____ pimgsrcu图片__IMG__z

z
z

c|jd}|jdry|jd}tjdd|}t j |j }tjdd|j }|s5tjj|jddxsd }tjj|}|jd s|jd rd |zn|}j||fd td}d|d|d|<|S)Nrz javascript:rr1<[^>]+>u^附件[:：]\s*r~ru附件rrr__LINK__rrrr)r6 startswithr*r+r'r(r,ospathbasenamesplitrQrRrSrUrO) r9rIinnertxtr\key attachments link_protectrts r" _link_replz html_to_text.._link_replswwqz ??= ) ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6!!"9:+2+=+=d+Cg'GGS>*\*+2.'y0B3%tL S r/z&]*href="([^"]+)"[^>]*>(.*?)) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingchj|jddtdz dS)Nr__TBL__rr)rUr6rO)r9 table_protects r" _tbl_replzhtml_to_text.._tbl_repls3QWWQZ(]+A-.b11r/z ]*>rz]*>.*?z

z  z4]*>z z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  r&z [ \t]{2,}r z z[\s\u200b\u200c\u200d\ufeff]+rz __LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r%)!r*r+rsrorrNrr4rPrrrrrQrRrSrrUrO replace_withrqrhrcpoprr,r)r'r(rV enumeratesetadd)!rdrt_souprGrI real_hrefrr\rrimgssrcsrrrtagattrr has_tablepartsblockbplainkvrtbloutseenfinalrrrs! ` @@@r" html_to_textrs: 1byff]Brtt/6/A/A$/G'G+W   ~ .s<014C"+G94Fse4 PL  NN3   ^^C $zz%    .D ~ggen?(9?R$ll228SAGKK)G94FrwwGWGWX[XaXabeXfghXiGjGvnvFww{ |}  ~ L 12"5$)GLL,>$>$G S!s# $%jG"ff> G[][_[_bdbfbf[fgG g}5>>$' *CH * dD) * *e*M2YY>AIff/G244RTRVRV;WGffWj'>Gff\4=GffLbRYacaeaefG Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  &&( DAq !QA  . 0FAs GA3b/3/A 0 QWWY% ( ++e C""$ 1kk!Q M*03kkGA3b/3/0 &&!2s +C &&"c *C &&"c *C &&#R -C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&}    s-A`)) `65`6c t|jd|tf}|jry d} ddlm}||}|jd|||||tt t||f }|jdkDrdtjdd|} tjd d | jdd } |jd |j|t| fy y#t $rYwxYw#tj$r=} dt| vr*t!j"dt%||||||cYd} ~ Sd} ~ wwxYw)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?Fotherr)classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)rrr%r&zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)TlockedrK)execute SITE_NAMEfetchone crawler_librr GROUP_NAME SCRIPT_NAMErowcountr*r+r, lastrowidsqlite3OperationalErrorrhrr store_record) connrrZrdr[rcurindustryrsummaryr!s r"rrs6 ,,PSVXaRb cC ||~  5(/Hll +#ugtY KQZ\d e   <dgt=j>dtrmtAtdd 5}|D]+}|jCtEjF|d!"d#z- dddtd$tdt!|d%d | |jI|jKtd&|d td'|d td(t4d)d y#1swYxYw)*N<)r zPRAGMA busy_timeout=60000zPRAGMA journal_mode=WALrz Total pages: z , running: T)flushr)pagezPage z fetch errorz: found z itemszEmpty page, stop paginationrrz] z skip empty/placeholder: ()rtrZrd publish_dater site_name group_name script_namez JSONL + z ()z + g?g?wr)rF) ensure_asciirzJSONL written: z rows)u新增: u跳过: z=== z done ===)&rrconnectrrrr#rTr=min_PAGESrr LIST_URL_PNformatr^rOrryrr*r+r,r4rorUrrrrrropenwritejsondumpscommitclose)r new_count skip_count seen_urls jsonl_rows first_html total_pages pages_to_runrrr8rVr\rZr[dhtmldtitleddaterurdratts plain_lenno_paraokfrows r"mainr=s D w3 01 ./IJIJK(J!*-Kv{+L M+k, @Ma)*2!QYkK,>,>D,>,I"&!)JC  E$|,D 9 9% dV8CJfETR    HYK - HZL !. D 9 %T2 D Ds 1M66M?__main__))2__doc__rrr*rrrhtmlr' urllib.parserQrbs4rBASE_URLrTrrrrenvironrrrrequests.packages.urllib3packagesurllib3disable_warningsrr _MAX_PAGESrargvrrGrr5r ValueErrorrOrr#r.r=r^ryrrr__name__r/r"rs   &C J $  $ **..$9 : ZZ^^L" -  $ ..0 @O'  chh  DAq||J QWWS!_Q/0J iAECM1 SXXa!e_-J "QA chh %DAqI~!a%#chh-/XXa!e_ %   6%,A'HBQ3h zF[          s62(F F)F4F&%F&)F10F14F=<F=