ujdZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl Z dZ dZ dZdZd Zd d d d ZdZdZdZdZdZedk(reyy)ut 昆明市东川区人民政府(www.kmdc.gov.cn) - 生态环境栏目 URL: http://www.kmdc.gov.cn/gsthj/ CMS: zcms (index_N.shtml 静态分页) 列表: div.data-table-item > p.w659 > a (标题) + p.w80 > a[title] (日期) 分页: index.shtml / index_N.shtml, 29页, 20条/页 详情: div.content > h1 + div.activity (正文, 含表格) 生产库 schema: gov_raw(id, site_name, source_url, page_url, title, publish_date, date_rank, summary, status, category, visits, content, tags, industry, attachments, group_name, has_table, script_name) gov_search: fts5(title, site_name, summary, tokenize=trigram) rowid 对齐 gov_raw.id N)unescape)urljoinzhttp://www.kmdc.gov.cnzhttp://www.kmdc.gov.cn/gsthj/u+昆明市东川区人民政府-生态环境zcrawl_kmdc_hjbh.pyu云南zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagect|xsd}tjdd|}tjdd|}tjdd|j}|S)u2清洗标题: strip ·  等实体前缀u^\s*(?:·|·| |\s)+u(\s*_\s*昆明市东川区人民政府.*$z\s+ )rresubstrip)ts $/root/gov_crawler/crawl_kmdc_hjbh.py clean_titler&sVbA 0"a8A :BBA vsA$$&A Hcg}tjdtj}|j|D]}|j d|j d|j dj }}}|j ds t||}d|vrft|}|st|r|ddnd }|sQtjd |} | r7| j dd | j dd | j dnd }|j|||ft} g} |D]/} | d| vs | j| d| j| 1| S) u1解析列表页, 返回 [(title, url, date), ...]z{

\s*]*>\s*([^<]+?)\s*\s*

\s*

\s*]*title="([^"]*)")zhttp://zhttps://z/c/N rz(\d{4})\.(\d{2})\.(\d{2})-) r compileSfinditergroupr startswithrrsearchappendsetadd) html_textbase_urlitemspatmhreftitletsdatedmseenoutits r parse_listr,/sP E ** A  C \\) $*''!*aggaj!''!*2B2B2DRe678T*D   E" r#2w"7CBCEbhhqk]!BHHQK="((1+?2D eT4()* 5D C a5  HHRUO JJrN Jrctjd|tj}|s%tjd|tj}|sdgfS|jd}tjd|}|Dcgc]4}tjd|tj s)t ||6}}||fScc}w)u3解析详情页, 返回 (content_html, attachments)zK

.*?
(.*?)
\s*
\s*z3
(.*?)
\s*\s*rrz]+href="([^"]+)"[^>]*>z0\.(?:pdf|docx?|xlsx?|zip|rar|wps|et|dps)(?:\?|$))r rrrfindallIr)rurlr#bodyattsas r parse_detailr4Ps `bkmomqmqrA IILiY[Y]Y] ^ 2v 771:D ::4d ;D%) uRYY7jlmoqosos-tGCO uD u : vs >*B=)B=cgfd}tjd||}tjdd|}tjdd|}tjdd|}|jdDcgc]}|j}}|Dcgc]}|s| }}dj |}t D]\}}|j d |d |}tjdd|}|jScc}wcc}w) u(HTML -> 文本, 保留表格、 分段chj|jddtdz dS)Nr@@TABLEr@@)rrlen)r#tabless r _save_tblzhtml_to_text.._save_tbl`s. aggaj!VQr**rzz"]*> z.]*>rz\n{3,} r7r8)r r splitr join enumeratereplace) rr;lnlinesr*textir r:s @r html_to_textrF]s F+0)YGI}|r5t;j@dd|jCd}tE|}|r|}tG|j2|\}}|jIst/d|ddtK|}|rdjM|nd}tOjP||ddzjSdjU}|r|ddnd}|j7dtV||||d |ddd |dd|tXd |vrdnd |f|j7d!|jZ|tV|f|j]|dz }t/d"|d#|dd$t_j`d&h|jct/d'|d(|d)|y#t,$r$} t/d| d| d| |dz }Yd} ~ d} ~ wwxYw#t,$r$} |dz }t/d%|ddd| Yd} ~ td} ~ wwxYw)*Nz--pagesru 抓取页数)typedefaulthelpz--dbz/mnt/data/search.dbuSQLite 数据库路径)rIrJz --script-name)rIz/mnt/rz#http://www.kmdc.gov.cn/gsthj/index_z.shtml)headerstimeoutzutf-8[u] 请求失败 z: u ] 列表 u 条 zASELECT COUNT(*) FROM gov_raw WHERE source_url=? AND script_name=?z&
\s*

(.*?)

z<[^>]+>ru 跳过空正文: r=izINSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, status, category, visits, content, tags, industry, attachments, group_name, has_table, script_name) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)otherzA AJ 1663' $yU E#78 (-& 2 $E7H% 2 _bikvawx<<>!$q(NI\\'7BG% YYH"''SUSWSWX B  Q XJacr |45G& 2N 3o7r JJL OI;j :i[ QRe  AdV?3%r!5 6 NI  X 2Q uSbzl"QC011 2s> #N0/N3 CN3$CN3 N0 N++N03 O <OO __main__)__doc__rQrjrYr r\rrhtmlr urllib.parserrbBASE_URLrarnrUrordrr,r4rFr__name__rr rs     # * 9 "  @O'  B &LS^ zFr