~j VdZddlZddlZddlZddlZddlZddlZddlZddlZejdddl m Z m Z dZ ejddD]IZejdr eej#dddZ 1ej'sBeeZ Ke ede ej*d d Zd Zd Ze j2e d z j5dZdeddddZedZdZdZdZ dZ!dZ"dZ#d dZ$e%dk(re$e xsdyy#e$rYwxYw)!ut 吉安井冈山经开区 - 企业动态 站点: jkq.jian.gov.cn/news-list-qiyedongtai.html CMS: 自建PHP + AJAX分页 (同 jkqjian_tzgg 公告公示模板) 列表: POST /api-ajax_list-{page}.html, ajax_type[]=70_news,22,70,news,Y-m-d,40,20,0,"" (20条/页) 详情: news-show-{id}.html (div.text-main 正文) 用法: python3 crawl_jkq_jian_qiyedongtai.py [--pages=N | N] Nignore)datetime timedeltaz--pages==z[AutoPg] max_pages= SEARCH_DBz/root/search.dbu%吉安井冈山经开区-企业动态zhttp://jkq.jian.gov.cniG)daysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36/news-list-qiyedongtai.htmlz!application/x-www-form-urlencodedXMLHttpRequest) User-AgentRefererz Content-TypezX-Requested-Withz/api-ajax_list-22c 0ddtfddddddd d g } tjt|d t|d d }|j dk7rt d|d|j gS|j}g}|jdgD]}|jdd}|jdxsdj}|jdxsdj}|jdxsdj} |r|stjd|s1|jdxsdj} | r td| d }|j|||| d|S#t$r} t d|d| gcYd} ~ Sd} ~ wwxYw)N) ajax_type[]70_newsr)r70)rnews)rzY-m-d)r40)r20)r0)r)is_ds1z.htmlF)headersdatatimeoutverifyz [WARN] z HTTP rurlrtitle inputtime descriptionznews-show-\d+\.html$idz /news-show-)r r!datedescz err: )CATIDrequestspostAPI_LISTHEADERS status_codeprintjsongetstripresearchBASE_URLappend Exception) pagerrjitemsdr r!r%r&sides //root/gov_crawler/crawl_jkq_jian_qiyedongtai.pyfetch_list_pager>)s"]E$:} | j d } | jds&|j| t| z}@t j d|tjD]R} | j d } | jdr t| z} |j| j dd| d}Tt jdd|tjtjz}t jdd|}t jdd|j} | s d|vsd|vs|jcSy)uvdiv.show_content / div.text-main / 微信 rich_media_content 平衡匹配, 附件/图片 urljoin 绝对化, 保留

zclass="show_content"rzclass="text-main"zclass="text_main"zclass="rich_media_contentrz rz>z(?:href|src)="([^"]+)"/z$]*src="([^"]+\.pdf)"[^>]*>z

PDF原文

z<(script|style)[^>]*>.*?)flagsz]*>z

<[^>]+>zB7A A||C(IIaA67%LaQSQUQUVUA A||C($qL !''!* QC?R.STA U FF;R"))TVTXTX.YFF;q1vvj"a0668:?fk779$+%, ct|}|syd}tjd|}|r|jdj }|s[tjd|tj }|r4tj dd|jdj }|s[tjd|tj }|r4tj dd|jdj }d}tjd|}|r|jddd }|sDtjd |dd }|r)tj d d |jddd }t||}|s|d|dSttj dd|j }d|v}|dkr|s|d|dS|||dS)Nrz/]*>(.*?)rIz(.*?)u [-_—]\s*.*$z*E  II-y")) D FF+R1A1A1CDEH ?KA771:cr?  IIN O vvlC 结束r%u页: u 条 (?z ~ )u 该页已全部早于截断日u，停止翻页g333333?u📊 列表共 u 条u 窗口内(>= z): rr rdr!r)r!rnrdrerng333333?u ✅ Result: z stored, z skipped)r-rsrMmin_MAX_PGr>rNmaxextendCUTOFFtimesleeprjr/r) max_pages all_itemspgitsrdateskeptstoredskippedr~s r=runrs+ GI;  IAs9g&:;a?@ b! E"/0 1 &)8RZF88 bTs3xjUs5z.LC^cPSTYPZilOmmnop SZ&( 4VHOL M  3  OC N+4 01" M2"V*6 f8LB MD M OF83s4yk 67FW bi( zz)$!'{BvJSUceflcmnF /8R[w!'N!;!Ir&z~vr )  aKF qLG 4  L '( ;<39 Ns G>&G>H.H__main__)r)rf)&__doc__r1rtsysrr.r(warningsrxfilterwarningsrrrargv_arQintsplitr5isdigitr-getenvrrsr3nowstrftimerr+r*r'r>rBr_rjrr__name__r`r=rse""!(  ((12,B }}Z  "((3*1-.G b'  y )* BIIk#4 5 3 # (,,.9%0 0 : :: FD677(   Z ' >&PCB0!=F z',QC   s#D  D('D(