ujudZddlZddlZddlZddlZddlZddlmZmZejddZ dZ dZ e dzZ d Z ejed z jd Zdd ZddZdZdZedk(reyy)uF乌兰察布察哈尔高新区-通知公告 (tzgg) - uses curl for APIN)datetime timedelta SEARCH_DBz/root/search.dbu+乌兰察布察哈尔高新区-通知公告zhttps://che.wulanchabu.gov.cnz^/rmt/tmpl/api/site/467045ff03994199b783c61da198b941/channelid/5b95810da38f4254b57e6d45e4db895d`5186ec152b6fdbf8f9925254821388e35f569179f80020ffd8a62f429078cef3261c7fef0b91d53fa73a3c6163670aea718fbf0304bac398d71847f294dca445cbec7fd627bdfb3fdca3f1eee07c5564bc35faadc6215e8761f16b528cff19678232098cc8d525003f03c4556dd2c379ff438b917bbbccd91e2ac3c1f256efe701ecd379aae27643ae4ed1342128b7eeb4d5878b8cba9868df5ac4ac63ebd6e8d34f4fcc60ae1cce16d06efef6b1b3ee96239a410222a06b02dcab0b8bf1069cacbc31875c748b7ab014547e9731b1fd057da8f919dd244d53714a5a8a185be89faa83f9f9e1581a8a1bbae5f9b8576f43cec72bcb6ab99b53a0e053830a993026175cb57f5554ec136ae9e6f4e092aa75f35a73a6b61116ecea4c9e56a73ac474804d9955c2a677f599be3a347289c2iG)days%Y-%m-%dcgd}|r|ddgz }|ddgz }|d|gz }n|ddgz }|j|tj|ddd}|jS) N)curlz-sz --max-time30z-Hz,Content-Type: application/json;charset=utf-8zUser-Agent: Mozilla/5.0z-dT#)capture_outputtexttimeout)append subprocessrunstdout)urldatacmdresults /root/gov_crawler/crawl_che.pyr r sq ,C  DEE /00 d| /00JJsO ^^C4 LF ==ctj|dtd}tt|}|sgS tj |}|j dgS#gcYSxYw)N)currpagepagesizeparamsr)jsondumpsPARAMSr APIloadsget)pagebodyrawrs r fetch_listr(sY ::4R6J KD sD/C  zz#xx## s &AA ctd|d}t|}tjd|}|r|j dj nd}|sPtjd|}|r|j dj nd}tj dd|}d}d D]a}tjd |zd z|tj}|s1|j dj }t|d kDs_|}n|dk(rfd D]a}tjd|zd z|tj}|s1|j dj }t|d kDs_|}n|dk7rbtj dd|tj}tj dd|tj}|j }|sd}d} tjd|}|r!|j djdd} | s)tjd|}|r|j d} || |fS)N/tzgg/.htmlz]*>([^<]+)z([^<]+)u5\s*-\s*乌兰察布察哈尔高新技术开发区\s*$ 正文为空)article_contentz article-concontentrarticlezclass="z"[^>]*>(.*?)2)r0r1maintextzoomzid="z9]*style="[^"]*display:\s*none[^"]*"[^>]*>.*?)flagsz]*>.*?u?(?:发布时间|发布日期)[：:]\s*(\d{4}[-/]\d{2}[-/]\d{2})/-z!(\d{4}-\d{2}-\d{2})\s*\d{2}:\d{2}) BASEr researchgroupstripsubDOTALLlenreplace) docnorhtmlmtitler0selectorcid_namedates r fetch_detailrI%s F&u %C 9D *D1A"#AGGAJ   E  II/ 6&'   "ROQSUZ[GT IIj8+.BBD")) T    "A1v{ . A G 'G+.BBD"))TAGGAJ$$&q6B;G  . &&UWY[bjljsjst&&5r7"))T--/. D TVZ[A##C-$  II:D A QWWQZd $ rcddl}|jt}|j}|j dt d}d}|dzdz}t d|d}d}td|dzD]}t |} | st d|d nd} | D]R} | jd d j} | jd d } | jd d}|r(tj|dz jd}nd }| r| sy|r|tkr|dz }td| d}|j d|f|jr|dz } t!| \}}}|dk(s|s|dz }|j dt"||xs| |xs||t%|j'dd df|j(dkDr%|dz }| dz } t d|d|d| ddd|n|dz }U|j-t d|d| d |d!|j/t d"|d#|d$y#t*$r'}t d|d| ddd||dz }Yd}~d}~wwxYw)%NrzPRAGMA journal_mode=WALr,rz Total pages: z Page z: empty response, stoppingrDr-rA inputTimeirr*r+z'SELECT id FROM gov_raw WHERE page_url=?r.z}INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, publish_date, content, date_rank, category) VALUES (?,?,?,?,?,?,?)r7u 通知公告z P[z] r z | z[ERR] z: z done: z new (total )u 结果: u 新增, u 跳过)sqlite3connectrcursorexecuter(printranger$r<r fromtimestampstrftimeCUTOFFr8fetchonerI SITE_NAMEintr@rowcount Exceptioncommitclose)rQconncuritems_p1 total_count total_pages total_new total_skippedr%itemspage_newitemrDrAtsrHpage_url detail_titlepub_dater0es rmainrpQs ??9 %D ++-CLL*+!}HK#*K M+ '(IMaq)3I4  GD6!;< = ( #DHHWb)//1EHHWb)E+q)B--b4i8AA*Mv " veWE2H KKAH; O||~"  #2>u2E/ hn,G!Q&M T,*?%AQTSZ\_`d`l`lmpqs`t\uxFG<rysM$$$( BIIk#4 5 9 & mm l  (,,.9%0 0 : :: F  * XCCJ zFr