Ë
    ¸¬uj+  ã                   óº  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
m
Z
mZ dZdZdZd	Zd
Z e
j"                  «        ed¬«      z
  j%                  d«      ZddiZ ej*                  «       Zej.                  j1                  e«       de_        ddlZ ej6                  ej8                  j:                  «       d„ Zd„ Zd„ Z d„ Z!d„ Z"e#dk(  r e"«        yy)u<   Crawler for hbj.hbdawu.gov.cn - è¡Œæ”¿è®¸å�¯ column (JEECMS)é    N)ÚBeautifulSoup)Úurljoin)ÚdatetimeÚ	timedeltazhttp://hbj.hbdawu.gov.cnz)http://hbj.hbdawu.gov.cn/xzxk/index.jhtmlz/root/search.dbé   u   hbj.hbdawu.gov.cn-è¡Œæ”¿è®¸å�¯iG  )Údaysz%Y-%m-%dz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36Fc                 óf   — t         g}t        d| dz   «      D ]  }|j                  d|› d�«       Œ |S )z8Generate list page URLs: index.jhtml, index_2.jhtml, ...é   é   z$http://hbj.hbdawu.gov.cn/xzxk/index_z.jhtml)ÚLIST_URLÚrangeÚappend)Ú	num_pagesÚurlsÚis      ú/root/gov_crawler/crawl_dawu.pyÚget_page_urlsr       s@   € äˆ:€DÜ�1�i !‘mÓ$ò FˆØ�‰Ð:¸1¸#¸VÐDÕEðFà€Kó    c                 ó  — t        | d«      }|j                  dd¬«      }|st        d«       g S g }|j                  dd¬«      D ]¿  }|j                  d	d
¬«      }|j                  d«      }|r|sŒ,|j	                  dd«      }|r|j                  d«      s|dk(  rŒWt        t        |«      }|j	                  d«      xs |j                  d¬«      j                  «       }	|j                  d¬«      }
|j                  |	||
f«       ŒÁ |S )z2Parse list page, return list of (title, url, date)úhtml.parserÚulÚnewslist©Úidz  [WARN] ul#newslist not foundÚliF)Ú	recursiveÚaÚbt_link)Úclass_ÚspanÚhrefÚ Ú
javascriptú#ÚtitleT©Ústrip)r   ÚfindÚprintÚfind_allÚgetÚ
startswithr   ÚBASE_URLÚget_textr'   r   )ÚhtmlÚsoupr   Úitemsr   r   r    r!   Úfull_urlr%   Údates              r   Ú
parse_listr4   (   sü   € ä˜˜}Ó-€DØ	�‰�4˜JˆÓ	'€BÙÜÐ.Ô/Øˆ	Ø€EØ�k‰k˜$¨%ˆkÓ0ò .ˆØ�G‰G�C 	ˆGÓ*ˆØ�w‰w�v‹ˆÙ™ØØ�u‰u�V˜RÓ ˆÙ�t—‘ |Ô4¸ÀºØÜœ8 TÓ*ˆà—‘�w“Ò9 1§:¡:°D :Ó#9×@Ñ@ÓBˆØ�}‰} 4ˆ}Ó(ˆØ�‰�e˜X tÐ,Õ-ð.ð €Lr   c                 ó*  — t        | d«      }|j                  ddd„ ¬«      }d}|rd|j                  d¬«      }t        j                  d	t        |«      t        j                  «      }|r |j                  d
«      j                  «       }n|}|j                  dd¬«      }|st        d|› �«       |dfS t        |«      }t        j                  dd|«      }t        j                  dd|«      }t        j                  dd|«      }|j                  «       }||fS )zJExtract content from detail page. Returns (title_from_page, content_html).r   ÚtdÚcenterc                 ó   — | r| xr d| v S dS )Nzfont-size:20pxF© )Úvs    r   ú<lambda>zparse_detail.<locals>.<lambda>E   s   € Ñ_`ÀÒ@[ÐFVÐZ[ÐF[€ Ðfk€ r   )ÚalignÚstyler"   Tr&   u7   <!--<\$\[æ ‡é¢˜\]>begin-->(.*?)<!--<\$\[æ ‡é¢˜\]>end-->r   ÚdivÚcontent_txtr   z  [WARN] No div#content_txt in z<!--caiji-contenttxt-start-->z<!--caiji-contenttxt-end-->z\s*\n\s*ú
)r   r(   r.   ÚreÚsearchÚstrÚDOTALLÚgroupr'   r)   Úsub)	r/   Úurlr0   Ú	title_tagÚdetail_titleÚtxtÚmatchÚcontent_divÚcontent_htmls	            r   Úparse_detailrN   @   s  € ä˜˜}Ó-€Dð —	‘	˜$ hÑ6k�	Ól€IØ€LÙØ× Ñ  tÐ Ó,ˆä—	‘	ÐTÔVYÐZcÓVdÔfh×foÑfoÓpˆÙØ Ÿ;™; q›>×/Ñ/Ó1‰LàˆLð —)‘)˜E m�)Ó4€KÙÜÐ/°¨uÐ5Ô6Ø˜RÐÐô �{Ó#€LÜ—6‘6Ð:¸BÀÓM€LÜ—6‘6Ð8¸"¸lÓK€Lô —6‘6˜+ t¨\Ó:€LØ×%Ñ%Ó'€Là˜Ð%Ð%r   c                 ó’  — t        | d«      }g }|j                  dd¬«      D ]¢  }|d   }|j                  d«      r7t        j                  d|t        j
                  «      r|j                  |«       ŒP|j                  d«      sŒbt        j                  d|t        j
                  «      sŒˆ|j                  t        ||«      «       Œ¤ |S )	z*Extract attachment links from detail HTML.r   r   T)r!   r!   Úhttpz"\.(pdf|doc|docx|xls|xlsx|zip|rar)$ú/)r   r*   r,   rA   rB   ÚIr   r   )r/   Úbase_urlr0   Úattachmentsr   r!   s         r   Úextract_attachmentsrU   b   s«   € ä˜˜}Ó-€DØ€KØ�]‰]˜3 Tˆ]Ó*ò 8ˆØ�‰yˆØ�?‰?˜6Ô"¤r§y¡yÐ1VÐX\Ô^`×^bÑ^bÔ'cØ×Ñ˜tÕ$Ø�_‰_˜SÕ!¤b§i¡iÐ0UÐW[Ô]_×]aÑ]aÕ&bØ×Ñœw x°Ó6Õ7ð8ð Ðr   c                  ób  — dd l } t        t        «      }g }t        dt        › dt        |«      › d�«       |D ]b  }	 t        j                  |d¬«      }d|_        t        |j                  «      }t        d|› d	t        |«      › d
�«       |j                  |«       Œd t        «       }g }|D ]/  }	|	d   |vsŒ|j                  |	d   «       |j                  |	«       Œ1 t        dt        › dt        |«      › �«       g }
|D ]$  \  }}}	 |t         k\  r|
j                  |||f«       Œ& t        dt        › dt         › dt        |
«      › �«       t#        j$                  t&        «      }|j)                  «       }|j+                  dt        f«       |j-                  «       d   }t        dt        › d|› �«       d}d}d}|
D ]â  \  }}}|j+                  d|f«       |j-                  «       d   dkD  r|dz  }Œ6	 t        j                  |d¬«      }d|_        t/        |j                  |«      \  }}|st        d|› �«       |dz  }Œ„|xs |}|j+                  dt        ||||t        |«      dk  rdn|d d f«       |j1                  «        |dz  }dd l}|j5                  d«       Œä t        dt        › d|› d|› d|› d�	«       |j+                  dt        f«       |j-                  «       d   }t        dt        › d|› �«       |j7                  «        t        ||||t        |«      d œ}t        d!| j9                  |d"¬#«      › �«       y # t        $ r}t        d|› d|› �«       Y d }~�Œ)d }~ww xY w#  |
j                  |||f«       Y �ŒƒxY w# t        $ r!}t        d|› d|› �«       |dz  }Y d }~�Œïd }~ww xY w)$Nr   ú[z] Scanning z	 pages...é   )Útimeoutzutf-8z  z -> z itemsz  [ERR] z: r   z
[z] Total unique items: z] Items within 3 years (>z): z0SELECT COUNT(*) FROM gov_raw WHERE site_name = ?z] Existing records: z/SELECT COUNT(*) FROM gov_raw WHERE page_url = ?z  [ERR] Fetch z  [SKIP] Empty content: ztINSERT OR IGNORE INTO gov_raw (site_name, title, content, page_url, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)éÈ   r"   g      à?z] Done: z new, z
 skipped, z errorsz] Total in DB: )ÚsiteÚnewÚskippedÚerrorsÚtotalÚpages_scannedz
JSON_OUTPUT:F)Úensure_ascii)Újsonr   Ú	MAX_PAGESr)   Ú	SITE_NAMEÚlenÚsessionr+   Úencodingr4   ÚtextÚextendÚ	ExceptionÚsetÚaddr   ÚCUTOFF_DATEÚsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚfetchonerN   ÚcommitÚtimeÚsleepÚcloseÚdumps)rb   Ú	page_urlsÚ	all_itemsÚpuÚrespr1   ÚeÚseenÚunique_itemsÚitemÚfilteredr%   rG   r3   ÚconnÚcÚexisting_countÚ	new_countÚ
skip_countÚerror_countrI   rM   Úfinal_titleru   r_   Úsummarys                             r   ÚmainrŠ   o   sß  € ÛäœiÓ(€IØ€Iä	ˆAŒiˆ[˜¤C¨	£NÐ#3°9Ð
=Ô>Øò (ˆð	(Ü—;‘;˜r¨2�;Ó.ˆDØ#ˆDŒMÜ˜tŸy™yÓ)ˆEÜ�B�r�d˜$œs 5›z˜l¨&Ð1Ô2Ø×Ñ˜UÕ#ð(ô ‹5€DØ€LØò &ˆØ�‰7˜$ÒØ�H‰H�T˜!‘WÔØ×Ñ Õ%ð&ô
 
ˆC”	ˆ{Ð0´°\Ó1BÐ0CÐ
DÔEð €HØ(ò 0Ñˆˆs�Dð	0Ø”{Ò"Ø—‘ ¨¨TÐ 2Ô3øð0ô 
ˆAŒiˆ[Ð1´+°¸cÄ#ÀhÃ-ÀÐ
QÔRô �?‰?œ7Ó#€DØ�‰‹€Að ‡I�IÐ@Ä9À,ÔOØ—Z‘Z“\ !‘_€NÜ	ˆAŒiˆ[Ð,¨^Ð,<Ð
=Ô>ð €IØ€JØ€Kà$ò "Ñˆˆs�Dà	�	‰	ÐCÀcÀVÔLØ�:‰:‹<˜‰?˜QÒØ˜!‰OˆJØð	Ü—;‘;˜s¨B�;Ó/ˆDØ#ˆDŒMÜ)5°d·i±iÀÓ)EÑ&ˆL˜,ñ ÜÐ,¨U¨GÐ4Ô5Ø˜!‰OˆJØð #Ò+ eˆð 	
�	‰	ð CÜ˜ \°3¸ÄCÈÓDTÐX[ÒD[¹bÐalÐmqÐnqÐarÐsô	
ð 	�‰ŒØ�Q‰ˆ	ãØ�
‰
�3�ðE"ôH 
ˆC”	ˆ{˜( 9 +¨V°J°<¸zÈ+ÈÐV]Ð
^Ô_ð ‡I�IÐ@Ä9À,ÔOØ�J‰J‹L˜‰O€EÜ	ˆAŒiˆ[˜¨ wÐ
/Ô0à‡J�J„Lô ØØØØÜ˜Y›ñ€Gô 
ˆN˜4Ÿ:™: g¸E˜:ÓBÐCÐ
DÕEøôy ò 	(Ü�H˜R˜D  1 #Ð&×'Ò'ûð	(ûð&	0Ø�O‰O˜U C¨Ð.×/ûô< ò 	Ü�N 3 % r¨!¨Ð-Ô.Ø˜1ÑˆKÝûð	ús<   »AM ÄM(Ç17NÍ 	M%Í	M Í M%Í(NÎ	N.ÎN)Î)N.Ú__main__)$Ú__doc__Úrequestsrn   rA   ÚsysÚosÚbs4r   Úurllib.parser   r   r   r-   r   rp   rc   rd   ÚnowÚstrftimerm   ÚHEADERSÚSessionrf   ÚheadersÚupdateÚverifyÚurllib3Údisable_warningsÚ
exceptionsÚInsecureRequestWarningr   r4   rN   rU   rŠ   Ú__name__r9   r   r   ú<module>rž      sè   ðá Bã Û Û 	Û 
Û 	Ý Ý  ß (à%€Ø6€Ø
€Ø€	Ø,€	Øˆx�|‰|‹~¡	¨wÔ 7Ñ7×AÑAÀ*ÓM€ð ð  Dð€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ Ø€„ã Ø €× Ñ ˜×+Ñ+×BÑBÔ Còòò0&òD
òjFðZ ˆzÒÙ…Fð r   