Ë
    ¡Š£j¹'  ã                   ó8  — d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	 dZ
dZdZdZdZd	Z ej                   d
d«      Zde
dz   dœZ ej&                  «       Zej*                  j-                  e«       h d£Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uã   é˜³æ±Ÿå¸‚äººæ°‘æ”¿åºœ - å…¬å‘Šå…¬ç¤º çˆ¬è™«
URL: http://www.yangjiang.gov.cn/yj/ywdt/gggs/index.html â†’ index_{page}.html
ç³»ç»Ÿ: å¹¿ä¸œçœ�ç»Ÿä¸€æ”¿åºœCMS (NFCMS)
è¯¦æƒ…å®¹å™¨: div.zw#zoomcon â†’ <p>æ®µè�½ï¼Œdiv.fj â†’ é™„ä»¶
é    N)ÚBeautifulSoupzhttp://www.yangjiang.gov.cnzwww.yangjiang.gov.cnu"   é˜³æ±Ÿå¸‚äººæ°‘æ”¿åºœ-å…¬å‘Šå…¬ç¤ºu   å…¬å‘Šå…¬ç¤ºu   å¹¿ä¸œé   Ú	SEARCH_DBz/root/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z/yj/ywdt/gggs/)z
User-AgentÚReferer>)   ÚpÚddÚdlÚdtÚh1Úh2Úh3Úh4Úh5Úh6ÚhrÚliÚolÚtdÚthÚtrÚulÚdivÚnavÚpreÚformÚmainÚasideÚtableÚtbodyÚtfootÚtheadÚcenterÚdialogÚfigureÚfooterÚheaderÚhgroupÚaddressÚarticleÚcaptionÚdetailsÚsectionÚfieldsetÚ
blockquoteÚ
figcaptionc                 óÒ   ‡‡‡— | €yddl }ddlmŠ g Šˆˆˆfd„Š ‰| «       dj                  ‰«      }|j	                  dd|«      }|j	                  dd	|«      }|j                  «       S )
ul   å�—çº§è¾¹ç•Œå‡ºæ�¢è¡Œã€�è¡Œå†…èŠ‚ç‚¹ç›´æŽ¥æ‹¼æŽ¥ã€�<br> å‡ºæ�¢è¡Œï¼ˆâ‰ˆ æµ�è§ˆå™¨çœ‹åˆ°çš„æ�¢è¡Œç»“æž„ï¼‰ã€‚NÚ r   )ÚNavigableStringc                 óF  •— | j                   D ]‘  }t        |‰«      r‰j                  t        |«      «       Œ*t	        |dd «      dk(  r‰j                  d«       ŒLt	        |dd «      t
        v r+‰j                  d«        ‰|«       ‰j                  d«       ŒŠ ‰|«       Œ“ y )NÚnameÚbrú
)ÚchildrenÚ
isinstanceÚappendÚstrÚgetattrÚ_BLOCK_TAGS)ÚnodeÚchr2   ÚoutÚwalks     €€€ú$/root/gov_crawler/crawl_yangjiang.pyr@   zbody_text.<locals>.walk4   s„   ø€ Ø—-‘-ò 
	ˆBÜ˜"˜oÔ.Ø—
‘
œ3˜r›7Õ#Ü˜˜V TÓ*¨dÒ2Ø—
‘
˜4Õ Ü˜˜V TÓ*¬kÑ9Ø—
‘
˜4Ô Ù�R”Ø—
‘
˜4Õ á�R•ñ
	ó    z[ \t\r\f\v]*\n[ \t\r\f\v]*r6   z\n{3,}ú

)ÚreÚbs4r2   ÚjoinÚsubÚstrip)ÚelÚ_reÚtr2   r?   r@   s      @@@rA   Ú	body_textrL   ,   sb   ú€ à	€zØÛÝ#Ø
€Cöñ 	ˆ„HØ
�‰�‹€AØ�‰Ð-¨t°QÓ7€AØ�‰�	˜6 1Ó%€AØ�7‰7‹9ÐrB   c                 ó<   — t        | t        j                  d¬«       y )NT)ÚfileÚflush)ÚprintÚsysÚstderr)Úmsgs    rA   ÚlogrT   G   s   € Ü	ˆ#”C—J‘J dÖ+rB   c                 óÞ  — | dk(  r
t         › d�}nt         › d| › d�}	 t        j                  |d¬«      }d|_        t	        |j
                  d«      }g }|j                  d	«      }|st        d
| › �«       |S |j                  d«      D ]º  }|j                  d«      }|j                  dd¬«      }|sŒ*|j                  dd«      }	|j                  d«      xs |j                  d¬«      }
|r|j                  d¬«      nd}d|	vrt        d|	dd › �«       Œ�|	j                  d«      s	t         |	z   }	|j                  |
|	|f«       Œ¼ |S # t        $ r}t        d| › d|› �«       g cY d}~S d}~ww xY w)z2Fetch list page, return list of (title, url, date)é   z/yj/ywdt/gggs/index.htmlz/yj/ywdt/gggs/index_z.htmlé   ©Útimeoutúutf-8úhtml.parserzul.listz  [WARN] No ul.list on page r   ÚaÚspanÚtime©Úclass_Úhrefr1   ÚtitleT©rH   z/yj/ywdt/gggs/content/z)  [SKIP] External/different domain link: Né<   Úhttpz  [ERR] List page z	 failed: )ÚBASE_URLÚsessionÚgetÚencodingr   ÚtextÚ
select_onerT   Úfind_allÚfindÚget_textÚ
startswithr9   Ú	Exception)ÚpageÚurlÚrÚsoupÚitemsr   r   r\   r]   ra   rb   ÚdateÚes                rA   Ú
fetch_listrx   K   sx  € àˆq‚yÜ�
Ð2Ð3‰ä�
Ð.¨t¨f°EÐ:ˆðÜ�K‰K˜ RˆKÓ(ˆØˆŒ
Ü˜QŸV™V ]Ó3ˆàˆØ�_‰_˜YÓ'ˆÙÜÐ.¨t¨fÐ5Ô6ØˆLà—+‘+˜dÓ#ò 	2ˆBØ—‘˜“ˆAØ—7‘7˜6¨&�7Ó1ˆDÚØ—u‘u˜V RÓ(�ØŸ™˜g›Ò@¨!¯*©*¸4¨*Ó*@�Ù48�t—}‘}¨4�}Ô0¸b�à+°4Ñ7ÜÐCÀDÈÈ"ÀIÀ;ÐOÔPØØ—‘ vÔ.Ü# d™?�DØ—‘˜e T¨4Ð0Õ1ð	2ð ˆøÜò ÜÐ    i°¨sÐ3Ô4Ø�	ûðús+   �AE Á6:E Â1BE Å	E,ÅE'Å!E,Å'E,c                 ó\  ‡‡— 	 t         j                  | d¬«      }d|_        t        |j                  d«      }|j                  d«      }|st	        d	| › �«       yd
}|j                  d«      }|r|j                  d¬«      }|s]|j                  ddt        j                  dt        j                  «      i¬«      }|r$|j                  d«      r|d   j                  «       }d
}|j                  d«      }	|	rK|	j                  d¬«      }
t        j                  d|
«      }|r!|j                  d«      j!                  dd«      }|s`|j                  ddt        j                  dt        j                  «      i¬«      }|r'|j                  d«      r|d   j                  «       dd }g }g }|j                  d«      }|rk|j#                  dd¬«      D ]!  }t%        |«      }|sŒ|j'                  |«       Œ# |j#                  d«      D ]   }|j'                  dt)        |«      › d �«       Œ" |j                  d!«      }|r†|j#                  d"d#¬$«      D ]p  }|j                  d%d
«      Š|j                  d¬«      }‰sŒ*‰j+                  d&«      s	t,        ‰z   Št/        ˆfd'„|D «       «      rŒY|j'                  ‰|xs d(d)œ«       Œr |r±|j#                  d"d¬*«      D ]›  }|d%   }t        j                  d+|j1                  «       «      sŒ-|j+                  d&«      r|n|j+                  d«      r	t,        |z   nd
Š‰sŒ_t/        ˆfd,„|D «       «      rŒt|j'                  ‰|j                  d¬«      xs d(d)œ«       Œ� d-j3                  |«      }||||fS # t        $ r}t	        d| › d|› �«       Y d}~yd}~ww xY w).zBFetch detail page, return (title, date, content_text, attachments)rW   rX   rZ   z  [ERR] Detail fetch failed: ú - Nr[   zdiv.acticlez  [WARN] No div.acticle found: r1   r   Trc   Úmetar4   ÚArticleTitle)ÚattrsÚcontentz	span.timez(\d{4}[-/]\d{1,2}[-/]\d{1,2})rV   ú/ú-zpubdate|PubDateé
   zdiv.zw#zoomconr   F)Ú	recursiver   u	   [è¡¨æ ¼]
u
   
[/è¡¨æ ¼]zdiv.fjr\   rN   r_   ra   re   c              3   ó.   •K  — | ]  }|d    ‰k(  –— Œ y­w©rr   N© )Ú.0Úattra   s     €rA   ú	<genexpr>zfetch_detail.<locals>.<genexpr>°   s   øè ø€ ÒE°#˜3˜u™:¨Õ-ÑEùó   ƒu   é™„ä»¶)rr   r4   )ra   z%\.(pdf|doc|docx|xls|xlsx|zip|rar|et)$c              3   ó.   •K  — | ]  }|d    ‰k(  –— Œ y­wr„   r…   )r†   r‡   Úfull_urls     €rA   rˆ   zfetch_detail.<locals>.<genexpr>¹   s   øè ø€ Ò'VÀ3¨¨E©
°hÕ(>Ñ'Vùr‰   rC   )rg   rh   ri   rp   rT   r   rj   rk   rm   rn   rD   ÚcompileÚIrH   ÚsearchÚgroupÚreplacerl   rL   r9   r:   ro   rf   ÚanyÚlowerrF   )rr   rs   rw   rt   r)   rb   r   ÚmtÚpubdateÚ	time_spanÚ	time_textÚ
date_matchÚmeta_pubÚcontent_partsÚattachmentsÚzoomÚchildrj   r   Úfjr\   r4   ÚhÚcontent_textr‹   ra   s                           @@rA   Úfetch_detailr    r   s€  ù€ ðÜ�K‰K˜ RˆKÓ(ˆØˆŒ
ô
 ˜Ÿ™ Ó/€DØ�o‰o˜mÓ,€GÙÜÐ-¨c¨UÐ3Ô4Øð €EØ	�‰�dÓ	€BÙ	Ø—‘ $�Ó'ˆÙØ�Y‰Y�v f¬b¯j©j¸Ì"Ï$É$Ó.OÐ%PˆYÓQˆÙ�"—&‘&˜Ô#Ø�y‘M×'Ñ'Ó)ˆEð €GØ×"Ñ" ;Ó/€IÙØ×&Ñ&¨TÐ&Ó2ˆ	Ü—Y‘YÐ?ÀÓKˆ
ÙØ ×&Ñ& qÓ)×1Ñ1°#°sÓ;ˆGÙØ—9‘9˜V¨F´B·J±JÐ?QÔSU×SWÑSWÓ4XÐ+Y�9ÓZˆÙ˜Ÿ™ YÔ/Ø˜yÑ)×/Ñ/Ó1°#°2Ð6ˆGð €MØ€Kà×ÑÐ.Ó/€DÙØ—]‘] 3°%�]Ó8ò 	+ˆEÜ˜UÓ#ˆDÚØ×$Ñ$ TÕ*ð	+ð —]‘] 7Ó+ò 	GˆEØ× Ñ  :¬c°%«j¨\¸Ð!EÕFð	Gð 
×	Ñ	˜HÓ	%€BÙ	Ø—‘˜S¨�Ó0ò 	PˆAØ—5‘5˜ Ó$ˆDØ—:‘: D�:Ó)ˆDÚØ—‘ vÔ.Ü# d™?�DÜÓE¸ÔEÕEØ×&Ñ&¨t¸TÒ=MÀXÑ'NÕOð	Pñ Ø—‘˜s¨�Ó.ò 	fˆAØ�&‘	ˆAÜ�y‰yÐAÀ1Ç7Á7Ã9ÕMØ !§¡¨VÔ 4™1È1Ï<É<ÐX[ÔK\¼8Àaº<Ðbd�Ú¤CÓ'VÈ+Ô'VÕ$VØ×&Ñ&¨xÀÇÁÐRVÀÓAWÒAcÐ[cÑ'dÕeð	fð —;‘;˜}Ó-€Là�'˜<¨Ð4Ð4øôO ò ÜÐ+¨C¨5°°A°3Ð7Ô8Üûðús   „N Î	N+ÎN&Î&N+c           
      óœ  — 	 t        j                  t        d¬«      }| j                  d«      xs dd d }| j                  d«      xs dd d }| j                  d«      xs d}| j                  d	«      xs dd d
 }| j                  d«      xs dd d }t	        j
                  | j                  d«      xs g d¬«      }|j                  d|f«      j                  «       }|r)|j                  d|d   f«       |j                  d|f«       |j                  d|||||||f«       |j                  d«      j                  «       d   }	|r|d d n|d d }
|j                  d|	|||
f«       |j                  «        |j                  «        t        d|d d › �«       y# t        $ r+}t        d| j                  dd«      › d|› �«       Y d }~yd }~ww xY w)Nr�   rX   rb   r1   iô  Úpage_urliè  r~   Úpublish_dater   Ú	site_nameÚunknownéd   rš   F)Úensure_asciiz,SELECT rowid FROM gov_raw WHERE page_url = ?z&DELETE FROM gov_search WHERE rowid = ?r   z&DELETE FROM gov_raw WHERE page_url = ?z�INSERT INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?)zSELECT last_insert_rowid()zWINSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)u     âœ… rW   Tz  [ERR] DB import failed for z: )Úsqlite3ÚconnectÚDB_PATHrh   ÚjsonÚdumpsÚexecuteÚfetchoneÚcommitÚcloserT   rp   )ÚrecordÚdbrb   r¢   r~   r£   r¤   Úattachments_strÚoldÚ	new_rowidÚsummaryrw   s               rA   Úimport_to_dbr·   Á   sÈ  € ðÜ�_‰_œW¨bÔ1ˆØ—‘˜GÓ$Ò*¨¨D¨SÐ1ˆØ—J‘J˜zÓ*Ò0¨b°%°4Ð8ˆØ—*‘*˜YÓ'Ò-¨2ˆØŸ
™
 >Ó2Ò8°b¸#¸2Ð>ˆØ—Z‘Z Ó,Ò9°	¸4¸CÐ@ˆ	ÜŸ*™* V§Z¡Z°Ó%>Ò%DÀ"ÐSXÔYˆà�j‰jÐGÈ(ÈÓU×^Ñ^Ó`ˆÙØ�J‰JÐ?À#ÀaÁ&ÀÔKØ�J‰JÐ?À(ÀÔMà
�
‰
ð ^Ø�H˜g |°YÀÈ/ÐZô	
ð —J‘JÐ;Ó<×EÑEÓGÈÑJˆ	Ù#*�'˜$˜3‘-°°d°s°ˆØ
�
‰
ÐlØ˜u i°Ð9ô	;à
�	‰	ŒØ
�‰Œ
Üˆf�U˜3˜B�Z�LÐ!Ô"ØøÜò ÜÐ+¨F¯J©J°w¸rÓ,BÐ+CÀ2ÀaÀSÐIÔJÜûðús   ‚FF Æ	GÆ !GÇGc                  óT  — dd l } | j                  d¬«      }|j                  dt        dd¬«       |j	                  «       }d}t        d|j                  dz   «      D �]  }t        d	|› d
�«       t        |«      }|st        d«        nät        dt        |«      › d�«       t        |d«      D ]•  \  }\  }}}	t        d|› dt        |«      › d|d d › d�«       	 t        |«      }
|
st        d«       ŒE|
\  }}}}|xs |}|xs |	}|||||t        t        t        dœ}t        |«      }|r|dz  }t#        j$                  d«       Œ— ||j                  k  sŒùt#        j$                  d«       �Œ t        dt        › d|› d�«       y # t         $ r}t        d|› d|› �«       Y d }~Œud }~ww xY w)Nr   u(   é˜³æ±Ÿå¸‚äººæ°‘æ”¿åºœ-å…¬å‘Šå…¬ç¤ºçˆ¬è™«)Údescriptionz--pagesé   u6   çˆ¬å�–å‰�Né¡µï¼ˆæ¯�é¡µçº¦20æ�¡ï¼Œé»˜è®¤5é¡µ=~100æ�¡ï¼‰)ÚtypeÚdefaultÚhelprV   u   ðŸ“„ ç¬¬u   é¡µ...u     â†’ æ— æ›´å¤šæ•°æ�®ï¼Œç»“æ�Ÿu     â†’ u    æ�¡z  (r   z) é(   z...z  [SKIP] Could not fetch detail)rb   r¢   r£   r~   rš   r¤   ÚcolumnÚprovinceu     [ERR] è¯¦æƒ…é¡µå¤„ç�†å¤±è´¥: rz   g333333Ó?g      ø?u   
âœ… u    çˆ¬å�–å®Œæˆ�ï¼Œå…±å…¥åº“ )ÚargparseÚArgumentParserÚadd_argumentÚintÚ
parse_argsÚrangeÚpagesrT   rx   ÚlenÚ	enumerater    ÚSITEÚCOLUMNÚPROVINCEr·   rp   r^   Úsleep)rÁ   ÚparserÚargsÚcountrq   ru   Úidxrb   rr   rv   ÚresultÚdetail_titler”   r~   rš   Úfinal_titleÚ
final_dater±   Úokrw   s                       rA   r   r   á   sÓ  € ÛØ×$Ñ$Ð1[Ð$Ó\€FØ
×Ñ˜	¬°QÐ=uÐÔvØ×ÑÓ€Dà€EÜ�a˜Ÿ™ a™Ó(ó *ˆÜˆh�t�f˜FÐ#Ô$Ü˜4Ó ˆÙÜÐ0Ô2Ùäˆf”S˜“Z�L Ð%Ô&ä'0°¸Ó':ò 	Ñ#ˆCÑ#�%˜˜dÜ�#�c�U˜!œC ›J˜< r¨%°°¨*¨°SÐ9Ô:ðCÜ% cÓ*�ÙÜÐ9Ô;Øà>DÑ;�˜g w°Ø*Ò3¨e�Ø$š_¨�
ð )Ø #Ø$.Ø&Ø#.Ü!%Ü$Ü (ñ	�ô " &Ó)�ÙØ˜Q‘J�Eô �J‰J�s�Oð=	ð@ �$—*‘*ÓÜ�J‰J�sŽOðU*ôX ˆ&”�Ð0°°°tÐ<Õ=øô ò CÜÐ5°c°U¸#¸a¸SÐA×BÑBûðCús   ÃFÃ1<FÆ	F'ÆF"Æ"F'Ú__main__)Ú__doc__Úrequestsr«   rQ   ÚosrD   r^   r¨   rE   r   rf   ÚDOMAINrÊ   rË   rÌ   ÚPER_PAGEÚgetenvrª   ÚHEADERSÚSessionrg   ÚheadersÚupdater<   rL   rT   rx   r    r·   r   Ú__name__r…   rB   rA   ú<module>rã      sÍ   ðñó Û Û 
Û 	Û 	Û Û Ý à(€Ø	€Ø+€Ø	€Ø€Ø€Ø
ˆ"�)‰)�KÐ!2Ó
3€ð DØÐ*Ñ*ñ€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ ò*€òò6,ò$òNL5ò^ò@3>ðl ˆzÒÙ…Fð rB   