Ë
    Í«£jY'  ã                   ó   — d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl	m
Z
 ddl	ZdZedz   ZdZdZd	d
dœZdZdZdZdZ ej*                  dd«      Zd„ Zd„ Zd„ Zd„ Zdd„Zd„ Zd„ Zd„ Zd„ Ze dk(  r e«        yy)uì   
crawl_ningguo_sthj.py - å®�å›½å¸‚ç”Ÿæ€�çŽ¯å¢ƒåˆ†å±€-å…¶ä»–ä¿¡æ�¯/é€šçŸ¥å…¬å‘Š
Site: www.ningguo.gov.cn
Branch: 383 (å®�å›½å¸‚ç”Ÿæ€�çŽ¯å¢ƒåˆ†å±€)
Channel: 22474 (å…¶ä»–ä¿¡æ�¯/é€šçŸ¥å…¬å‘Š)
WAF: Openness CMS token_verified cookie bypass
é    N©ÚBeautifulSoup)Úurljoinzhttps://www.ningguo.gov.cnz,/XxgkContent/showList/383/22474/page_{}.htmlÚningguo_sthju'   å®�å›½å¸‚ç”Ÿæ€�çŽ¯å¢ƒåˆ†å±€å…¶ä»–ä¿¡æ�¯z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36zzh-CN,zh;q=0.9)z
User-AgentzAccept-Languageé   é   g      à?é   Ú	SEARCH_DBz/root/search.dbc                  óZ   — dd l } | j                  t        «      }|j                  d«       |S )Nr   zPRAGMA journal_mode=WAL)Úsqlite3ÚconnectÚDB_PATHÚexecute)r   Úconns     úcrawl_ningguo_sthj.pyÚget_connr       s%   € ÛØ�?‰?œ7Ó#€DØ‡L�LÐ*Ô+Ø€Kó    c                 ó´  — t        t        «      D ]Ü  }	 t        j                  «       }|j                  j                  ddd¬«       |j                  | t        t        ¬«      }d|_	        |j                  dk(  r|j                  c S |j                  dk(  r,|t        d	z
  k  r t        j                  t        |d	z   z  «       Œ±t        d
|j                  › d| › �t         j"                  ¬«        y y# t$        $ r^}|t        d	z
  k  r t        j                  t        |d	z   z  «       n't        d| › d|› �t         j"                  ¬«       Y d}~ yY d}~�ŒBd}~ww xY w)z=Fetch with token_verified cookie bypass for Openness CMS WAF.Útoken_verifiedÚtruezwww.ningguo.gov.cn)Údomain)ÚheadersÚtimeoutzutf-8éÈ   i˜  é   z  [WARN] Status z for ©ÚfileNz  [WARN] Failed ú: )ÚrangeÚMAX_RETRIESÚrequestsÚSessionÚcookiesÚsetÚgetÚHEADERSÚTIMEOUTÚencodingÚstatus_codeÚtextÚtimeÚsleepÚSLEEPÚprintÚsysÚstderrÚ	Exception)ÚurlÚattemptÚsÚrÚes        r   Úfetchr7   '   s  € äœÓ%ò ˆð	Ü× Ñ Ó"ˆAØ�I‰I�M‰MÐ*¨FÐ;OˆMÔPØ—‘�c¤7´G�Ó<ˆAØ ˆAŒJØ�}‰} Ò#Ø—v‘v’Ø—‘ #Ò%¨'´KÀ!±OÒ*CÜ—
‘
œ5 G¨a¡KÑ0Ô1ØäÐ(¨¯©¨°u¸S¸EÐBÌÏÉÕTÙðð( øô ò 	Øœ q™Ò(Ü—
‘
œ5 G¨a¡KÑ0Õ1äÐ(¨¨¨R°¨sÐ3¼#¿*¹*ÕEÝõ 2ûð	ús%   ”A/C0Â:C0Ã+C0Ã0	EÃ9AEÅEc                 ó  — t        | d«      }g }t        «       }|j                  d«      D ]Õ  }|j                  dt	        j
                  d«      ¬«      }|sŒ,|j                  dd«      }t        t        |«      }||v rŒS|j                  |«       |j                  dd«      xs |j                  d	¬
«      }|sŒ�|j                  d«      }	|	r|	j                  d	¬
«      nd}
|j                  |j                  «       ||
f«       Œ× |S )z*Extract (title, url, date) from list page.úhtml.parserÚliÚaz/OpennessContent/show/\d+\.html©Úhrefr=   Ú ÚtitleT©ÚstripÚspan)r   r$   Úfind_allÚfindÚreÚcompiler%   r   ÚBASE_URLÚaddÚget_textÚappendrA   )ÚhtmlÚsoupÚitemsÚ	seen_urlsr:   r;   r=   Úfull_urlr?   rB   Údates              r   Úparse_list_pagerQ   @   sî   € ä˜˜}Ó-€DØ€EÜ“€IØ�m‰m˜DÓ!ò 6ˆØ�G‰G�CœbŸj™jÐ)KÓLˆGÓMˆÙØØ�u‰u�V˜RÓ ˆÜœ8 TÓ*ˆØ�yÑ ØØ�‰�hÔØ—‘�g˜rÓ"Ò< a§j¡j°t jÓ&<ˆÙØØ�w‰w�v‹ˆÙ,0ˆt�}‰} 4ˆ}Ô(°bˆØ�‰�e—k‘k“m X¨tÐ4Õ5ð6ð €Lr   c                 óF  — g }| j                  dd¬«      }|sy|j                  D ]ê  }|j                  dk(  r'|j                  d¬«      }|sŒ'|j	                  |«       Œ9|j                  dk(  r t        |«      }|sŒV|j	                  |«       Œh|j                  dk(  r,|j                  d¬«      }|sŒŒ|d	vsŒ‘|j	                  |«       Œ£t        |t        «      sŒ´|j                  «       sŒÅ|j                  «       }|d	vsŒÚ|j	                  |«       Œì d
j                  |«      S )z4Extract text paragraphs and tables from detail page.Údivzg-detailbox©Úclass_r>   ÚpTr@   Útable)õ	   å¯Œæ–‡æœ¬õ   æ–‡æœ¬ä¸‹è½½z

)
rD   ÚchildrenÚnamerI   rJ   Úhtml_table_to_htmlÚ
isinstanceÚstrrA   Újoin)rL   ÚpartsÚ	detailboxÚchildÚtxtÚmd_tables         r   Úextract_content_partsre   W   sú   € à€EØ—	‘	˜%¨�	Ó6€IÙØØ×#Ñ#ò "ˆØ�:‰:˜ÒØ—.‘. t�.Ó,ˆCÚØ—‘˜SÕ!Ø�Z‰Z˜7Ò"Ü)¨%Ó0ˆHÚØ—‘˜XÕ&Ø�Z‰Z˜5Ò Ø—.‘. t�.Ó,ˆCÚ�sÐ"?Ò?Ø—‘˜SÕ!Ü˜œsÕ#¨¯©­Ø—+‘+“-ˆCØÐ7Ò7Ø—‘˜SÕ!ð!"ð" �;‰;�uÓÐr   c                 óØ  — ddl m}  |t        | «      d«      }|j                  d«      D ]P  }|j	                  dd«      }|sŒ|j                  d«      rŒ*|r t        j                  j                  ||«      n||d<   ŒR |j                  d«      D ]P  }|j	                  d	d«      }|sŒ|j                  d
«      rŒ*|r t        j                  j                  ||«      n||d	<   ŒR t        |«      S )uA   ä¿�ç•™ HTML è¡¨æ ¼ç»“æž„ï¼Œä»…å°†ç›¸å¯¹é“¾æŽ¥/å›¾ç‰‡è½¬ç»�å¯¹ URLr   r   r9   r;   r=   r>   )ÚhttpÚ
javascriptú#ÚimgÚsrc)rg   z//zdata:)	Úbs4r   r^   rC   r%   Ú
startswithÚurllibÚparser   )rW   Úbase_urlr   Útblr;   r=   rj   rk   s           r   r\   r\   q   sÐ   € å!Ù
œ˜E›
 MÓ
2€CØ�\‰\˜#Óò SˆØ�u‰u�V˜RÓ ˆÚ˜Ÿ™Ð(CÕDÙ@HœŸ™×,Ñ,¨X°tÔ<ÈdˆAˆfŠIðSð �|‰|˜EÓ"ò RˆØ�g‰g�e˜RÓ ˆÚ�s—~‘~Ð&=Õ>Ù@HœŸ™×-Ñ-¨h¸Ô<ÈcˆC�ŠJðRô ˆs‹8€Or   c                 ó0  — g }| j                  dd¬«      D ]}  }|d   }t        j                  d|t        j                  «      sŒ.t	        ||«      }|j                  d¬«      xs |j                  d«      d   }|sŒe|d	vsŒj|j                  ||d
œ«       Œ |S )z*Extract file attachments from detail page.r;   Tr<   r=   z"\.(xls|xlsx|pdf|doc|docx|zip|rar)$r@   ú/éÿÿÿÿ)rY   rX   )r[   r2   )rC   rE   ÚsearchÚIr   rI   ÚsplitrJ   )rL   Úpage_urlÚattachmentsr;   r=   rO   r[   s          r   Úextract_attachmentsrz   €   s—   € à€KØ�]‰]˜3 Tˆ]Ó*ò DˆØ�‰yˆÜ�9‰9Ð:¸DÄ"Ç$Á$ÕGÜ˜x¨Ó.ˆHØ—:‘: D�:Ó)Ò@¨T¯Z©Z¸«_¸RÑ-@ˆDÚ˜Ð$AÒAØ×"Ñ"¨D¸Ñ#BÕCðDð Ðr   c                 óì  — t        | d«      }d}|j                  dd¬«      }|r'|j                  dd¬«      }|r|j                  d¬«      }|s<|j                  d	«      }|r)|j                  d¬«      }t        j                  d
d|«      }d}|rL|j                  dd¬«      }|r7t        j
                  d|j                  «       «      }	|	r|	j                  d«      }t        |«      }
t        ||«      }|||
|fS )zEExtract (title, publish_date, content, attachments) from detail page.r9   r>   rS   zm-detailboxrT   zu-titleTr@   r?   u(   \s*[-â€“â€”]\s*å®�å›½å¸‚äººæ°‘æ”¿åºœ\s*$zu-descz(\d{4}-\d{2}-\d{2})r   )	r   rD   rI   rE   Úsubru   Úgroupre   rz   )rK   rx   rL   r?   ÚmdetailÚutitleÚ	title_tagÚpublish_dateÚudescÚdmÚcontentry   s               r   Úparse_detailr…   �   sù   € ä˜˜}Ó-€Dð €EØ�i‰i˜ mˆiÓ4€GÙØ—‘˜e¨I�Ó6ˆÙØ—O‘O¨$�OÓ/ˆEÙØ—I‘I˜gÓ&ˆ	ÙØ×&Ñ&¨TÐ&Ó2ˆEÜ—F‘FÐFÈÈEÓRˆEð €LÙØ—‘˜U¨8�Ó4ˆÙÜ—‘Ð1°5·>±>Ó3CÓDˆBÙØ!Ÿx™x¨›{�ô $ DÓ)€Gô & d¨HÓ5€Kà�, ¨Ð4Ð4r   c                 óŒ  — t        «       }d}d}| D ]Ö  \  }}}}}	|r|dd nd}
|	rt        j                  |	d¬«      nd}	 |j                  d|f«      j	                  «       }|r|dz  }ŒW|j                  d	|||||||
|f«       |j                  d
«      j	                  «       d   }|r|j                  d||||
f«       |j                  «        |dz  }t        d|› d|dd › d�d¬«       ŒØ |j                  «        ||fS # t        $ r<}t        d|› d|› �t        j                  ¬«       |j                  «        Y d}~�Œ-d}~ww xY w)z&Insert items into gov_raw + FTS5 sync.r   Niô  r>   F)Úensure_asciiz'SELECT id FROM gov_raw WHERE page_url=?r   z·
                INSERT INTO gov_raw (site_name, title, content, publish_date, source_url, page_url, summary, attachments)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            zSELECT last_insert_rowid()zWINSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)z  [NEW] ú é<   ú...T)Úflushz  [ERROR] DB insert failed for r   r   )r   ÚjsonÚdumpsr   ÚfetchoneÚcommitr.   r1   r/   r0   ÚrollbackÚclose)rM   Ú	site_namer   Ú	new_countÚ
skip_countr?   r2   r„   r�   ry   ÚsummaryÚattach_jsonÚexistingÚ	record_idr6   s                  r   Úpush_to_searchdbr™   °   sm  € ä‹:€DØ€IØ€Jà:?ò Ñ6ˆˆs�G˜\¨;Ù#*�'˜$˜3‘-°ˆÙEP”d—j‘j ¸5ÕAÐVXˆð	Ø—|‘|Ø9¸C¸6óç‰h‹jð ñ Ø˜a‘�
Øà�L‰Lð ð ˜U G¨\¸3ÀÀWÈkÐZô\ð
 Ÿ™Ð%AÓB×KÑKÓMÈaÑPˆIÙØ—‘ØmØ  y°'Ð:ôð
 �K‰KŒMØ˜‰NˆIÜ�H˜\˜N¨!¨E°#°2¨J¨<°sÐ;À4ÖHð3ð< 	‡J�J„LØ�jÐ Ð øô ò 	ÜÐ3°C°5¸¸1¸#Ð>ÄSÇZÁZÕPØ�M‰M�OŠOûð	ús   ¿(C>Á(B C>Ã>	EÄ1D>Ä>Ec                  ó†  — dd l } | j                  d¬«      }|j                  dt        t        d¬«       |j                  «       }t        |j                  d«      }g }t        dt        › d	|› d
�«       t        d|dz   «      D �]Œ  }t        j                  |«      }t        d|› d|› d|› �«       t        |«      }|st        d|› d�«       ŒJt        |«      }t        dt        |«      › d�«       t!        |«      D ]ù  \  }	\  }
}}t        d|	dz   › dt        |«      › d|
d d › d�dd¬«       t        |«      }|st        d«       ŒJt#        ||«      \  }}}}|xs |
}|xs |}|}|rUg }|D ]   }|j%                  d|d   › d|d   › d�«       Œ" |r|d d!j'                  |«      z   z  }nd"d!j'                  |«      z   }|j%                  |||||f«       t        d#«       t)        j*                  t,        «       Œû t)        j*                  t,        «       �Œ� |st        d$«       y t/        «       }|j1                  d%t2        f«      j5                  «       d   }|j7                  «        t        d&t2        › d|› �«       t        d't        |«      › �«       t9        |t2        «      \  }}t/        «       }|j1                  d%t2        f«      j5                  «       d   }|j1                  d(t2        f«      j5                  «       d   }|j7                  «        t        d)«       t        d*|› d+|› �«       t        d,|› �«       t        d-|› �«       y ).Nr   u-   å®�å›½å¸‚ç”Ÿæ€�çŽ¯å¢ƒåˆ†å±€é€šçŸ¥å…¬å‘Šçˆ¬è™«)Údescriptionz--pageszPages to crawl)ÚtypeÚdefaultÚhelpé   z	Starting z crawl, z	 pages...r   z  Page rs   r   z  [SKIP] Page z failedz
    Found z itemsz    [z] é(   rŠ   rˆ   T)Úendr‹   z[SKIP] detail fetch failedú[r[   z](r2   ú)u   

**é™„ä»¶ï¼š**
ú
u   **é™„ä»¶ï¼š**
ÚokzNo items collected!z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z
Existing records for zNew items to insert: z1SELECT COUNT(*) FROM gov_search WHERE site_name=?z
=== Result ===z  New: z, Skipped: z  Total in gov_raw: z  FTS sync verified: )ÚargparseÚArgumentParserÚadd_argumentÚintÚ	MAX_PAGESÚ
parse_argsÚminÚpagesr.   ÚSOURCEr   ÚLIST_TPLÚformatr7   rQ   ÚlenÚ	enumerater…   rJ   r_   r+   r,   r-   r   r   Ú	SITE_NAMErŽ   r‘   r™   )r¦   ÚparserÚargsr­   Ú	all_itemsÚpager2   rK   rM   Úidxr?   Úlist_urlrP   Údetail_htmlÚdetail_titler�   r„   ry   Úfinal_titleÚ
final_dateÚfull_contentÚattach_linksÚattr   r—   r“   r”   ÚtotalÚ	fts_counts                                r   ÚmainrÃ   Ø   sM  € ÛØ×$Ñ$Ð1`Ð$Óa€FØ
×Ñ˜	¬´YÐEUÐÔVØ×ÑÓ€Dä�—
‘
˜BÓ€EØ€Iä	ˆI”f�X˜X e W¨IÐ
6Ô7ä�a˜ ™Ó#ó $ˆÜ�o‰o˜dÓ#ˆÜ�˜�v˜Q˜u˜g R¨ uÐ-Ô.Ü�S‹zˆÙÜ�N 4 &¨Ð0Ô1ØÜ Ó%ˆÜ�
œ3˜u›:˜, fÐ-Ô.ä,5°eÓ,<ò 	Ñ(ˆCÑ(�%˜ 4Ü�E˜#˜a™%˜ ¤# e£* ¨R°°c°r°
¨|¸3Ð?ÀSÐPTÕUÜ ›/ˆKÙÜÐ2Ô3Øä?KÈKÐYaÓ?bÑ<ˆL˜,¨°Ø&Ò/¨%ˆKØ%Ò-¨ˆJð #ˆLÙØ!�Ø&ò J�CØ ×'Ñ'¨!¨C°©K¨=¸¸3¸u¹:¸,ÀaÐ(HÕIðJáØ Ð$9¸D¿I¹IÀlÓ<SÑ$SÑS‘Là#4°t·y±yÀÓ7NÑ#N�Là×Ñ˜k¨8°\À:È{Ð[Ô\Ü�$ŒKÜ�J‰J”uÕð1	ô4 	�
‰
”5ÖðI$ñL ÜÐ#Ô$Øô ‹:€DØ�|‰|Ø8¼9¸,óç�hƒj�ñ€Hð 	‡J�J„Lä	Ð#¤I ;¨b°°
Ð
;Ô<Ü	Ð!¤# i£.Ð!1Ð
2Ô3ä,¨Y¼	ÓBÑ€Iˆzô ‹:€DØ�L‰LØ8¼9¸,óç�hƒj�ñ€Eð —‘Ø;¼i¸\óç�hƒj�ñ€Ið 	‡J�J„Lä	ÐÔÜ	ˆG�I�;˜k¨*¨Ð
6Ô7Ü	Ð   Ð
(Ô)Ü	Ð! ) Ð
-Õ.r   Ú__main__)r>   )!Ú__doc__r/   rE   r+   ÚosrŒ   r!   rl   r   Úurllib.parser   rn   rG   r¯   r³   r®   r&   r'   r    r-   rª   Úgetenvr   r   r7   rQ   re   r\   rz   r…   r™   rÃ   Ú__name__© r   r   ú<module>rË      s¶   ðñ÷ × Ó Û Ý Ý  Û à'€ØÐDÑD€Ø€	Ø	2€àPØ'ñ€ð €Ø€Ø€Ø€	à
ˆ"�)‰)�KÐ!2Ó
3€òòò2ò.ó4ò
ò 5òF%!òPN/ðb ˆzÒÙ…Fð r   