Ë
    †É…j]  ã                   óB  — d Z ddlZddlZ eej
                  «      dkD  r2ej
                  d   j                  «       r eej
                  d   «      ndZe� e	d e
e«      z   «       ddlZddlZddlZddlZddlmZmZ ddlmZ  ej$                  dd«      Zd	Zd
ZdZdZ ej0                  «        ed¬«      z
  j3                  d«      Z e	de› �«       ddddœZ ej8                  «       Zej<                  j?                  e«       d„ Z d„ Z!d„ Z"d„ Z#e$dk(  r e#«        yy)uø   
é�’é˜³åŽ¿äººæ°‘æ”¿åºœ - å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·å®¡æ‰¹
CMS: è‡ªå®šä¹‰æ”¿åºœCMS (OpennessContent + AJAXåˆ—è¡¨)
åˆ—è¡¨AJAX: /OpennessTarget/672/100097/page_N.html (N=1~29)
è¯¦æƒ…: /OpennessContent/show/ID.html
æ€»è®¡: 567æ�¡, 29é¡µ, 20æ�¡/é¡µ
é    Né   z[AutoPg] max_pages=)ÚdatetimeÚ	timedelta)ÚBeautifulSoupÚ	SEARCH_DBz/root/search.dbu   ahqy.gov.cn-çŽ¯è¯„å®¡æ‰¹zhttps://www.ahqy.gov.cnz>https://www.ahqy.gov.cn/OpennessTarget/672/100097/page_{}.htmlé   iG  )Údaysz%Y-%m-%dzCutoff date: zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagec                 ó¦   — 	 t         j                  | d¬«      }d|_        |j                  S # t        $ r}t        d| › d|› �«       Y d }~y d }~ww xY w)Né   ©Útimeoutzutf-8z  [ERR] fetch failed: z - )ÚsessionÚgetÚencodingÚtextÚ	ExceptionÚprint)ÚurlÚrÚes      ú/root/gov_crawler/crawl_ahqy.pyÚfetchr   +   sR   € ðÜ�K‰K˜ RˆKÓ(ˆØˆŒ
Ø�v‰vˆøÜò ÜÐ& s e¨3¨q¨cÐ2Ô3Üûðús   ‚), ¬	AµAÁAc                 ól  — g }t        | d«      }|j                  d«      }|D �]  }|j                  d«      }t        |«      dk  rŒ$t        |«      dkD  r|d   j	                  d«      nd}|r|j                  d«      sŒ\|d   j                  «       }|j                  d	¬
«      }|j                  d«      s&|j                  d«      r	t        |z   nt        dz   |z   }d}	t        |«      dkD  r-|d   j                  d	¬
«      }	t        j                  d|	«      sd}	|sŒø|sŒû|j                  |||	f«       �Œ |S )z—
    Parse the AJAX list HTML and return list of (url, title, date_str).
    Items from table rows with OpennessContent/show links and cwrq dates.
    úhtml.parserztable trÚtdé   r   ÚaNÚhrefT©ÚstripÚhttpú/Ú é   z\d{4}-\d{2}-\d{2})r   ÚselectÚfind_allÚlenÚfindr   r!   Úget_textÚ
startswithÚBASE_URLÚreÚmatchÚappend)
ÚhtmlÚitemsÚsoupÚrowsÚtrÚtdsÚa_tagr   ÚtitleÚdate_strs
             r   Ú
parse_listr9   5   s$  € ð
 €EÜ˜˜}Ó-€DØ�;‰;�zÓ"€DØó 2ˆØ�k‰k˜$ÓˆÜˆs‹8�aŠ<Øô %(¨£H¨q¢L��A‘—‘˜CÔ °dˆÙ˜EŸI™I fÔ-ØØ�V‰}×"Ñ"Ó$ˆØ—‘ T�Ó*ˆØ�‰˜vÔ&Ø&*§o¡o°cÔ&:”8˜d’?ÄÈ3ÁÐQUÑ@UˆDàˆÜˆs‹8�aŠ<Ø˜1‘v—‘¨T�Ó2ˆHä—8‘8Ð0°(Ô;Ø�Ú’TØ�L‰L˜$  xÐ0Ö1ð+2ð, €Ló    c                 ó6  — t        | d«      }d}|j                  dddi¬«      }|r$|j                  d«      r|d   j                  «       }|s%|j	                  d«      }|r|j                  d	¬
«      }|st        d|› �«       yd}|j                  dddi¬«      }|r=|j                  d«      r,t        j                  d|d   «      }|r|j                  d«      }|j	                  d«      }	|	st        d|› d�«       d}
nt        |	«      j                  «       }
|||
fS )z=Parse detail page, return (title, publish_date, content_html)r   NÚmetaÚnameÚArticleTitle)ÚattrsÚcontentzdiv.u-titleTr    z  [ERR] no title for )NNNr$   ÚPubDatez(\d{4}-\d{2}-\d{2})r   zdiv#zoomz  [WARN] no content div for z, saving title only)r   r)   r   r!   Ú
select_oner*   r   r-   ÚsearchÚgroupÚstr)r0   r   r2   r7   Ú
meta_titleÚtitle_elÚpub_dateÚ	meta_dateÚmÚcontent_divÚcontent_htmls              r   Úparse_detailrM   V   s)  € ä˜˜}Ó-€Dð €EØ—‘˜6¨&°.Ð)A�ÓB€JÙ�j—n‘n YÔ/Ø˜9Ñ%×+Ñ+Ó-ˆñ Ø—?‘? =Ó1ˆÙØ×%Ñ%¨DÐ%Ó1ˆEáÜÐ% c UÐ+Ô,Øð €HØ—	‘	˜&¨°Ð(;�	Ó<€IÙ�Y—]‘] 9Ô-Ü�I‰IÐ,¨i¸	Ñ.BÓCˆÙØ—w‘w˜q“zˆHð —/‘/ *Ó-€KÙÜÐ,¨S¨EÐ1DÐEÔFØ‰ä˜;Ó'×-Ñ-Ó/ˆà�(˜LÐ(Ð(r:   c                  óZ  — dd l } | j                  t        d¬«      }|j                  d«       |j                  d«       |j	                  «       }d}d}d}t        dt        t        t        xs t        «      dz   «      D �]Î  }t        j                  |«      }t        d|› dt        › d	�«       t        |«      }|st        d
|› d�«       ŒLt        |«      }	|	st        d
|› d�«        �ngt        dt        |	«      › d�«       d}
d}|	D �]  \  }}}|r|t        k  r|dz  }|dz  }Œt        |«      }|st        d|d d › �«       Œ=t!        ||«      \  }}}|sŒP|r(|t        k  r|dz  }|dz  }t        d|› d|d d › �«       Œzd}|rt#        |j%                  dd«      «      }	 |j                  dt&        |||||df«       |j(                  dkD  r#|
dz  }
|dz  }|
dz  dk(  r|j+                  «        n|dz  }t/        j0                  d«       �Œ |j+                  «        t        d|› d|
› d|› d|› d|› d�«       t/        j0                  d«       �ŒÑ t        dd › �«       t        d!|› d|› d"|› d#�«       |dkD  rQt        d$«       	 |j                  d%t&        › d&�«       |j                  d'«       |j+                  «        t        d(«       |j3                  «        t        d*«       y # t,        $ r}t        d|› �«       Y d }~�Œd }~ww xY w# t,        $ r}t        d)|› �«       Y d }~Œ]d }~ww xY w)+Nr   é<   r   zPRAGMA journal_mode=WALzPRAGMA busy_timeout=10000r   z

--- Page r#   z ---z  [SKIP] page z fetch failedz	 no itemsz  Found z itemsz  [SKIP] detail fetch: é(   u     è·³è¿‡(è¶…3å¹´): ú ú-r$   zƒINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category) VALUES (?, ?, ?, ?, ?, ?, ?)Úhjxxé
   z  [ERR] insert: g333333Ó?z  Page z: +z new, z skipped (total +z skip)ú
z2==================================================zFinal: z dup, z skippedz
Rebuilding FTS index...z(DELETE FROM gov_search WHERE site_name='ú'z4INSERT INTO gov_search(gov_search) VALUES('rebuild')z  FTS index rebuilt.z  [ERR] FTS rebuild: zDone.)Úsqlite3Úconnectr   ÚexecuteÚcursorÚrangeÚminÚTOTAL_PAGESÚ_MAX_PGÚAPI_TPLÚformatr   r   r9   r(   ÚCUTOFFrM   ÚintÚreplaceÚ	SITE_NAMEÚrowcountÚcommitr   ÚtimeÚsleepÚclose)rW   ÚconnÚcurÚ	total_newÚ
total_skipÚ	total_dupÚpage_numÚpage_urlr0   r1   Úpage_newÚ	page_skipr   Ú
list_titleÚ	list_dateÚdetail_htmlr7   r8   r@   Ú	date_rankr   s                        r   Úmainrw   }   s\  € Ûà�?‰?œ9¨bˆ?Ó1€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø
�+‰+‹-€Cà€IØ€JØ€Iä˜!œS¤¬gÒ.D¼ÓEÈÑIÓJó CˆÜ—>‘> (Ó+ˆä�˜H˜: Q¤{ m°4Ð8Ô9Ü�X‹ˆÙÜ�N 8 *¨MÐ:Ô;Øä˜4Ó ˆÙÜ�N 8 *¨IÐ6Ô7Úä�œ˜U›˜ FÐ+Ô,àˆØˆ	Ø*/ó -	Ñ&ˆC�˜Yá˜Y¬Ò/Ø˜Q‘�	Ø˜a‘�
Øô   ›*ˆKÙÜÐ/°
¸3¸B°Ð/@ÐAÔBØä'3°KÀÓ'EÑ$ˆE�8˜WÙØñ ˜H¤vÒ-Ø˜Q‘�	Ø˜a‘�
ÜÐ+¨H¨:°Q°u¸S¸b°z°lÐCÔDØàˆIÙÜ × 0Ñ 0°°bÓ 9Ó:�	ð.Ø—‘ð3ô   s¨H°g¸yÈ&ÐQô	ð —<‘< !Ò#Ø ‘M�HØ ‘N�IØ "‘}¨Ò)ØŸ™�à ‘N�Iô �J‰J�sŽOð[-	ð^ 	�‰ŒÜ�˜�z  X J¨f°Y°KÐ?PÐQZÐP[Ð[aÐblÐamÐmsÐtÔuÜ�
‰
�3ŽðGCôJ 
ˆBˆvˆhˆ-ÔÜ	ˆG�I�;˜f Y K¨v°j°\ÀÐ
JÔKà�1‚}ÜÐ)Ô*ð	/Ø�K‰KÐBÄ9À+ÈQÐOÔPØ�K‰KÐNÔOØ�K‰KŒMÜÐ(Ô)ð 	‡J�J„LÜ	ˆ'…Nøô/ ò .ÜÐ(¨¨Ð,×-Ò-ûð.ûô& ò 	/ÜÐ)¨!¨Ð-×.Ñ.ûð	/ús2   Æ AK$ÊAL	 Ë$	LË-LÌLÌ		L*ÌL%Ì%L*Ú__main__)%Ú__doc__ÚosÚsysÚ_SYSr(   ÚargvÚisdigitrb   r^   r   rE   r-   rg   Úrequestsr   r   Úbs4r   Úgetenvr   rd   r,   r_   r]   ÚnowÚstrftimera   ÚHEADERSÚSessionr   ÚheadersÚupdater   r9   rM   rw   Ú__name__© r:   r   ú<module>rŠ      s.  ðñó 
ã Ù" 4§9¡9›~°Ò1°d·i±iÀ±l×6JÑ6JÔ6L‰#ˆd�i‰i˜‰lÔ
ÐRV€Ø
ÐÙ	Ð
¡# g£,Ñ
.Ô/ã 	Û 
Û Û ß (Ý àˆB�I‰I�kÐ#4Ó5€	Ø&€	Ø$€Ø
J€Ø€Ø
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€Ù ˆ�f�XÐÔ ðGàOØ0ñ	€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ òòòB$)òN_ðD ˆzÒÙ…Fð r:   