Ë
    ¸¬ujk  ã                   óø  — d dl Z 	 d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 dZ
e
dz   Ze
dz   Ze
dz   Z e j                  dd	«      Zd
ZdZdZdZdedœZ ej*                  «        e	d¬«      z
  j-                  d«      Z ej0                  «       Zej4                  j7                  e«       de_        ej:                  j<                  j?                  «        d„ Z dd„Z!d„ Z"dd„Z#d„ Z$d„ Z%e&dk(  r e%«        yy)é    N)ÚBeautifulSoup)ÚdatetimeÚ	timedeltazhttp://www.zgzhijiang.gov.cnz/show/listsz/show/detailz)/zfxxgk/list.html?depid=172&catid=663&t=0Ú	SEARCH_DBz/root/search.dbu*   æž�æ±Ÿå¸‚æ”¿åºœä¿¡æ�¯å…¬å¼€ - è¡Œæ”¿è®¸å�¯é	   é¬   i—  zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36)z
User-AgentÚRefereriG  )Údaysú%Y-%m-%dFc                 ó  — | syt        | «      j                  «       } dD ],  }	 t        j                  | dd |«      j	                  d«      c S  t        j                  d| «      }|r|j                  d«      S dS # t
        $ r Y Œew xY w)z(Parse various date formats to YYYY-MM-DDÚ )r   z%Y/%m/%du   %Yå¹´%mæœˆ%dæ—¥Né
   r   z(\d{4}[-/]\d{1,2}[-/]\d{1,2})é   )	ÚstrÚstripr   ÚstrptimeÚstrftimeÚ
ValueErrorÚreÚsearchÚgroup)Údate_strÚfmtÚms      ú%/root/gov_crawler/crawl_zgzhijiang.pyÚ
parse_dater   '   sŽ   € áØÜ�8‹}×"Ñ"Ó$€HØ:ò ˆð	Ü×$Ñ$ X¨c¨r ]°CÓ8×AÑAÀ*ÓMÒMðô
 	�	‰	Ð2°HÓ=€AÙˆ1�7‰7�1‹:Ð" Ð"øô ò 	Ùð	ús   £'A9Á9	BÂBc                 óX  — dt         › dt        › dt        › d| › d|› d�}t        |z   }	 t        j                  |d¬«      }|j                  d	k7  ry
|j                  «       }|j                  dd«      }|j                  dg «      }||fS # t        $ r}t        d|› �«       Y d}~y
d}~ww xY w)z$Call list API, return (items, total)ú?areaid=z&webid=z&cid=z&page=z
&pagenums=z
&orderby=0é   ©ÚtimeoutéÈ   )Nr   Úallnumsr   Úlistsz  API error: N)
ÚAREAIDÚWEBIDÚCIDÚLIST_APIÚsessionÚgetÚstatus_codeÚjsonÚ	ExceptionÚprint)	ÚpageÚpagenumsÚparamsÚurlÚrÚdataÚtotalÚitemsÚes	            r   Úfetch_list_pager8   4   s°   € àœ�x˜w¤u g¨U´3°%°v¸d¸VÀ:ÈhÈZÐWaÐb€FÜ
�VÑ
€Cð
Ü�K‰K˜ RˆKÓ(ˆØ�=‰=˜CÒØØ�v‰v‹xˆØ—‘˜ AÓ&ˆØ—‘˜ "Ó%ˆØ�eˆ|ÐøÜò Ü�˜a˜SÐ!Ô"Üûðús   ©&B Á7B Â	B)ÂB$Â$B)c                 ó  — t         › dt        › d| › �}	 t        j                  |d¬«      }|j                  dk7  ry|j                  «       }t        |t        «      rt        |«      dkD  r|d   }n$t        |t        «      r|j                  d|«      }ny|j                  d	d
«      }t        |j                  dd
«      «      }|st        |j                  dd
«      «      }||fS # t        $ r}t        d| › d|› �«       Y d}~yd}~ww xY w)z0Call detail API, return (content_html, pub_date)r   ú&id=r   r    r"   )r   r   r   r4   Úcontentr   Úvc_inputtimeÚ
createtimez  Detail API error (z): N)Ú
DETAIL_APIr%   r)   r*   r+   r,   Ú
isinstanceÚlistÚlenÚdictr   r-   r.   )Ún_idr2   r3   r4   Údetailr;   Úpub_dater7   s           r   Ú
get_detailrF   D   sô   € äˆL˜¤ ¨¨T¨FÐ
3€CðÜ�K‰K˜ RˆKÓ(ˆØ�=‰=˜CÒØØ�v‰v‹xˆÜ�dœDÔ!¤c¨$£i°!¢mØ˜!‘W‰FÜ˜œdÔ#Ø—X‘X˜f dÓ+‰Fàà—*‘*˜Y¨Ó+ˆÜ˜fŸj™j¨¸Ó<Ó=ˆÙÜ! &§*¡*¨\¸2Ó">Ó?ˆHà˜Ð Ð øÜò ÜÐ$ T F¨#¨a¨SÐ1Ô2Üûðús$   ”&C! »AC! ÂAC! Ã!	DÃ*D Ä Dc                 óD  — g }t        «       }d}d}d}| rdnd}	 |dz  }||kD  r�nŽt        |«      \  }}|€�n||s|dz   dz  }	t        d	|› d
|	› d�«       d}|s�nZg }
d}|D ]ó  }|j                  d«      }||v rŒ|j	                  |«       |j                  dd«      }t        |j                  dd«      «      }|st        |j                  dd«      «      }t        › dt        › d|› �}|||||j                  dd«      |j                  dd«      |j                  dd«      dœ}|
j                  |«       |j                  |«       |sŒå|t        k\  sŒï|dz  }Œõ t        d|› dt        |
«      › d|› d�«       | rn=|dk(  r|dz  }nd}|dk\  r|dk\  rt        d|› �«       nt        j                  d«       �Œš|D �cg c]  }|d   sŒ	|d   t        k\  sŒ|‘Œ }}|r&t        d t        |«      › d!t        |«      › d"�«       |S |dd# }t        d$t        |«      › d%�«       |S c c}w )&z"Fetch list pages with smart cutoffr   Fr   iç  TNé   r   z	  Total: z records, ~z pagesrC   Útitler   r<   r=   r   r:   Ú	vc_numberÚvc_departmentÚ
class_name)rI   r2   rC   rE   rJ   rK   rL   z  Page z: z records (recent: ú)é   é   z<  2 consecutive pages with no recent data, stopping at page g333333Ó?rE   z  Recent (3yr): z from z totaléd   z'  All data >3yr old, fallback to first z records)Úsetr8   r.   r*   Úaddr   r>   r%   ÚappendÚTHREE_YEARS_AGOrA   ÚtimeÚsleep)ÚincrementalÚ	all_itemsÚseen_idsr/   Úempty_recent_streakÚtotal_announcedÚ	max_pagesr6   r5   Útotal_pagesÚ
page_itemsÚpage_recentÚitÚnidrI   rE   Ú
detail_urlÚentryÚrecentÚresults                       r   Úget_all_list_itemsrf   ]   s€  € à€IÜ‹u€HØ€DØÐØ€Oñ !‘ c€Ià
Ø�‰	ˆØ�)ÒÙä& tÓ,‰ˆˆuØˆ=ÙáØ  2™:¨"Ñ,ˆKÜ�I˜e˜W K°¨}¸FÐCÔDØ"ˆOáÙàˆ
ØˆØò 	!ˆBØ—&‘&˜“.ˆCØ�h‰ØØ�L‰L˜Ôà—F‘F˜7 BÓ'ˆEÜ! "§&¡&¨¸Ó"<Ó=ˆHÙÜ% b§f¡f¨\¸2Ó&>Ó?�ä&˜< x´¨x°t¸C¸5ÐAˆJð Ø!ØØ$ØŸV™V K°Ó4Ø!#§¡¨¸Ó!<Ø Ÿf™f \°2Ó6ñˆEð ×Ñ˜eÔ$Ø×Ñ˜UÔ#â˜H¬Ó7Ø˜qÑ ‘ð7	!ô: 	�˜�v˜R¤ J£Ð0Ð0BÀ;À-ÈqÐQÔRáØð ˜!ÒØ 1Ñ$Ñà"#Ðà�1Š9Ð,°Ò1ÜÐPÐQUÐPVÐWÔXØä�
‰
�3Œñ ðD %Ö]�R¨¨:«¸2¸j¹>Ì_Ó;\ŠbÐ]€FÐ]ÙÜÐ ¤ V£ ¨V´C¸	³NÐ3CÀ6ÐJÔKØˆà˜4˜C�ˆÜÐ7¼¸F»°}ÀHÐMÔNØˆùò ^s   Æ8
HÇHÇHc                 óÚ  — |j                  «       }d}d}| D ]›  }|d   r|d   t        k  r|dz  }Œ	 |j                  d|d   t        |d   d|j	                  dd	«      › d
|j	                  dd	«      › �|j	                  dd	«      |j	                  dd	«      t
        f«       |j                  dkD  r|dz  }Œ� |j                  «        ||fS # t        $ r}t        d|› �«       Y d}~ŒÎd}~ww xY w)zInsert records into gov_rawr   rE   r   zÂ
                INSERT OR IGNORE INTO gov_raw 
                (page_url, source_url, title, summary, content, publish_date, site_name)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            r2   rI   u   æ–‡å�·: rJ   r   u    | éƒ¨é—¨: rK   r;   z  DB error: N)
ÚcursorrT   ÚexecuteÚBASE_URLr*   Ú	SITE_NAMEÚrowcountr-   r.   Úcommit)ÚrecordsÚconnrh   ÚinsertedÚskippedÚrecr7   s          r   Úinsert_to_dbrs   ³   s  € à�[‰[‹]€FØ€HØ€Gàò &ˆØˆzŠ?˜s :™´Ò@Ø�q‰LˆGØð	&Ø�N‰Nð ð
 �E‘
ÜØ�G‘Ø˜3Ÿ7™7 ;¨rÓ2Ð3°;¸s¿w¹wÀÐWYÓ?ZÐ>[Ð\Ø—‘˜	 2Ó&Ø—‘˜
 BÓ'Üðô	ð �‰ Ò"Ø˜A‘�øð)&ð0 	‡K�K„MØ�WÐÐøô	 ò 	&Ü�L  Ð$×%Ñ%ûð	&ús   ²BC	Ã		C*ÃC%Ã%C*c            	      ór  — dt         j                  v xs t         j                  d   dk(  } dd l}|j                  t        «      }t        dt        › d�«       | rt        d«       t        | ¬«      }|st        d	«       |j                  «        y t        d
t        |«      › d�«       t        |«      D ]I  \  }}|dz  dk(  rt        d|› dt        |«      › �d¬«       t        |d   «      \  }}|r||d<   |sŒE||d<   ŒK t        ||«      \  }}	|j                  «        t        d«       t        d|› d|	› �«       y )NrW   éÿÿÿÿÚ1r   z=== z ===zMode: incremental (page 1 only))rW   zNo items foundz	Fetching z details...r   z  ú/T)ÚflushrC   r;   rE   z
=== SUMMARY ===z
Inserted: z, Skipped (old): )ÚsysÚargvÚsqlite3ÚconnectÚDB_PATHr.   rk   rf   ÚcloserA   Ú	enumeraterF   rs   )
rW   r{   ro   r6   ÚiÚitemr;   Údate_from_detailrp   rq   s
             r   Úmainrƒ   Ô   s/  € Ø¤3§8¡8Ð+ÒB¬s¯x©x¸©|¸sÑ/B€KãØ�?‰?œ7Ó#€Dä	ˆD”�˜4Ð
 Ô!ÙÜÐ/Ô0ô ¨;Ô7€EÙÜÐÔØ�
‰
ŒØä	ˆI”c˜%“j�\ Ð
-Ô.Ü˜UÓ#ò 0‰ˆˆ4Øˆr‰6�QŠ;Ü�B�q�c˜œ3˜u›:˜,Ð'¨tÕ4ä$.¨t°F©|Ó$<Ñ!ˆÐ!ÙØ%ˆD�‰OÚØ/ˆD�Òð0ô % U¨DÓ1Ñ€HˆgØ‡J�J„Lä	ÐÔÜ	ˆJ�x�jÐ 1°'°Ð
;Õ<ó    Ú__main__)r   r   )F)'Úosry   r   rU   r,   ÚrequestsÚbs4r   r   r   rj   r(   r>   ÚREFERERÚgetenvr}   rk   r%   r&   r'   ÚHEADERSÚnowr   rT   ÚSessionr)   ÚheadersÚupdateÚverifyÚpackagesÚurllib3Údisable_warningsr   r8   rF   rf   rs   rƒ   Ú__name__© r„   r   ú<module>r–      s  ðã 	ð÷ × Û Ý ß (à)€Ø�mÑ#€Ø˜Ñ&€
Ø
Ð@Ñ
@€Ø
ˆ"�)‰)�KÐ!2Ó
3€Ø8€	à	
€Ø€Ø	€ð DØñ€ð
  �8—<‘<“>¡I°5Ô$9Ñ9×CÑCÀJÓO€à
ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ Ø€„Ø × Ñ × Ñ × *Ñ *Ô ,ò#óò ó2TòlòB!=ðF ˆzÒÙ…Fð r„   