Ë
    ¸¬uj…  ã                   ó  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 dZ
dZdZdZd	d
ddœZ ej                  «       Zej"                  j%                  e«       dd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uo   
ä¼šä¸œåŽ¿ï¼ˆschd.gov.cnï¼‰â€” çŽ¯å¢ƒä¿�æŠ¤ > çŽ¯è¯„å…¬å‘Š çˆ¬è™«
å†™å…¥ /root/search.db çš„ gov_raw è¡¨ + FTS
é    N)ÚBeautifulSoup)Úurljoinzhttp://www.schd.gov.cnz/http://www.schd.gov.cn/xxgk/zdlyxxgk/hjbh/hpgg/z/root/search.dbu   ä¼šä¸œåŽ¿çŽ¯è¯„å…¬å‘ŠzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagec                 óÔ   — 	 t         j                  | |¬«      }d|_        t        |j                  d«      S # t
        $ r(}t        d|z  t        j                  ¬«       Y d }~y d }~ww xY w)N)Útimeoutzutf-8úhtml.parserz  ERROR: %s)Úfile)	ÚsessionÚgetÚencodingr   ÚtextÚ	ExceptionÚprintÚsysÚstderr)Úurlr   ÚrespÚes       ú/root/gov_crawler/crawl_schd.pyÚget_soupr      sW   € ðÜ�{‰{˜3¨ˆ{Ó0ˆØˆŒÜ˜TŸY™Y¨Ó6Ð6øÜò Üˆm˜aÑ¤c§j¡jÕ1Üûðús   ‚36 ¶	A'¿A"Á"A'c                 ón  — g }| j                  d«      D ]ž  }|j                  dd¬«      }|j                  d«      }|r|sŒ,|j                  d¬«      }|j                  d¬«      }|rt        j                  d|«      sŒit        |«      dk  rŒxt        t        |d	   «      }|j                  |||f«       Œ  |S )
z1Extract (date, title, detail_url) from list page.ÚliÚaT)ÚhrefÚspan©Ústripz\d{4}-\d{2}-\d{2}é
   r   )	Úfind_allÚfindÚget_textÚreÚmatchÚlenr   ÚLIST_URLÚappend)ÚsoupÚitemsr   r   r   ÚtitleÚdateÚ
detail_urls           r   Úextract_list_itemsr,   )   s²   € à€EØ�m‰m˜DÓ!ò 0ˆØ�G‰G�C˜dˆGÓ#ˆØ�w‰w�v‹ˆÙ™ØØ—
‘
 �
Ó&ˆØ�}‰} 4ˆ}Ó(ˆáœ2Ÿ8™8Ð$8¸$Ô?ØÜˆu‹:˜Š?ØÜœX q¨¡yÓ1ˆ
Ø�‰�d˜E :Ð.Õ/ð0ð €Ló    c                 ó  — dD ]ˆ  }| j                  |«      }|sŒ|j                  ddg«      D ]  }|j                  «        Œ t        |«      }t	        j
                  dd|«      }t	        j
                  dd|«      }|j                  «       c S  y)z+Extract main content HTML from detail page.)zdiv.xl-contzdiv.trs_editor_viewz
div#font-sÚstyleÚscriptz\s*style="[^"]*"Ú z\s*style='[^']*')Ú
select_oner   Ú	decomposeÚstrr"   Úsubr   )r'   ÚselÚdivÚtagÚcontent_htmls        r   Úextract_contentr:   =   sŽ   € ð Dò 	(ˆØ�o‰o˜cÓ"ˆÚØ—|‘| W¨hÐ$7Ó8ò  �Ø—‘•ð ä˜s›8ˆLÜŸ6™6Ð"5°r¸<ÓHˆLÜŸ6™6Ð"5°r¸<ÓHˆLà×%Ñ%Ó'Ò'ð	(ð r-   c                 óœ   — | j                  dd¬«      }|r7t        j                  d|j                  «       «      }|r|j	                  d«      S y)z&Extract publish date from detail page.Úpzxl-p2)Úclass_z(\d{4}-\d{2}-\d{2})é   r1   )r    r"   Úsearchr!   Úgroup)r'   r<   Úms      r   Úextract_daterB   M   sB   € à�	‰	�#˜gˆ	Ó&€AÙÜ�I‰IÐ,¨a¯j©j«lÓ;ˆÙØ—7‘7˜1“:ÐØr-   c                 ó˜  — | syt        | d«      }|j                  d«      D ]  }|j                  «        Œ g }|j                  g d¢«      D ](  }|j                  d¬«      }|sŒ|j	                  |«       Œ* t        |j                  d«      «      }|r|j	                  d|z  «       |rd	j                  |«      S |j                  d
d¬«      S )zBConvert HTML to plain text with paragraph breaks, keep image refs.r1   r   r   )r<   r7   Útrr   Tr   Úimgu   [æœ¬é¡µå�« %d å¼ å›¾ç‰‡é™„ä»¶]z

ú
)Ú	separatorr   )r   r   Úunwrapr!   r&   r$   Újoin)Úhtml_contentÚsr   Úpartsr8   r   Ú	img_counts          r   Úextract_text_simplerN   W   s¿   € áØÜ�l MÓ2€AØ—
‘
˜6Ó"ò ˆØ�‰�ðà€EØ�z‰zÒ2Ó3ò ˆØ�|‰| $ˆ|Ó'ˆÚØ�L‰L˜Õðô
 �A—J‘J˜uÓ%Ó&€IÙØ�‰Ð5¸	ÑAÔBÙ!&ˆ6�;‰;�uÓÐR¨A¯J©JÀÈT¨JÓ,RÐRr-   c                 ó¼  — t        j                  t        «      }|j                  «       }t	        «       }|j                  dd«      D ]  }|j                  |d   «       Œ d}d}| D ]®  \  }}}	}
}|	|v r|dz  }Œ|r|d d n|}dd l}t        |j                  j                  «       j                  d«      «      }|j                  dt        |||
|	||f«       |j                  }	 |j                  d||t        |d d	 f«       |dz  }|j                  |	«       Œ° |j                  «        |j!                  «        ||fS # t        $ r}t        d
|z  «       Y d }~ŒXd }~ww xY w)Nz2SELECT page_url FROM gov_raw WHERE page_url LIKE ?)z%schd.gov.cn%r   r>   i,  z%Y%m%dzš
            INSERT INTO gov_raw (site_name, title, summary, content, page_url, publish_date, date_rank)
            VALUES (?, ?, ?, ?, ?, ?, ?)
        zLINSERT INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)éÈ   z  FTS error: %s)Úsqlite3ÚconnectÚDB_PATHÚcursorÚsetÚexecuteÚaddÚdatetimeÚintÚnowÚstrftimeÚ	SITE_NAMEÚ	lastrowidr   r   ÚcommitÚclose)Ú
items_dataÚconnÚcÚexistingÚrowÚ	count_newÚ
count_skipr*   r)   r+   r9   Ú
plain_textÚsummaryrX   Údate_rank_valÚrow_idr   s                    r   Ú
save_to_dbrk   j   sp  € Ü�?‰?œ7Ó#€DØ�‰‹€Aä‹u€HØ�y‰yÐMÐOaÓbò ˆØ�‰�S˜‘VÕðð €IØ€Jà=Gò !Ñ9ˆˆe�Z ¨zØ˜Ñ!Ø˜!‰OˆJØá&0�*˜T˜cÑ"°eˆãÜ˜H×-Ñ-×1Ñ1Ó3×<Ñ<¸XÓFÓGˆØ	�	‰	ð ô ˜ ¨°zÀ4ÈÐWô	Yð
 —‘ˆð	)Ø�I‰IÐdØ˜u¤i°¸¸#°Ð?ôAð
 	�Q‰ˆ	Ø�‰�ZÕ ð-!ð0 	‡K�K„MØ‡J�J„LØ�jÐ Ð øô ò 	)ÜÐ# aÑ'×(Ñ(ûð	)ús   Ã!D:Ä:	EÅEÅEc            	      óü  — t        d«       t        d«       t        d«       t        dt        z  «       t        t        «      } | s t        d«       t        j                  d«       t        | «      }t        dt        |«      z  «       |st        d«       y g }t        |d«      D ]Ä  \  }\  }}}t        d|t        |«      |d d	 fz  d
¬«       t        j                  j                  «        t        |«      } | st        d«       Œ`|st        | «      }t        | «      }t        |«      }|j                  |||||f«       t        dt        |«      z  «       t        j                  d«       ŒÆ t        d«       t!        |«      \  }	}
t        d|	|
fz  «       t#        j$                  t&        «      }|j)                  dt*        f«      j-                  «       d   }|j/                  «        t        d|z  «       t        d«       y )Nz<============================================================u,   ä¼šä¸œåŽ¿ï¼ˆschd.gov.cnï¼‰çŽ¯è¯„å…¬å‘Šçˆ¬è™«z
Fetching list: %szFAILED to fetch list pager>   zFound %d itemszNo items to crawlz  [%d/%d] %sé2   ú )ÚendÚFAILzOK (%dc)g      à?z
Saving to database...zNew: %d, Skip: %dz0SELECT COUNT(*) FROM gov_raw WHERE site_name = ?r   zTotal in DB: %dz
Done!)r   r%   r   r   Úexitr,   r$   Ú	enumerateÚstdoutÚflushrB   r:   rN   r&   ÚtimeÚsleeprk   rQ   rR   rS   rV   r\   Úfetchoner_   )r'   r(   Úall_dataÚidxr*   r)   r+   r9   rg   Ú	new_countÚ
skip_countra   Útotals                r   Úmainr}   ’   s²  € Ü	ˆ(„OÜ	Ð
8Ô9Ü	ˆ(„Oä	Ð
¤(Ñ
*Ô+Ü”HÓ€DÙÜÐ)Ô*Ü�‰�Œä˜tÓ$€EÜ	Ð
œS ›ZÑ
'Ô(áÜÐ!Ô"Øà€HÜ*3°E¸1Ó*=ò Ñ&ˆÑ&ˆd�E˜:Üˆn ¤S¨£Z°°s¸°Ð<Ñ<À#ÕFÜ�
‰
×ÑÔä˜
Ó#ˆÙÜ�&ŒMØáÜ Ó%ˆDä& tÓ,ˆÜ(¨Ó6ˆ
à�‰˜˜u j°,À
ÐKÔLÜˆjœ3˜|Ó,Ñ,Ô-ä�
‰
�3�ð%ô( 
Ð
#Ô$Ü& xÓ0Ñ€IˆzÜ	Ð
 ¨JÐ 7Ñ
7Ô8ä�?‰?œ7Ó#€DØ�L‰LÐKÌiÈ\ÓZ×cÑcÓeÐfgÑh€EØ‡J�J„LÜ	Ð
˜eÑ
#Ô$Ü	ˆ)Õr-   Ú__main__)é   )Ú__doc__ÚrequestsrQ   ru   r"   r   Úbs4r   Úurllib.parser   ÚBASE_URLr%   rS   r\   ÚHEADERSÚSessionr
   ÚheadersÚupdater   r,   r:   rB   rN   rk   r}   Ú__name__© r-   r   ú<module>r‹      s¦   ðñó
 Û Û Û 	Û 
Ý Ý  à#€Ø<€Ø
€Ø#€	ðGàOØ0ñ	€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ óòò(ò òSò&%!òP/ðd ˆzÒÙ…Fð r-   