Ë
    ¸¬ují  ã                   ó  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 dZ
dZdZdZd	d
ddœZ ej                  «       Zej"                  j%                  e«       dd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)ut   
ä½™æ±ŸåŒºï¼ˆyujiang.gov.cnï¼‰â€” åŒºç”Ÿæ€�çŽ¯å¢ƒå±€ä¿¡æ�¯å…¬å¼€çˆ¬è™«
å†™å…¥ /root/search.db çš„ gov_raw è¡¨ + FTS
é    N)ÚBeautifulSoup)Úurljoinzhttp://www.yujiang.gov.cnzWhttp://www.yujiang.gov.cn/module/xxgk/subjectinfo.jsp?standardXxgk=1&area=Y02024&page=1z/root/search.dbu   ä½™æ±ŸåŒºç”Ÿæ€�çŽ¯å¢ƒå±€zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagec                 óÔ   — 	 t         j                  | |¬«      }d|_        t        |j                  d«      S # t
        $ r(}t        d|z  t        j                  ¬«       Y d }~y d }~ww xY w)N)Útimeoutzutf-8úhtml.parserz  ERROR: %s)Úfile)	ÚsessionÚgetÚencodingr   ÚtextÚ	ExceptionÚprintÚsysÚstderr)Úurlr   ÚrespÚes       ú"/root/gov_crawler/crawl_yujiang.pyÚget_soupr      sW   € ðÜ�{‰{˜3¨ˆ{Ó0ˆØˆŒÜ˜TŸY™Y¨Ó6Ð6øÜò Üˆm˜aÑ¤c§j¡jÕ1Üûðús   ‚36 ¶	A'¿A"Á"A'c                 óp  — g }| j                  dd¬«      D ]�  }|j                  d¬«      }|d   }|rt        |«      dk  sd|vrŒ/|j                  d«      }|sŒC|j                  d¬«      j	                  «       }t        j                  d	|«      sŒzt        t        |«      }|j                  |||f«       ŒŸ |S )
z7Extract (date, title, detail_url) from subjectinfo.jsp.ÚaT©Úhref©Ústripr   é
   zart/Úbz\d{4}-\d{2}-\d{2})
Úfind_allÚget_textÚlenÚ	find_nextr   ÚreÚmatchr   ÚBASE_URLÚappend)ÚsoupÚitemsr   ÚtxtÚhr   ÚdateÚ
detail_urls           r   Úextract_list_itemsr-   )   s·   € à€EØ�]‰]˜3 Tˆ]Ó*ò .ˆØ�j‰j˜tˆjÓ$ˆØˆf‰IˆÙ”c˜#“h ’m v°Q¡ØØ�K‰K˜ÓˆÙØØ�z‰z ˆzÓ%×+Ñ+Ó-ˆÜ�x‰xÐ,¨dÔ3ØÜœX qÓ)ˆ
Ø�‰�d˜C Ð,Õ-ð.ð €Ló    c                 óâ  — dD ]Ÿ  }| j                  |«      }|sŒ|j                  ddg«      D ]  }|j                  «        Œ t        |«      }t	        j
                  dd|«      }t	        j
                  dd|«      }t	        j
                  dd|«      }|j                  «       c S  | j                  d	d
¬«      }|r6|j                  ddg«      D ]  }|j                  «        Œ t        |«      dd S y)z!Extract content from detail page.)zdiv.screen2zdiv.pcxxÚstyleÚscriptz\s*style="[^"]*"Ú z\s*style='[^']*'zsrc="\.\./([^"]+)"z"src="http://www.yujiang.gov.cn/\1"ÚdivÚbarrierfree_container)ÚidNiPÃ  )Ú
select_oner   Ú	decomposeÚstrr#   Úsubr   Úfind)r'   Úselr3   ÚtagÚcontent_htmls        r   Úextract_contentr>   <   sï   € ð +ò (ˆØ�o‰o˜cÓ"ˆÚà—|‘| W¨hÐ$7Ó8ò  �Ø—‘•ð ä˜s›8ˆLÜŸ6™6Ð"5°r¸<ÓHˆLÜŸ6™6Ð"5°r¸<ÓHˆLäŸ6™6Ð"7Ð9^Ð`lÓmˆLØ×%Ñ%Ó'Ò'ð(ð �)‰)�EÐ5ˆ)Ó
6€CÙ
Ø—<‘< ¨(Ð 3Ó4ò 	ˆCØ�M‰M�Oð	ä�3‹x˜˜ÐÐØr.   c                 óô   — g }| j                  dd¬«      D ]_  }|d   }d|v s	d|v sd|v sŒt        t        |«      }|j                  d¬«      xs |j	                  d	«      d
   }|j                  ||f«       Œa |S )zExtract attachment links.r   Tr   r   zdownload/downfilez.pdfz.docr   ú/éÿÿÿÿ)r   r   r%   r    Úsplitr&   )r'   Úattachmentsr   r*   Úfull_urlr)   s         r   Úextract_attachmentsrE   T   sˆ   € à€KØ�]‰]˜3 Tˆ]Ó*ò 0ˆØˆf‰IˆØ !Ñ# v°¡{°fÀ²kÜœx¨Ó+ˆHØ—*‘* 4�*Ó(Ò<¨A¯G©G°C«L¸Ñ,<ˆCØ×Ñ  X˜Õ/ð0ð Ðr.   c                 óx   — | j                  «       }t        j                  d|«      }|r|j                  d«      S dS )z&Extract publish date from detail page.z(\d{4}-\d{2}-\d{2})é   r2   )r    r#   ÚsearchÚgroup)r'   r   Úms      r   Úextract_daterK   `   s5   € ð �=‰=‹?€DÜ
�	‰	Ð(¨$Ó/€AÙˆ1�7‰7�1‹:Ð" Ð"r.   c                 ó8  — | syt        | d«      }|j                  d«      D ]  }|j                  «        Œ g }|j                  g d¢«      D ](  }|j                  d¬«      }|sŒ|j	                  |«       Œ* |rdj                  |«      S |j                  dd¬	«      S )
z1Convert HTML to plain text with paragraph breaks.r2   r   Úspan)Úpr3   ÚtrÚtdÚliTr   z

ú
)Ú	separatorr   )r   r   Úunwrapr    r&   Újoin)Úhtml_contentÚsrM   Úpartsr<   r   s         r   Úextract_text_simplerY   h   s—   € áØÜ�l MÓ2€AØ—
‘
˜6Ó"ò ˆØ�‰�ðà€EØ�z‰zÒ8Ó9ò ˆØ�|‰| $ˆ|Ó'ˆÚØ�L‰L˜Õðñ "'ˆ6�;‰;�uÓÐR¨A¯J©JÀÈT¨JÓ,RÐRr.   c                 óR   — | }|r"d}|D ]  \  }}|d|›d|›d�z  }Œ |dz  }||z  }|S )z#Append attachment links to content.u   

<h3>é™„ä»¶</h3>
<ul>z
<li><a href="z">z	</a></li>z
</ul>© )r=   rC   ÚresultÚattach_textÚnamer   s         r   Úformat_with_attachmentsr_   w   sL   € à€FÙØ1ˆØ$ò 	J‰IˆD�#ÙºsÂDÐIÑI‰Kð	Jà�yÑ ˆØ�+ÑˆØ€Mr.   c                 óì  — t        j                  t        «      }|j                  «       }t	        «       }|j                  dd«      D ]  }|j                  |d   «       Œ d}d}| D ]Æ  \  }}}	}
}}|	|v r|dz  }Œ|r|d d n|}t        |
|«      }t        |«      }dd l	}t        |j                  j                  «       j                  d«      «      }|j                  dt        ||||	||f«       |j                  }	 |j                  d||t        |d d	 f«       |dz  }|j                  |	«       ŒÈ |j#                  «        |j%                  «        ||fS # t        $ r}t!        d
|z  «       Y d }~ŒXd }~ww xY w)Nz2SELECT page_url FROM gov_raw WHERE page_url LIKE ?)z%yujiang.gov.cn%r   rG   i,  z%Y%m%dzš
            INSERT INTO gov_raw (site_name, title, summary, content, page_url, publish_date, date_rank)
            VALUES (?, ?, ?, ?, ?, ?, ?)
        zLINSERT INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)éÈ   z  FTS error: %s)Úsqlite3ÚconnectÚDB_PATHÚcursorÚsetÚexecuteÚaddr_   rY   ÚdatetimeÚintÚnowÚstrftimeÚ	SITE_NAMEÚ	lastrowidr   r   ÚcommitÚclose)Ú
items_dataÚconnÚcÚexistingÚrowÚ	count_newÚ
count_skipr+   Útitler,   r=   rC   Ú
plain_textÚsummaryÚfull_contentÚplain_text_fullri   Údate_rank_valÚrow_idr   s                       r   Ú
save_to_dbr   ƒ   sˆ  € Ü�?‰?œ7Ó#€DØ�‰‹€Aä‹u€HØ�y‰yÐMÐOdÓeò ˆØ�‰�S˜‘VÕðð €IØ€JàJTò !ÑFˆˆe�Z ¨{¸JØ˜Ñ!Ø˜!‰OˆJØá&0�*˜T˜cÑ"°eˆÜ.¨|¸[ÓIˆÜ-¨lÓ;ˆãÜ˜H×-Ñ-×1Ñ1Ó3×<Ñ<¸XÓFÓGˆØ	�	‰	ð ô ˜ ¨°zÀ4ÈÐWô	Yð
 —‘ˆð	)Ø�I‰IÐdØ˜u¤i°¸¸#°Ð?ôAð
 	�Q‰ˆ	Ø�‰�ZÕ ð1!ð4 	‡K�K„MØ‡J�J„LØ�jÐ Ð øô ò 	)ÜÐ# aÑ'×(Ñ(ûð	)ús   Ã9EÅ	E3ÅE.Å.E3c            	      ó  — t        d«       t        d«       t        d«       t        d«       t        t        «      } | s t        d«       t        j                  d«       t        | «      }t        dt        |«      z  «       |st        d«       y g }t        |d«      D ]Û  \  }\  }}}t        d|t        |«      |d d	 fz  d
¬«       t        j                  j                  «        t        |«      } | st        d«       Œ`|st        | «      }t        | «      }t        | «      }t        |«      }	|j                  ||||||	f«       t        dt        |«      t        |«      fz  «       t        j                   d«       ŒÝ t        d«       t#        |«      \  }
}t        d|
|fz  «       t%        j&                  t(        «      }|j+                  dt,        f«      j/                  «       d   }|j1                  «        t        d|z  «       t        d«       y )Nz<============================================================u8   ä½™æ±ŸåŒºï¼ˆyujiang.gov.cnï¼‰ç”Ÿæ€�çŽ¯å¢ƒå±€ä¿¡æ�¯çˆ¬è™«z
Fetching list...zFAILED to fetch listrG   zFound %d itemszNo items to crawlz  [%d/%d] %sé2   ú )ÚendÚFAILu   OK (%dc, %dé™„ä»¶)g      à?z
Saving to database...zNew: %d, Skip: %dz0SELECT COUNT(*) FROM gov_raw WHERE site_name = ?r   zTotal in DB: %dz
Done!)r   r   ÚLIST_URLr   Úexitr-   r!   Ú	enumerateÚstdoutÚflushrK   r>   rE   rY   r&   ÚtimeÚsleepr   rb   rc   rd   rg   rm   Úfetchonerp   )r'   r(   Úall_dataÚidxr+   rx   r,   r=   rC   ry   Ú	new_countÚ
skip_countrr   Útotals                 r   Úmainr’   ­   sÄ  € Ü	ˆ(„OÜ	Ð
DÔEÜ	ˆ(„Oä	Ð
ÔÜ”HÓ€DÙÜÐ$Ô%Ü�‰�Œä˜tÓ$€EÜ	Ð
œS ›ZÑ
'Ô(áÜÐ!Ô"Øà€HÜ*3°E¸1Ó*=ò Ñ&ˆÑ&ˆd�E˜:Üˆn ¤S¨£Z°°s¸°Ð<Ñ<À#ÕFÜ�
‰
×ÑÔä˜
Ó#ˆÙÜ�&ŒMØáÜ Ó%ˆDä& tÓ,ˆÜ)¨$Ó/ˆÜ(¨Ó6ˆ
à�‰˜˜u j°,ÀÈZÐXÔYÜÐ"¤c¨,Ó&7¼¸[Ó9IÐ%JÑJÔKä�
‰
�3�ð'ô* 
Ð
#Ô$Ü& xÓ0Ñ€IˆzÜ	Ð
 ¨JÐ 7Ñ
7Ô8ä�?‰?œ7Ó#€DØ�L‰LÐKÌiÈ\ÓZ×cÑcÓeÐfgÑh€EØ‡J�J„LÜ	Ð
˜eÑ
#Ô$Ü	ˆ)Õr.   Ú__main__)é   )Ú__doc__Úrequestsrb   rŠ   r#   r   Úbs4r   Úurllib.parser   r%   r…   rd   rm   ÚHEADERSÚSessionr
   ÚheadersÚupdater   r-   r>   rE   rK   rY   r_   r   r’   Ú__name__r[   r.   r   ú<module>rž      s°   ðñó
 Û Û Û 	Û 
Ý Ý  à&€Ød€Ø
€Ø&€	ðGàOØ0ñ	€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ óòò&ò0	ò#òSò	ò'!òT0ðf ˆzÒÙ…Fð r.   