Ë
    ¸¬uj°"  ã                   ó”   — d Z ddlZddlZddlZddlZddlZddlmZ dZdZ	dZ
ddiZd	„ Zd
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u†   
éƒ½æ˜ŒåŽ¿äººæ°‘æ”¿åºœ - çŽ¯è¯„ä¿¡æ�¯ çˆ¬è™«è„šæœ¬
æŠ“å�– https://www.duchang.gov.cn/zwgk/zfxxgkzl/bmxxgk/sthjj/zdly/03/ æ‰€æœ‰æ–‡ç« 
é    N)ÚBeautifulSoupz>https://www.duchang.gov.cn/zwgk/zfxxgkzl/bmxxgk/sthjj/zdly/03/u   éƒ½æ˜ŒåŽ¿-çŽ¯è¯„ä¿¡æ�¯é   z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c                  óB   — t         j                  j                  dd«      S )z8Get database path from SEARCH_DB env var or use default.Ú	SEARCH_DBz/root/search.db)ÚosÚenvironÚget© ó    ú'/root/gov_crawler/crawl_duchang_sthj.pyÚget_db_pathr      s   € ä�:‰:�>‰>˜+Ð'8Ó9Ð9r   c                 ór   — t        j                  | «      }|j                  d«       |j                  «        |S )zEnsure the table exists.a4  
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_name TEXT NOT NULL,
            source_url TEXT NOT NULL,
            page_url TEXT NOT NULL UNIQUE,
            title TEXT,
            publish_date TEXT,
            content TEXT
        )
    )Úsqlite3ÚconnectÚexecuteÚcommit)Údb_pathÚconns     r   Úinit_dbr      s1   € ä�?‰?˜7Ó#€DØ‡L�Lð 
ô 
	ð 	‡K�K„MØ€Kr   c                 óÈ  — 	 t        j                  | t        d¬«      }d|_        |j                  dk7  r-t        d	|j                  › d
| › �t        j                  ¬«       g S t        |j                  d«      }|j                  d«      }g }|D ]š  }|j                  d«      }t        |«      dk  rŒ#|d   j                  d«      }|sŒ:|j                  dd«      j                  «       }	|j                  d¬«      }
|d   j                  d¬«      }|	r|
sŒ†|j!                  |
|	|dœ«       Œœ |S # t        $ r-}t        d| › d|› �t        j                  ¬«       g cY d}~S d}~ww xY w)z~
    Parse a list page and extract (title, page_url, date) tuples.
    Returns a list of dicts, or empty list on failure.
    é   ©ÚheadersÚtimeoutúutf-8z$  [ERROR] Failed to fetch list page ú: ©ÚfileNéÈ   z  [ERROR] HTTP ú for úhtml.parserÚtrÚtdé   é   ÚaÚhrefÚ T©Ústripé   )ÚtitleÚpage_urlÚpublish_date)Úrequestsr	   ÚHEADERSÚencodingÚ	ExceptionÚprintÚsysÚstderrÚstatus_coder   ÚtextÚfind_allÚlenÚfindr*   Úget_textÚappend)ÚurlÚrÚeÚsoupÚrowsÚitemsr"   ÚtdsÚa_tagr'   r,   Údate_strs               r   Úparse_list_pagerF   /   sQ  € ð
Ü�L‰L˜¤g°rÔ:ˆØˆŒ
ð
 	‡}�}˜ÒÜ� §¡˜¨e°C°5Ð9ÄÇ
Á
ÕKØˆ	ä˜Ÿ™ Ó/€DØ�=‰=˜Ó€DØ€Eàò ˆØ�k‰k˜$ÓˆÜˆs‹8�aŠ<Øà�A‘—‘˜CÓ ˆÙØØ�y‰y˜ Ó$×*Ñ*Ó,ˆØ—‘ T�Ó*ˆØ�q‘6—?‘?¨�?Ó.ˆÙ™5ØØ�‰ØØØ$ñ
õ 	ðð& €Løô? ò ÜÐ4°S°E¸¸A¸3Ð?ÄcÇjÁjÕQØ�	ûðús   ‚#D+ Ä+	E!Ä4"EÅE!ÅE!c                  ód  — 	 t        j                  t        t        d¬«      } d| _        | j                  dk7  r*t        d	| j                  › d
�t        j                  ¬«       yt        | j                  d«      }|j                  d«      }|D ]l  }|j                  sŒd|j                  v sŒt        j                  d|j                  «      }|sŒBt!        |j#                  d«      «      }t        d|› �«       |c S  |j%                  «       }t        j                  d|«      }|r@t!        |j#                  d«      «      }|t&        z   dz
  t&        z  }t        d|› d|› �«       |S t        dt        j                  ¬«       y# t
        $ r(}t        d|› �t        j                  ¬«       Y d}~yd}~ww xY w)z—
    Fetch first page to determine total page count.
    Parses the JS createPage() call embedded in the page.
    Returns total page count (int).
    r   r   r   z#[ERROR] Failed to fetch list page: r   Nr%   r   z[ERROR] HTTP z for list pager!   ÚscriptÚ
createPagezcreatePage\s*\(\s*(\d+)z  Total pages detected: u   å…±\s*(\d+)\s*æ�¡z  Total records: z	, pages: z5  [WARN] Could not detect page count, defaulting to 1)r/   r	   ÚBASE_LIST_URLr0   r1   r2   r3   r4   r5   r6   r   r7   r8   ÚstringÚreÚsearchÚintÚgroupr;   ÚPER_PAGE)	r>   r?   r@   ÚscriptsrH   ÚmÚtotal_pagesÚ	body_textÚtotal_recordss	            r   Úget_page_countrV   Y   si  € ðÜ�L‰Lœ´ÀÔDˆØˆŒ
ð
 	‡}�}˜ÒÜ�˜aŸm™m˜_¨NÐ;Ä#Ç*Á*ÕMØä˜Ÿ™ Ó/€DØ�m‰m˜HÓ%€GØò #ˆØ�=‹=˜\¨V¯]©]Ò:ä—	‘	Ð4°f·m±mÓDˆAÚÜ! !§'¡'¨!£*›o�ÜÐ0°°Ð>Ô?Ø"Ò"ð#ð —‘“€IÜ
�	‰	Ð&¨	Ó2€AÙÜ˜AŸG™G A›J›ˆØ$¤xÑ/°!Ñ3¼Ñ@ˆÜÐ! - °	¸+¸ÐGÔHØÐä	Ð
AÌÏ
É
ÕSØøô; ò ÜÐ3°A°3Ð7¼c¿j¹jÕIÜûðús   ‚'E> Å>	F/ÆF*Æ*F/c                 ó2   — | dk(  rt         S t         d| › d�z   S )z+Build URL for a given page index (0-based).r   Úindex_z.html)rJ   )Ú
page_indexs    r   Úbuild_page_urlrZ   ‚   s#   € à�Q‚ÜÐÜ˜V J <¨uÐ5Ñ5Ð5r   c                 ón  — 	 t        j                  | t        d¬«      }d|_        |j                  dk7  r,t        d	|j                  › d
| › �t        j                  ¬«       yt        |j                  d«      }d}|j                  dd¬«      }|r|j                  d¬«      }|sK|j                  d«      }|r8|j                  d¬«      }d|v r"|d|j                  d«       j                  «       }|sE|j                  d«      }|r2|j                  d¬«      j!                  d«      d   j                  «       }d}	|j                  dd¬«      }
|
r|
j                  d¬«      }	|	sJ|j                  d«      }|r7t#        j$                  d|j                  «       «      }|r|j'                  d«      }	d}dD ])  }|j                  |«      }|sŒ|j                  d¬«      } n |s’|j                  d«      }|r|j                  d¬«      }d|v r&||j                  d«      dz   d j                  «       }nCd|v r?|j                  d«      }||d }d|v r%||j                  d«      dz   d j                  «       }|xs d|	xs d|xs ddœ}|S # t        $ r+}t        d| › d|› �t        j                  ¬«       Y d}~yd}~ww xY w) zl
    Parse a detail page and extract title, publish_date, content.
    Returns dict or None on failure.
    r   r   r   z#    [ERROR] Failed to fetch detail r   r   Nr   z    [ERROR] HTTP r    r!   ÚdivÚNewsArticleTitle)ÚidTr)   zdiv.content-bgu   æ—¥æœŸr,   ú,r   ÚNewsArticlePubDayu"   æ—¥æœŸ[ï¼š:]\s*(\d{4}-\d{2}-\d{2})r%   )zdiv.content.qjcontentzdiv.view.TRS_UEDITORzdiv.trs_editor_view.TRS_UEDITORu	   åˆ†äº«ï¼šr$   u	   æ�¥æº�ï¼šu   ã€‘r(   )r,   r.   Úcontent)r/   r	   r0   r1   r2   r3   r4   r5   r6   r   r7   r:   r;   Ú
select_oneÚindexr*   ÚsplitrL   rM   rO   )r-   r>   r?   r@   r,   Ú	title_divÚ
content_bgÚtxtÚ	title_tagr.   Údate_divrR   ra   ÚselectorÚcontent_divÚidxÚafter_sourceÚresults                     r   Úparse_detail_pagero   ‰   sÊ  € ð
Ü�L‰L˜¬7¸BÔ?ˆØˆŒ
ð
 	‡}�}˜ÒÜÐ! !§-¡- °°h°ZÐ@ÄsÇzÁzÕRØä˜Ÿ™ Ó/€Dð €EØ—	‘	˜%Ð$6�	Ó7€IÙØ×"Ñ"¨Ð"Ó.ˆÙà—_‘_Ð%5Ó6ˆ
ÙØ×%Ñ%¨DÐ%Ó1ˆCØ˜3‰ØÐ0˜SŸY™Y xÓ0Ð1×7Ñ7Ó9�ÙØ—I‘I˜gÓ&ˆ	ÙØ×&Ñ&¨TÐ&Ó2×8Ñ8¸Ó=¸aÑ@×FÑFÓHˆEð €LØ�y‰y˜Ð#6ˆyÓ7€HÙØ×(Ñ(¨tÐ(Ó4ˆÙà—_‘_Ð%5Ó6ˆ
ÙÜ—	‘	Ð?À×ATÑATÓAVÓWˆAÙØ Ÿw™w q›z�ð €Gðò ˆð
 —o‘o hÓ/ˆÚØ!×*Ñ*°Ð*Ó6ˆGÙðñ Ø—_‘_Ð%5Ó6ˆ
Ùà×%Ñ%¨DÐ%Ó1ˆCà˜cÑ!Ø˜cŸi™i¨Ó4°qÑ8Ð9Ð:×@Ñ@ÓB‘Ø Ñ#Ø—i‘i Ó,�à" 3 4˜y�Ø˜LÑ(Ø! #§)¡)¨EÓ"2°QÑ"6Ð"7Ð8×>Ñ>Ó@�Gð ’˜"Ø$Ò*¨Ø’=˜bñ€Fð
 €MøôO ò ÜÐ3°H°:¸RÀ¸sÐCÌ#Ï*É*ÕUÜûðús   ‚#J  Ê 	J4Ê	!J/Ê/J4c                  ó¼  — t        «       } t        d| › �«       t        | «      }|j                  «       }t	        «       }t        d|› �«       d}d}t        |«      D �]  }t        |«      }t        d|dz   › d|› d|› �«       t        |«      }t        dt        |«      › d	�«       |D ]À  }	|dz  }|	d
   }
|	d   }|	d   }t        d|› d|dd › d�«       t        |
«      }|€t        d«       ŒE|d   xs |}|d   xs |}|d   }	 |j                  dt        t        |
|||f«       |j                  dkD  r|dz  }t        d|› d|dd › �«       nt        d«       |j                  «        ŒÂ �Œ |j)                  «        t        d|› d|› d�«       y# t        j                   $ r9}t        d|› �t"        j$                  ¬«       |j'                  «        Y d}~�Œ0d}~ww xY w)zMain crawl function.z
Database: zTotal pages to crawl: r   z

--- Page r%   ú/r   z  Found z items on pager-   r,   r.   z  [z] Né2   z...z!    [SKIP] Failed to parse detailra   z¥INSERT OR IGNORE INTO gov_raw
                       (site_name, source_url, page_url, title, publish_date, content)
                       VALUES (?, ?, ?, ?, ?, ?)z    [INSERTED] z | z    [SKIP] already existsz    [DB ERROR] r   z
=== Done! Total: z, New: z ===)r   r3   r   ÚcursorrV   ÚrangerZ   rF   r9   ro   r   Ú	SITE_NAMErJ   Úrowcountr   r   ÚErrorr4   r5   ÚrollbackÚclose)r   r   rs   rS   Útotal_itemsÚ	new_itemsÚpage_idxÚlist_urlrB   Úitemr-   r,   rE   ÚdetailÚfinal_titleÚ
final_datera   r?   s                     r   Úcrawlr‚   Û   s  € ä‹m€GÜ	ˆJ�w�iÐ
 Ô!Ü�7Ó€DØ�[‰[‹]€Fä Ó"€KÜ	Ð" ; -Ð
0Ô1à€KØ€Iä˜+Ó&ó ( ˆÜ! (Ó+ˆÜ�˜H q™L˜>¨¨;¨-°r¸(¸ÐDÔEÜ Ó)ˆÜ�œ˜U›˜ NÐ3Ô4àò "	 ˆDØ˜1ÑˆKØ˜JÑ'ˆHØ˜‘MˆEØ˜NÑ+ˆHä�C˜�} B u¨S¨b z l°#Ð6Ô7ô ' xÓ0ˆFØˆ~ÜÐ9Ô;Øð ! ™/Ò2¨UˆKØ Ñ/Ò;°8ˆJØ˜YÑ'ˆGð Ø—‘ð4ô ¤¨x¸ÀjÐRYÐZô	ð —?‘? QÒ&Ø ‘N�IÜ˜O¨J¨<°s¸;ÀsÈÐ;KÐ:LÐMÕNäÐ5Ô7Ø—‘•ò?"	 ð( ðT 	‡J�J„LÜ	Ð ˜}¨G°I°;¸dÐ
CÕDøô —=‘=ò  Ü˜¨ sÐ+´#·*±*Õ=Ø—‘—’ûð ús   ÄA$FÆGÆ".GÇGÚ__main__)Ú__doc__r   r4   rL   r   r/   Úbs4r   rJ   ru   rP   r0   r   r   rF   rV   rZ   ro   r‚   Ú__name__r
   r   r   ú<module>r‡      s|   ðñó
 
Û 
Û 	Û Û Ý àP€Ø$€	Ø€ð ð Gð€ò:ò
ò$'òT&òR6òOòd8Eðv ˆzÒÙ	…Gð r   