Ë
    Y}j  ã                   óæ  — d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
mZ ddlmZ ddlZddlZ eej"                  «      dkD  r2ej"                  d   j%                  «       r eej"                  d   «      ndZe� ed ee«      z   «        ej.                  dd	«      Zd
ZdZdZedz   Z ej:                  «        e	d¬«      z
  j=                  d«      ZddiZ  ede› �«       d„ Z! ed«        ejD                   e!«       «      Z# ed ee#«      › �«       e#D � cg c]  } | d   ek\  sŒ| d   | d   | d   f‘Œ c} Z$ ee#«       ee$«      z
  Z% ed ee$«      › de%› �«        ejL                  ed¬«      Z'e'jQ                  «       Z)dxZ*xZ+Z, e-e$d«      D �]\  \  Z.\  Z/Z0Z1e0je                  d«      re0nee0z   Z3e)ji                  de3ef«       e)jk                  «       re+dz  Z+ŒM	  ejl                  e3e d ¬!«      Z7d"e7_8        e7jr                  Z:e/Z=e1Z>d'Z? ej€                  d(e:«      ZAeAr>eAj…                  d«      j‡                  «       reAj…                  d«      j‡                  «       Z= ej€                  d)e:«      ZDeDr5 ej€                  d*eDj…                  d«      «      ZEeEreEj…                  d«      Z> ej€                  d+e:ejŒ                  «      ZGeGr eGj…                  d«      j‡                  «       Z?n? ej€                  d,e:ejŒ                  «      ZHeHreHj…                  d«      j‡                  «       Z?e?se,dz  Z,�Œ¡e>r ee>j“                  d-d'«      «      ndZJ ee?d.«      ZKeKj™                  d/¬0«      dd1 ZMe)ji                  d2ee=e3e>e?eJeeMd3f	«       e)jœ                  dkD  re*dz  Z*e'jŸ                  «        e.d4z  dk(  se. ee$«      k(  r ed5e.› d6 ee$«      › d7e*› d8e+› d9e,› d:�«        ej                   d;«       �Œ_ e'j£                  «         ed<e*› d8e+› d9e,› d:�«        e ej¤                  ee*e+e,d=œd>¬?«      «       yc c} w # e;$ r&Z<e,dz  Z,e.d#k  r ed$e3dd% › d&e<› �«       Y dZ<[<�ŒÍdZ<[<ww xY w)@u‘   
crawl_lnjp.py - å»ºå¹³åŽ¿äººæ°‘æ”¿åºœ-å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·æ–‡ä»¶å®¡æ‰¹
éœ€è¦�PlaywrightèŽ·å�–åˆ—è¡¨é¡µï¼ˆCMS APIéœ€æµ�è§ˆå™¨çŽ¯å¢ƒï¼‰
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoup)Úasync_playwrighté   z[AutoPg] max_pages=Ú	SEARCH_DBz/root/search.dbu@   å»ºå¹³åŽ¿äººæ°‘æ”¿åºœ-å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·æ–‡ä»¶å®¡æ‰¹u   çŽ¯è¯„å…¬ç¤ºzhttps://www.lnjp.gov.cnz0/jpxzf/26gly/sthj/xzxk/jsxmhjyxpjwjsp/glist.htmliG  )Údaysz%Y-%m-%dú
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36z[lnjp] 3-year cutoff: c            
   ƒ   ó,  K  — g } t        «       4 ƒd{  –—† }|j                  j                  d¬«      ƒ d{  –—† }|j                  t        d   ¬«      ƒ d{  –—† }|j                  t        dd¬«      ƒ d{  –—†  |j                  d	«      ƒ d{  –—†  |j                  d
«      ƒ d{  –—† }d}|j                  d«      ƒ d{  –—† }| j                  |«       t        dt        |«      › d�«       t        dt        |t        xs |«      dz   «      D ]œ  }|j                  d|«      ƒ d{  –—† }|s |j                  d|dz
  › d�«      ƒ d{  –—†  |j                  d	«      ƒ d{  –—†  |j                  d«      ƒ d{  –—† }| j                  |«       t        d|› dt        |«      › d�«       Œž |j!                  «       ƒ d{  –—†  ddd«      ƒd{  –—†  | S 7 �ŒÊ7 �Œ©7 �Œ‹7 �Œn7 �ŒX7 �ŒB7 �Œ*7 ŒÅ7 Œ¥7 ŒŽ7 Œw7 Œ37 Œ%# 1 ƒd{  –—†7  sw Y   | S xY w­w)z/Use Playwright to get list items from all pagesNT)Úheadlessr
   )Ú
user_agentÚcommiti0u  )Ú
wait_untilÚtimeouti¸  z.nowpageé   aÞ  () => {
            var lis = document.querySelectorAll('ul.list-wrap1 li');
            return Array.from(lis).map(function(li) {
                var a = li.querySelector('a');
                var span = li.querySelector('span');
                return {
                    title: a ? a.textContent.trim() : '',
                    url: a ? a.getAttribute('href') : '',
                    date: span ? span.textContent.trim() : ''
                };
            });
        }z
  Page 1: z itemsé   r   a  (pg) => {
                var links = document.querySelectorAll('a.nextpage, a.lastpage');
                if (links.length > 0) {
                    // Click the next page link
                    var nextLink = document.querySelector('a.nextpage');
                    if (nextLink) { nextLink.click(); return true; }
                    var lastLink = document.querySelector('a.lastpage');
                    if (lastLink) { lastLink.click(); return true; }
                }
                return false;
            }zpageAutoLink(ú)a
  () => {
                var lis = document.querySelectorAll('ul.list-wrap1 li');
                return Array.from(lis).map(function(li) {
                    var a = li.querySelector('a');
                    var span = li.querySelector('span');
                    return {
                        title: a ? a.textContent.trim() : '',
                        url: a ? a.getAttribute('href') : '',
                        date: span ? span.textContent.trim() : ''
                    };
                });
            }z  Page z: )r   ÚchromiumÚlaunchÚnew_pageÚHEADERSÚgotoÚLIST_URLÚwait_for_timeoutÚtext_contentÚevaluateÚextendÚprintÚlenÚrangeÚminÚ_MAX_PGÚclose)	Ú	all_itemsÚpÚbrowserÚpageÚtotal_pages_textÚtotal_pagesÚitemsÚpgÚclickeds	            ú/root/gov_crawler/crawl_lnjp.pyÚget_all_itemsr.      s  è ø€ à€IÜÓ!÷ >ð > QØŸ
™
×)Ñ)°4Ð)Ó8×8ˆØ×%Ñ%´¸Ñ1FÐ%ÓG×Gˆà�i‰iœ¨X¸uˆiÓE×EÐEØ×#Ñ# DÓ)×)Ð)ð "&×!2Ñ!2°:Ó!>×>ÐØˆð —m‘mð %ó ÷ ˆð 	×Ñ˜ÔÜ�
œ3˜u›:˜, fÐ-Ô.ô ˜œ3˜{¬GÒ,B°{ÓCÀaÑGÓHò  	6ˆBØ ŸM™Mð 
+ð ó
÷ 
ˆGñ à—m‘m m°B¸±F°8¸1Ð$=Ó>×>Ð>à×'Ñ'¨Ó-×-Ð-àŸ-™-ð )ó ÷ ˆEð ×Ñ˜UÔ#Ü�G˜B˜4˜r¤# e£* ¨VÐ4Õ5ðA 	6ðD �m‰m‹o×Ð÷}>÷ >ð~ Ðð>ùØ8ùØGùàEùØ)ùð ?ùðùð"
øð ?øà-øðøð 	øð}>ø÷ >÷ >ð >ð~ Ðüs
  ‚H“G”H— G>·G ¸ G>ÁG#ÁG>Á8G&Á9G>ÂG)ÂG>Â*G,Â+G>ÃG/ÃA&G>Ä,G2Ä-!G>ÅG4ÅG>Å'G6Å(G>Æ G8ÆAG>ÇG:ÇG>ÇHÇG<ÇHÇ G>Ç#G>Ç&G>Ç)G>Ç,G>Ç/G>Ç2G>Ç4G>Ç6G>Ç8G>Ç:G>Ç<HÇ>HÈHÈHÈHz&[lnjp] Fetching list via Playwright...z[lnjp] Total items: ÚdateÚtitleÚurlz[lnjp] Within 3yr: z	, older: é   )r   Úhttpz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?é   )Úheadersr   zutf-8é   z  [WARN] Fetch error: é<   z -> Ú z!ArticleTitle"\s*content="([^"]+)"zPubDate"\s*content="([^"]+)"z(\d{4}-\d{2}-\d{2})z><div class="center-info"[^>]*>(.*?)</div>\s*<div class="other"z)<div class="center-info"[^>]*>(.*?)</div>ú-zhtml.parserT)ÚstripéÈ   zŸINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category, summary, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)zcrawl_lnjp.pyé
   z  Progress ú/z: +z new, z dup, z errg333333Ó?z
[lnjp] Summary: +)ÚsiteÚnewÚdupÚerrF)Úensure_ascii)SÚ__doc__ÚosÚreÚsysÚjsonÚtimeÚasyncioÚsqlite3r   r   Úbs4r   Úplaywright.async_apir   ÚrequestsÚ_SYSr   ÚargvÚisdigitÚintr"   r   ÚstrÚgetenvÚDB_PATHÚ	SITE_NAMEÚCATEGORYÚBASEr   ÚnowÚstrftimeÚCUTOFF_DATEr   r.   Úrunr$   ÚactiveÚskippedÚconnectÚconnÚcursorÚcurÚ	new_countÚ	dup_countÚerror_countÚ	enumerateÚidxr0   ÚhrefÚdate_strÚ
startswithÚ
detail_urlÚexecuteÚfetchoneÚgetÚrespÚencodingÚtextÚhtmlÚ	ExceptionÚeÚd_titleÚd_dateÚbodyÚsearchÚmtÚgroupr:   ÚmdÚmÚDOTALLÚciÚci2ÚreplaceÚ	date_rankÚ	text_soupÚget_textÚsummaryÚrowcountr   Úsleepr#   Údumps)Úitems   0r-   ú<module>rˆ      s°  ðñ÷ 1× 0× 0Ó 0ß (Ý Ý 1Û Û Ù" 4§9¡9›~°Ò1°d·i±iÀ±l×6JÑ6JÔ6L‰#ˆd�i‰i˜‰lÔ
ÐRV€Ø
ÐÙ	Ð
¡# g£,Ñ
.Ô/ð ˆ"�)‰)�KÐ!2Ó
3€ØN€	Ø€Ø €ØÐDÑD€Øˆx�|‰|‹~¡	¨uÔ 5Ñ5×?Ñ?À
ÓK€ØÐWÐ
X€á Ð˜{˜mÐ,Ô -òBñH Ð.Ô /ØˆG�K‰K™›Ó(€	Ù Ð™S ›^Ð,Ð-Ô .ð BKÖ	j¸ÈdÐSYÉlÐ^iÓNiˆ4�‰=˜$˜u™+ t¨F¡|Ò
4Ò	j€Ù
ˆi‹.™3˜v›;Ñ
&€Ù Ð™C ›K˜=¨	°'°Ð;Ô <à€w‡��w¨Ô+€Ø
‡k�kƒm€Ø&'Ð '€	Ð 'ˆI˜á$-¨f°aÓ$8ó ?Ñ €CÑ	 ˆ%��xØŸ™¨Ô0‘°d¸T±k€Jà‡K�KÐIÈJÐXaÐKbÔcØ
‡|�|„~Ø�Q‰ˆ	ØðØˆx�|‰|˜J°ÀÔDˆØˆŒØ�y‰yˆð €GØ€FØ€Dð 
ˆ�‰Ð7¸Ó	>€BÙ	ˆb�h‰h�q‹k×ÑÔ!Ø—(‘(˜1“+×#Ñ#Ó%ˆà	ˆ�‰Ð2°DÓ	9€BÙ	ØˆB�I‰IÐ,¨b¯h©h°q«kÓ:ˆÙØ—W‘W˜Q“ZˆFð 
ˆ�‰ÐTÐVZÐ\^×\eÑ\eÓ	f€BÙ	Ø�x‰x˜‹{× Ñ Ó"‰àˆb�i‰iÐDÀdÈBÏIÉIÓVˆÙØ—9‘9˜Q“<×%Ñ%Ó'ˆDáØ�qÑˆÙá06‘�F—N‘N 3¨Ó+Ô,¸A€IÙ˜d MÓ2€IØ× Ñ  tÐ Ó,¨T¨cÐ2€Gà‡K�Kð	-ð 
�G˜Z¨°°yÀ(ÈGÐUdÐeô	ð ‡|�|�aÒØ�Q‰ˆ	à‡K�K„MØ
ˆR�x�1‚}˜™s 6›{Ò*Ù�˜C˜5 ¡# f£+ ¨c°)°¸FÀ9À+ÈVÐT_ÐS`Ð`dÐeÔfà€D‡J�Jˆs†Oð?ðB ‡
�
„Ù Ð˜I˜; f¨Y¨K°v¸k¸]È$ÐOÔ PÙ €j€d‡j�j˜)¨I¸iÐP[Ñ\ÐkpÔqÕ rùòW 
køð( ò Ø�qÑˆØ�!Š8ÙÐ*¨:°c°r¨?Ð*;¸4À¸sÐCÔDÝûð	ús$   Ä%Q Ä3Q Ç,(QÑQ0Ñ
Q+Ñ+Q0