Ë
    ¸¬uj	+  ã                   ó¬   — d Z ddlZddlZddlZddlZddlZddlZddlmZ ddlm	Z	 dZ
dZe› d�ZdZd	Zd
adZd„ Zdd„Zd„ Zd„ Zd„ Zedk(  r e«        yy)uN  
å®¿æ�¾åŽ¿ - åŽ¿ç»�å¼€åŒº > å›žåº”å…³åˆ‡ (susong.gov.cn)
CMS: é¾™è®¯ (Lonsun), AJAX åˆ†é¡µ
AJAX API: /site/label/8888?labelName=publicInfoList&...&pageIndex=N
Detail: /public/2000003991/{contentId}.html
  Content container: div.ls-content
  Attachments: PDF links inside content
Protection: Cloudflare JS Challenge (needs Playwright)
é    N)Údatetime)Úsync_playwrightu   å®¿æ�¾åŽ¿ç»�å¼€åŒº-å›žåº”å…³åˆ‡zhttps://www.susong.gov.cnzA/public/column/2000003991?type=4&catId=41334066&action=list&nav=3z/root/search.dbz
2023-01-01é   é   c                  ó4  — t        «       j                  «       } | j                  j                  dg d¢¬«      }|j	                  ddddœdd	¬
«      }|j                  d«       |j                  «       }t        dt        › �«       |j                  t        dd¬«       |j                  d«       |j                  «       }d|v s|sDt        d|› �«       |j                  d¬«       |j                  «        | j                  ddd«       yt        d|› �«       | |||fS )zGUse Playwright to solve Cloudflare JS challenge and return context/pageT©z---disable-blink-features=AutomationControlledz--no-sandboxz--disable-dev-shm-usage©ÚheadlessÚargszoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36é€  é8  ©ÚwidthÚheightúzh-CNúAsia/Shanghai©Ú
user_agentÚviewportÚlocaleÚtimezone_idz³
        Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
        Object.defineProperty(navigator, 'languages', { get: () => ['zh-CN', 'zh', 'en'] });
    z([INFO] Solving Cloudflare challenge for é`ê  Údomcontentloaded©ÚtimeoutÚ
wait_untilé@  Ú521z2[ERROR] Cloudflare challenge not bypassed, title: z/tmp/susong_cf_error.png)ÚpathN)NNNz[OK] Page loaded: )r   Ú	__enter__ÚchromiumÚlaunchÚnew_contextÚadd_init_scriptÚnew_pageÚprintÚLIST_URLÚgotoÚwait_for_timeoutÚtitleÚ
screenshotÚcloseÚ__exit__)ÚpÚbrowserÚcontextÚpager*   s        ú&/root/gov_crawler/crawl_susong_hyqg.pyÚsolve_cloudflarer3      s%  € äÓ×#Ñ#Ó%€AØ�j‰j×ÑØò
ð  ó €Gð ×!Ñ!ð EØ¨4Ñ0ØØ#ð	 "ó €Gð ×Ñð ô 	ð ×ÑÓ€Dä	Ð4´X°JÐ
?Ô@Ø‡I�IŒh Ð2D€IÔEØ×Ñ˜$Ôð �J‰J‹L€EØ��~™UÜÐBÀ5À'ÐJÔKØ�‰Ð7ˆÔ8Ø�‰ŒØ	�
‰
�4˜˜tÔ$Øä	Ð˜u˜gÐ
&Ô'Øˆg�w Ð$Ð$ó    c           
      ó¶  — t         › dt        j                  «       › dt        › d|› d�}	 | j                  d|› d�«      }t	        j
                  |«      }|j                  dg «      }|j                  dd	«      }|j                  d
d«      }t        d|› dt        |«      › d|› d|› �«       |||fS # t        $ r }t        d|› d|› �«       g d	d	fcY d}~S d}~ww xY w)z(Fetch list via AJAX API using Playwrightz/site/label/8888?_=zH&labelName=publicInfoList&siteId=2000005581&organId=2000003991&pageSize=z&pageIndex=z“&isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&isDriving=true&result=&isJson=true&keyWords=&isSetValue=true&catIds=&catId=41334066z4async () => {
            const resp = await fetch('z3');
            return await resp.text();
        }ÚdataÚtotalr   Ú	pageCountr   z  [API] pageIndex=z, z items, total=z, pages=z  [API ERROR] page ú: N)
ÚBASE_URLÚtimeÚITEMS_PER_PAGEÚevaluateÚjsonÚloadsÚgetr&   ÚlenÚ	Exception)	Úpage_objÚ
page_indexÚurlÚresultr6   Úitemsr7   Ú
page_countÚes	            r2   Úfetch_list_apirJ   G   s  € äˆZð Ü—‘“�ðô 'Ð'Ø˜˜ð?ð	@€CðØ×"Ñ"ð ('Ø'* eð ,ð$ó ˆô �z‰z˜&Ó!ˆØ—‘˜ Ó$ˆØ—‘˜ !Ó$ˆØ—X‘X˜k¨1Ó-ˆ
ÜÐ" : ,¨b´°U³°¸NÈ5È'ÐQYÐZdÐYeÐfÔgØ�e˜ZÐ'Ð'øÜò ÜÐ# J <¨r°!°Ð5Ô6Ø�1�aˆx�ûðús   ªBB/ Â/	CÂ8CÃCÃCc                 óÎ   — t         › d|› d�}	 | j                  d|› d�«      }t        j                  |«      }|S # t        $ r!}t        d|› d|› �«       ddg dœcY d	}~S d	}~ww xY w)
z8Fetch detail page using in-page fetch (faster than goto)z/public/2000003991/z.htmlzJasync () => {
            try {
                const resp = await fetch('uÃ  ');
                const html = await resp.text();
                
                // Parse HTML to extract content
                const parser = new DOMParser();
                const doc = parser.parseFromString(html, 'text/html');
                
                // Title
                const pageTitle = (doc.title || '').replace('_ä¿¡æ�¯å…¬å¼€_å®¿æ�¾åŽ¿äººæ°‘æ”¿åºœ', '').trim();
                
                // Content
                const contentEl = doc.querySelector('div.ls-content');
                if (!contentEl) return JSON.stringify({title: pageTitle, content: '', attachments: []});
                
                // Attachments
                const attachmentLinks = [];
                const anchors = contentEl.querySelectorAll('a[href$=".pdf"]');
                anchors.forEach(a => {
                    const href = a.href || a.getAttribute('href') || '';
                    const text = (a.innerText || '').trim();
                    const fullHref = href.startsWith('http') ? href : 'https://www.susong.gov.cn' + href;
                    if (fullHref && !attachmentLinks.some(x => x.href === fullHref)) {
                        attachmentLinks.push({href: fullHref, text: text || fullHref.split('/').pop()});
                    }
                });
                
                const textContent = contentEl.innerText || '';
                
                return JSON.stringify({
                    title: pageTitle,
                    content: textContent,
                    attachments: attachmentLinks
                });
            } catch(e) {
                return JSON.stringify({title: '', content: '', attachments: [], error: e.toString()});
            }
        }z  [DETAIL ERROR] r9   Ú )r*   ÚcontentÚattachmentsN)r:   r=   r>   r?   rB   r&   )rC   Ú
content_idrE   rF   r6   rI   s         r2   Úfetch_detailrP   d   s‡   € äˆJÐ)¨*¨°UÐ
;€Cð-?Ø×"Ñ"ð (+à+.¨%ð $0ð&$ó &ˆôP �z‰z˜&Ó!ˆØˆøÜò ?ÜÐ! #  b¨¨Ð,Ô-Ø¨¸2Ñ>Õ>ûð?ús   Ž+: º	A$ÁAÁA$ÁA$c                 óN  — t        j                  t        d¬«      }|j                  «       }d}d}| D ]Ÿ  }|j	                  dd«      }|j	                  dd«      }|j	                  dd«      }|j	                  dd«      }	|j	                  d	d«      }
|	s|s|d
z  }Œg	 |j                  d||t        ||	|
||f«       |j                  dkD  r|d
z  }n|d
z  }Œ¡ |j                  «        |j                  «        ||fS # t        $ r#}t        d|dd › d|› �«       |d
z  }Y d}~Œíd}~ww xY w)zInsert items into gov_rawé   ©r   r   r*   rL   rE   ÚdaterM   rN   r   a  
                INSERT OR IGNORE INTO gov_raw
                (title, page_url, site_name, publish_date, content, summary, attachments, source_url, date_rank)
                VALUES (?, ?, ?, ?, ?, '', ?, ?, CAST(strftime('%s', ?) AS INTEGER))
            z  [DB ERROR] Nr9   )Úsqlite3ÚconnectÚDB_PATHÚcursorr@   ÚexecuteÚ	SITE_NAMEÚrowcountrB   r&   Úcommitr,   )rG   ÚconnÚcÚinsertedÚskippedÚitemr*   rE   rT   rM   rN   rI   s               r2   Úinsert_to_dbrb   ˜   s:  € ä�?‰?œ7¨BÔ/€DØ�‰‹€AØ€HØ€Gàò ˆØ—‘˜ "Ó%ˆØ�h‰h�u˜bÓ!ˆØ�x‰x˜ Ó#ˆØ—(‘(˜9 bÓ)ˆØ—h‘h˜}¨bÓ1ˆá™uØ�q‰LˆGØð	Ø�I‰Ið ð
 �sœI t¨W°kÀ3Èðô	ð �z‰z˜AŠ~Ø˜A‘‘à˜1‘�øð-ð6 	‡K�K„MØ‡J�J„LØ�WÐÐøô ò 	Ü�M %¨¨ * ¨R°¨sÐ3Ô4Ø�q‰L�Gûð	ús   Â8C8Ã8	D$ÄDÄD$c                  óª  — t         } t        j                  dd  D ]  }|j                  «       sŒt	        |«      } Œ  t        dt        › d| › �«       d\  }}}}	 t        «       j                  «       }|j                  j                  dg d¢¬«      }|j                  dd	d
dœdd¬«      }|j                  d«       |j                  «       }t        d«       |j                  t        dd¬«       |j!                  d«       |j#                  «       }d|v s|s3t        d|› �«       	 |r|j%                  «        	 |j'                  «        y t        d|› �«       g }d}	d}
t)        d| dz   «      D �]I  }t+        ||«      \  }}}|st        d|› �«        �n&|D ]ë  }|j-                  d«      }|j-                  dd«      }|j-                  dd«      }|j-                  dd«      d d }|t.        k  rt        d |› d!t.        › �«        n�t        d"|› d#|› d$|d d% › d&�«       t1        ||«      }d}|j-                  d'g «      D ]  }|r|d(z  }|d)|d*   › d+|d,   › d-�z  }Œ |j-                  d.d«      }|j3                  |||||d/œ«       Œí |s�Œt5        |«      \  }}|	|z  }	|
|z  }
t        d0|› d1|› d2|› d3�«       g }�ŒL |rt5        |«      \  }}|	|z  }	|
|z  }
t        d4|	› d5|
› �«       |	dkD  rWt7        j8                  t:        d6¬7«      }|j=                  d8«       |j?                  «        |j%                  «        t        d9«       |r|j%                  «        	 j'                  «        y #  Y y xY w# t@        $ r,}t        d:|› �«       dd l!}|jE                  «        Y d }~Œ[d }~ww xY w#  Y y xY w# |r|j%                  «        	 j'                  «        w #  Y w xY wxY w);Nr   z[INFO] u    - çˆ¬è™«, max_pages=)NNNNTr   r	   z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36r   r   r   r   r   r   z¿
            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
            Object.defineProperty(navigator, 'languages', { get: () => ['zh-CN', 'zh', 'en'] });
        z&[INFO] Solving Cloudflare challenge...r   r   r   r   r   z)[ERROR] Cloudflare bypass failed, title: z[OK] Cloudflare bypassed: r   z  [STOP] No items on page Ú	contentIdr*   rL   ÚlinkÚpublishDateé
   z  [STOP] Date z < z  [z] ú é2   z...rN   ú
ú[Útextz](Úhrefú)rM   )r*   rE   rT   rM   rN   z  [DB] Page z: +z new, z	 existingu   
[DONE] æ–°å¢ž: u
   , è·³è¿‡: rR   rS   z4INSERT INTO gov_search(gov_search) VALUES('rebuild')z[FTS] Rebuiltz[ERROR] )#ÚPAGES_DEFAULTÚsysÚargvÚisdigitÚintr&   rZ   r   r    r!   r"   r#   r$   r%   r(   r'   r)   r*   r,   ÚstopÚrangerJ   r@   ÚCUTOFF_DATErP   Úappendrb   rU   rV   rW   rY   r\   rB   Ú	tracebackÚ	print_exc)Ú	max_pagesÚargÚp_objr/   r0   r1   Úpwr*   Ú	all_itemsÚ	total_newÚ	total_oldrD   rG   r7   rH   ra   rO   re   Úpub_dateÚdetailÚ
attach_strÚattÚcontent_textÚnewÚoldr]   rI   rx   s                               r2   Úmainrˆ   ¿   s&  € ä€IÜ�x‰x˜˜ˆ|ò !ˆØ�;‰;�=Ü˜C›‰Ið!ô 
ˆG”I�;Ð3°I°;Ð
?Ô@à$:Ñ!€Eˆ7�G˜TðjäÓ×(Ñ(Ó*ˆØ—+‘+×$Ñ$ØÚmð %ó 
ˆð ×%Ñ%ØUØ#¨tÑ4Ø¨ð &ó 
ˆð
 	×Ñð !ô 	ð ×ÑÓ!ˆäÐ6Ô8Ø�	‰	”( EÐ6Hˆ	ÔIØ×Ñ˜dÔ#à—
‘
“ˆØ�E‰>¡ÜÐ=¸e¸WÐEÔFØñX Ø�M‰MŒOð	Ø�G‰G�Iô] 	Ð*¨5¨'Ð2Ô3ð ˆ	Øˆ	Øˆ	ä  9¨q¡=Ó1ó .	ˆJÜ'5°d¸JÓ'GÑ$ˆE�5˜*áÜÐ2°:°,Ð?Ô@Úàò �Ø!ŸX™X kÓ2�
ØŸ™ ¨"Ó-�Ø—x‘x ¨Ó+�ØŸ8™8 M°2Ó6°s¸Ð;�àœkÒ)Ü˜N¨8¨*°C¼°}ÐEÔFÙä˜˜J˜< r¨(¨°1°U¸3¸B°Z°LÀÐDÔEô & d¨JÓ7�ð  �
Ø!Ÿ:™: m°RÓ8ò D�CÙ!Ø" dÑ*˜
Ø A c¨&¡k ]°"°S¸±[°MÀÐ"CÑC‘JðDð  &Ÿz™z¨)°RÓ8�à× Ñ Ø"ØØ$Ø+Ø#-ñ"õ ð3óD Ü'¨	Ó2‘��SØ˜SÑ �	Ø˜SÑ �	Ü˜ Z L°°C°5¸¸s¸eÀ9ÐMÔNØ’	ð].	ñ` Ü# IÓ.‰HˆC�Ø˜ÑˆIØ˜ÑˆIäÐ! ) ¨J°y°kÐBÔCà�qŠ=Ü—?‘?¤7°BÔ7ˆDØ�L‰LÐOÔPØ�K‰KŒMØ�J‰JŒLÜ�/Ô"ñ Ø�M‰MŒOð	Ø�G‰G�Iøð	Ùûô ò Ü�˜˜ˆnÔÛØ×Ñ×Ñûðûð	Ùûñ Ø�M‰MŒOð	Ø�G‰G�Iøð	Ùýsi   ÁCM' Ä4M  ÅD<M' ÊB9M' ÍN Í M$Í'	NÍ0"NÎN& ÎNÎN& ÎN#Î&OÎ:OÏ
OÏOÏOÚ__main__)r   )Ú__doc__rp   Úrer>   ÚosrU   r;   r   Úplaywright.sync_apir   rZ   r:   r'   rW   rv   ro   r<   r3   rJ   rP   rb   rˆ   Ú__name__© r4   r2   ú<module>r�      sƒ   ðñó Û 	Û Û 	Û Û Ý å /ð .€	Ø&€ØˆZÐXÐY€Ø
€Ø€Ø€Ø€ò%%óPò:1?òh$òNuðp ˆzÒÙ…Fð r4   