Ë
    ñµdj  ã                   ó¼   — d Z ddlZddlZddlZddlZddlZddlZddlmZ dZ	dZ
dZdZedz   Zed	z   Zd
dddœZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uy   
Crawler for æ¸…è¿œå¸‚æ¸…åŸŽåŒº-æ”¿åŠ¡åŠ¨æ€� (gkmlpt)
API: /qyqczsj/gkmlpt/api/all/3423 (list) + /api/post/{id} (detail)
é    N)ÚBeautifulSoupz/root/search.dbu   æ¸…è¿œå¸‚æ¸…åŸŽåŒº-æ”¿åŠ¡åŠ¨æ€�u   çŽ¯è¯„å…¬ç¤ºzhttp://www.qingcheng.gov.cnz/qyqczsj/gkmlpt/api/all/3423z/qyqczsj/gkmlpt/api/postz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36z!application/json, text/plain, */*zzh-CN,zh;q=0.9)z
User-AgentÚAcceptzAccept-Languageé.   c                 ó.   — | xs dj                  dd«      S )NÚ ú'z'')Úreplace)Úss    ú"/root/gov_crawler/crawl_qyqczsj.pyÚescr      s   € ØŠG�×Ñ˜S $Ó'Ð'ó    c                 ó°   — ddl }	 |j                   j                  | |j                  |j                  d¬«      «      ¬«      j	                  d«      S #  Y yxY w)z$Convert unix timestamp to YYYY-MM-DDr   Né   )Úhours)Útzz%Y-%m-%dr   )ÚdatetimeÚfromtimestampÚtimezoneÚ	timedeltaÚstrftime)Útsr   s     r   Útimestamp_to_dater      sV   € ãðØ× Ñ ×.Ñ.¨r°h×6GÑ6GÈ×HZÑHZÐabÐHZÓHcÓ6dÐ.Óe×nÑnÐoyÓzÐzøðÙús   †A
A ÁAc                 óî   — dt         | t        fz  }	 t        j                  |t        d¬«      }|j
                  dk(  r|j                  «       S 	 y# t        $ r}t        d| |fz  d¬«       Y d}~yd}~ww xY w)	zFetch list page from APIz%s?page=%d&limit=%dé   ©ÚheadersÚtimeoutéÈ   z  [ERR] List page %d: %sT©ÚflushN)	ÚLIST_APIÚPER_PAGEÚrequestsÚgetÚHEADERSÚstatus_codeÚjsonÚ	ExceptionÚprint)ÚpageÚurlÚrÚes       r   Ú
fetch_listr.   &   sz   € à
¤8¨T´8Ð"<Ñ
<€CðBÜ�L‰L˜¤g°rÔ:ˆØ�=‰=˜CÒØ—6‘6“8ˆOð  ð øô ò BÜÐ(¨D°!¨9Ñ4¸D×AÑAØûðBús   ’:A Á	A4ÁA/Á/A4c                 óä   — dt         | fz  }	 t        j                  |t        d¬«      }|j                  dk(  r|j                  «       S 	 y# t        $ r}t        d| |fz  d¬«       Y d}~yd}~ww xY w)	zFetch detail from APIú%s/%dr   r   r   z  [ERR] Detail %d: %sTr   N)Ú
DETAIL_APIr#   r$   r%   r&   r'   r(   r)   )Ú
article_idr+   r,   r-   s       r   Úfetch_detailr3   2   sx   € à
”Z Ð,Ñ
,€CðEÜ�L‰L˜¤g°rÔ:ˆØ�=‰=˜CÒØ—6‘6“8ˆOð  ð øô ò EÜÐ%¨°Q¨Ñ7¸t×DÑDØûðEús   �:A
 Á
	A/ÁA*Á*A/c                 óî   — | syt        | d«      }|j                  d«      }|rAg }|D ])  }|j                  dd¬«      }|sŒ|j                  |«       Œ+ dj	                  |«      S |j                  dd¬«      S )z*Extract clean text from HTML content fieldr   zhtml.parserÚpT)Ú	separatorÚstripz

)r   Úfind_allÚget_textÚappendÚjoin)Úhtml_contentÚsoupÚpsÚparasr5   Útxts         r   Úextract_contentrA   >   s~   € áØÜ˜ }Ó5€DØ	�‰�sÓ	€BÙ	ØˆØò 	"ˆAØ—*‘* r°�*Ó6ˆCÚØ—‘˜SÕ!ð	"ð �{‰{˜5Ó!Ð!Ø�=‰= 2¨Tˆ=Ó2Ð2r   c           
      óÆ  — |xs dd d }dt        | «      t        |«      t        |«      t        |«      t        t        t        |«      dœz  }t        j                  dt
        |gddd¬«      }|j                  d	k7  r7d
|j                  vr)t        d|j                  z  t        j                  ¬«       yddt        | «      iz  }t        j                  dt
        |gddd¬«       y)Nr   r   z�INSERT INTO gov_raw (page_url, title, publish_date, content, site_name, industry, summary)
VALUES ('%(u)s','%(t)s','%(d)s','%(c)s','%(s)s','%(i)s','%(sum)s'))ÚuÚtÚdÚcr
   ÚiÚsumÚsqlite3Té
   )Úcapture_outputÚtextr   r   ÚUNIQUEz  [DB ERROR] %s)ÚfileFzÑINSERT INTO gov_search (rowid, title, site_name, summary)
SELECT r.rowid, r.title, r.site_name, r.summary
FROM gov_raw r LEFT JOIN gov_search s ON r.rowid = s.rowid
WHERE r.page_url='%(u)s' AND s.rowid IS NULLrC   é   )
r   Ú	SITE_NAMEÚINDUSTRYÚ
subprocessÚrunÚDB_PATHÚ
returncodeÚstderrr)   Úsys)Úpage_urlÚtitleÚcontentÚpub_dateÚsummaryÚsqlÚresultÚfts_sqls           r   Ú
insert_oner`   N   sÚ   € ØŠ}˜"˜d˜sÐ#€GðFä�‹]¤ U£´#°h³-Ü�‹\¤	´ÄÀWÃñIñ€Cô
 �^‰^Ø	”G˜SÐ!Ø $°ô€Fð ×Ñ˜AÒ (°&·-±-Ñ"?ÜÐ &§-¡-Ñ/´c·j±jÕAØð0ð 47¼¸H»Ð2FñG€Gô ‡N�N�Iœw¨Ð0ÀÈDÐZ\Õ]Ør   c            	      óô  — d} t        t        j                  «      dkD  r	 t        t        j                  d   «      } t        dt        | fz  d¬«       g }t        d| dz   «      D ]s  }t        |«      }|rd|vr n`|d   }|s nWt        d|t        |«      fz  d¬«       |j                  |«       t        |«      t        k  r nt        j                  d«       Œu t        d	t        |«      z  d¬«       d
}d
}t        |«      D �]R  \  }}|d   }	|d   }
t        d|dz   t        |«      |
d d fz  d¬«       d}d|v r|d   rt        |d   «      }|s9d|v r5t        j                   dt#        |d   «      «      }|r|j%                  d«      }t'        |	«      }|rrt)        |j+                  dd«      «      }|j+                  dd«      xs |
}|sCd|v r?|d   r:t        j                   dt#        |d   «      «      }|r|j%                  d«      }n|
}|
}|j+                  ddt,        |	fz  «      }t/        ||||«      r|dz  }n|dz  }t        j                  d«       �ŒU t        d||fz  d¬«       y # t        $ r Y �Œ:w xY w)Né   é   z![%s] Starting crawl, max_pages=%dTr   Úarticlesz  Page %d: %d itemsg333333Ó?zTotal items: %dr   ÚidrY   z  [%d/%d] %s...é(   r   ÚdateÚ
created_atz(\d{4}-\d{2}-\d{2})rZ   Úpublish_timer+   r0   gš™™™™™É?z
Done! New: %d, Duplicates: %d)ÚlenrW   ÚargvÚintÚ
ValueErrorr)   rP   Úranger.   Úextendr"   ÚtimeÚsleepÚ	enumerater   ÚreÚsearchÚstrÚgroupr3   rA   r$   r1   r`   )Ú	max_pagesÚall_articlesr*   ÚdataÚartsÚ	new_countÚ	dup_countrG   ÚartÚaidrY   r[   ÚmÚdetailrZ   Údetail_titlerX   s                    r   Úcrawlr‚   e   s‡  € Ø€IÜ
Œ3�8‰8ƒ}�qÒð	ÜœCŸH™H Q™KÓ(ˆIô 
Ð
-´¸IÐ0FÑ
FÈdÕSà€LÜ�a˜ Q™Ó'ò ˆÜ˜$ÓˆÙ�z¨Ñ-ÙØ�JÑˆÙÙÜÐ# t¬S°«YÐ&7Ñ7¸tÕDØ×Ñ˜DÔ!Üˆt‹9”xÒÙÜ�
‰
�3�ðô 
Ð
œc ,Ó/Ñ
/°tÕ<à€IØ€IÜ˜LÓ)ó !‰ˆˆ3Ø�$‰iˆØ�G‘ˆÜÐ 1 Q¡3¬¨LÓ(9¸5ÀÀ"¸:Ð"FÑFÈdÕSð ˆØ�S‰=˜S š[Ü(¨¨V©Ó5ˆHÙ˜L¨CÑ/Ü—	‘	Ð0´#°c¸,Ñ6GÓ2HÓIˆAÙØŸ7™7 1›:�ô ˜cÓ"ˆÙÜ% f§j¡j°¸BÓ&?Ó@ˆGØ!Ÿ:™: g¨rÓ2Ò;°eˆLÙ °&Ñ 8Ø˜.Ò)ÜŸ	™	Ð"8¼#¸fÀ^Ñ>TÓ:UÓV�AÙØ#$§7¡7¨1£:™àˆGØ ˆLà—7‘7˜5 '¬Z¸Ð,=Ñ"=Ó>ˆÜ�h ¨g°xÔ@Ø˜‰N‰Ià˜‰NˆIÜ�
‰
�3ŽðC!ôF 
Ð
+¨y¸)Ð.DÑ
DÈDÖQøôu ò 	Úð	ús    I* É*	I7É6I7Ú__main__)Ú__doc__r#   rs   rR   rW   rp   r'   Úbs4r   rT   rP   rQ   ÚBASE_URLr!   r1   r%   r"   r   r   r.   r3   rA   r`   r‚   Ú__name__© r   r   ú<module>r‰      s•   ðñ÷ 1× 0× 0Ý à
€Ø-€	Ø€Ø(€ØÐ4Ñ4€ØÐ2Ñ2€
ð QØ1Ø'ñ€ð
 €ò(òò	ò	ò3ò ò.?RðD ˆzÒÙ	…Gð r   