Ë
    é¡j\  ã                   óü   — d Z ddlZddlZddlZddlZddlZej                  j                  dd«       ddlZ	dZ
dZej                  j                  «       j                  d«      Zd„ Zd„ Zd	„ Zd
„ Zedk(  r e«        yy)uA  
Re-import IC13 TXT files with content comparison:
Only append if content after normalization actually differs.

æ ‡é¢˜æ��å�– 2026-09-10 èµ·å�•ç‚¹å¤�ç”¨ /root/gov_crawler/originals_sync.py:
  - åˆ é™¤æœ¬æ–‡ä»¶è‡ªèº«çš„ç®€åŒ– extract_title(07-08 v4), æ”¹ä¸º import originals_sync çš„
    extract_title/_tidy_title(09-03 v3/v4+tidy å…¨å¥—: R1å‰�ç¼€æ—�/Method NL/å­¤å„¿æ‹¬å�·/
    è§„æ¨¡å°¾ç¼€/marker cut/ä¸­è‹±æ¯”60%/å‡€åº¦å®ˆæŠ¤) â€”â€” ä¸Ž originals note 62K å¼•æ“Žå�Œä¸€å¥—é€»è¾‘,
    æ ‡é¢˜çŸ¥è¯†å�ªç»´æŠ¤ originals_sync.py ä¸€å¤„(ä¸‰å¤„å�Œæ­¥ md5 æ ¡éªŒ)ã€‚
  - resolve_title() å¤�åˆ» originals_sync main() çš„å‡€åº¦å®ˆæŠ¤å…œåº•(extract ç»“æžœå�« tab/_____/
    é•¿è‹±æ–‡/è¶…100å­—/é‡�å¤�æ‹¼æŽ¥ â†’ _tidy_title â†’ ä»�è„�å›žé€€çº¯ id_code)ã€‚
  - id_code/source='pharma'/append-on-change(content normalize å¯¹æ¯”) è¯­ä¹‰ä¸�å�˜ã€‚
é    Nz/root/gov_crawlerz/root/originals.dbz/tmp/ic13_copy_new/z%Y-%m-%dc                 óZ   — | sy| j                  «       }t        j                  dd|«      }|S )NÚ z\s+ú )ÚstripÚreÚsub)ÚtextÚts     ú"/root/scripts/import_ic13_dedup.pyÚ	normalizer      s*   € ÙØØ�
‰
‹€AÜ
�‰ˆv�s˜AÓ€AØ€Hó    c                 ó  — dD ],  }	 t        | d|¬«      5 }|j                  «       cd d d «       c S  t        | dd¬«      5 }|j                  «       cd d d «       S # 1 sw Y   nxY wŒd# t        t        f$ r Y Œuw xY w# 1 sw Y   y xY w)N)ÚgbkÚgb18030zutf-8Úr)Úencodingzlatin-1)ÚopenÚreadÚUnicodeDecodeErrorÚUnicodeError)ÚfilepathÚencÚfs      r   Úread_txtr   !   s–   € Ø*ò ˆð	Ü�h ¨cÔ2ð  °aØ—v‘v“x÷ ó  ðô 
ˆh˜ iÔ	0ð °AØ�v‰v‹x÷ñ ÷	 ð  úð  øä"¤LÐ1ò 	Ùð	ú÷ð ús3   ‡A'•A¥	A'Á A<ÁA#	ÁA'Á'A9Á8A9Á<Bc                 ó&  — t        j                  | |«      }|s|S t        j                  d|«      }t        j                  d|«      s-t	        |«      dkD  s|r:t	        |j                  d«      «      dk\  rt        j                  |«      }|r|}|S |}|S )ul   å¤�åˆ» originals_sync main() çš„æ��å�–+å‡€åº¦å®ˆæŠ¤å…œåº•, ä¿�è¯� IC13 ä¸Ž originals note æ ‡é¢˜æ ‡å°ºä¸€è‡´z
(.{8,}?)\1z[\t_]{2,}|[A-Za-z]{4,}éd   é   é   )ÚosxÚextract_titler   ÚsearchÚlenÚgroupÚ_tidy_title)Úfnamer	   Úid_codeÚtitleÚ_dupÚ_tts         r   Úresolve_titler*   ,   sŒ   € ä×Ñ˜e TÓ*€EÙØˆÜ�9‰9�] EÓ*€DÜ	‡y�yÐ*¨EÔ2´c¸%³jÀ3Ò6FÉ4ÔTWÐX\×XbÑXbÐcdÓXeÓTfÐjkÒTkÜ�o‰o˜eÓ$ˆÙØˆEð €Lð ˆEØ€Lr   c                  óš  — t         j                  j                  t        «      st	        dt        › d�«       y t        j                  t        «      } | j                  d«       | j                  d«       | j                  «       }i }|j                  d«      D ]"  }|d   r|d   nd||d   j                  «       <   Œ$ |j                  d	«      j                  «       d   xs d}t        t        j                  t        «      D �cg c]  }|j                  d
«      sŒ|‘Œ c}«      }t	        dt        |«      › �«       t	        dt        |«      › �«       dx}x}x}	}
g }g }|D ]ý  }|j!                  d
d«      }t         j                  j#                  t        |«      }	 t%        |«      }|j)                  dd«      }t        |«      dkD  r|d   j                  «       nd}t+        |||«      }||v rT||   }t-        |«      }t-        |«      }||k(  r|	dz  }	Œ±dt.        › d�}|r||z   |z   }|j1                  |||f«       |dz  }Œß|dz  }|j1                  ||||f«       |dz  }Œÿ t	        d«       |r |j3                  d|«       t	        d|› �«       |r |j3                  d|«       t	        d|› �«       | j5                  «        |j                  d«      j                  «       d   }| j7                  «        t	        d|	› �«       t	        d|
› �«       t	        d|› �«       y c c}w # t&        $ r!}t	        d|› d|› �«       |
dz  }
Y d }~�Œæd }~ww xY w)Nz
Directory z not found. Nothing to do.zPRAGMA journal_mode=OFFzPRAGMA synchronous=OFFzeSELECT id_code, content FROM tsk_data WHERE source='pharma' AND id_code IS NOT NULL AND id_code != ''r   r   r   zSELECT MAX(id) FROM tsk_dataz.txtzExisting pharma id_codes: zIC13 files: z  ERROR reading z: ú
u   

---
æ›´æ–°æ—¥æœŸ: z
---

z
Processing...zJUPDATE tsk_data SET title=?, content=? WHERE id_code=? AND source='pharma'z  Updates: zXINSERT INTO tsk_data (id, source, id_code, title, content) VALUES (?, 'pharma', ?, ?, ?)z  New inserts: z3SELECT COUNT(*) FROM tsk_data WHERE source='pharma'zSkipped (no change): z
  Errors: z  Total pharma records: )ÚosÚpathÚisdirÚIC13_DIRÚprintÚsqlite3ÚconnectÚDBÚexecuteÚcursorr   ÚfetchoneÚsortedÚlistdirÚendswithr"   ÚreplaceÚjoinr   Ú	ExceptionÚsplitr*   r   ÚTODAYÚappendÚexecutemanyÚcommitÚclose)ÚconnÚcÚexistingÚrowÚmax_idr   ÚfilesÚ	new_countÚupdate_countÚ
skip_countÚerror_countÚupdatesÚinsertsr%   r&   r   r	   ÚeÚpartsÚcontent_newr'   Úold_contentÚold_normÚnew_normÚ	SEPARATORÚtotals                             r   ÚmainrX   ;   s=  € Ü�7‰7�=‰=œÔ"Ü�
œ8˜*Ð$>Ð?Ô@Øä�?‰?œ2Ó€DØ‡L�LÐ*Ô+Ø‡L�LÐ)Ô*Ø�‰‹€Aà€HØ�y‰yØoóò <ˆð .1°ªV 3 q¢6¸ˆ��Q‘—‘“Ò ð<ð
 �Y‰YÐ5Ó6×?Ñ?ÓAÀ!ÑDÒIÈ€FäœrŸz™z¬(Ó3ÖJ˜!°q·z±zÀ&Õ7I’AÒJÓK€EÜ	Ð&¤s¨8£} oÐ
6Ô7Ü	ˆLœ˜U›˜Ð
%Ô&à:;Ð;€IÐ;�Ð;˜z¨KØ€GØ€Gàò !ˆØ—-‘- ¨Ó+ˆÜ—7‘7—<‘<¤¨%Ó0ˆð	Ü˜HÓ%ˆDð —
‘
˜4 Ó#ˆÜ*-¨e«*°qª.�e˜A‘h—n‘nÔ&¸bˆÜ˜e T¨7Ó3ˆà�hÑØ" 7Ñ+ˆKÜ  Ó-ˆHÜ  Ó-ˆHà˜8Ò#Ø˜a‘�
Øà1´%°¸	ÐBˆIÙØ)¨IÑ5¸ÑC�Ø�N‰N˜E ;°Ð8Ô9Ø˜AÑ‰Là�a‰KˆFØ�N‰N˜F G¨U°KÐ@ÔAØ˜‰N‰IðC!ôF 
ˆOÔÙØ	�‰ØXØô	
ô 	�˜L˜>Ð*Ô+ÙØ	�‰ØfØô	
ô 	� 	˜{Ð+Ô,à‡K�K„MØ�I‰IÐKÓL×UÑUÓWÐXYÑZ€EØ‡J�J„Lä	Ð! * Ð
.Ô/Ü	ˆJ�{�mÐ
$Ô%Ü	Ð$ U GÐ
,Õ-ùò Køô ò 	ÜÐ$ U G¨2¨a¨SÐ1Ô2Ø˜1ÑˆKÝûð	ús$   ÄLÄLÆL Ì 	M
Ì)MÍM
Ú__main__)Ú__doc__r-   Úsysr2   Údatetimer   r.   ÚinsertÚoriginals_syncr   r4   r0   ÚdateÚtodayÚstrftimer?   r   r   r*   rX   Ú__name__© r   r   ú<module>rd      s|   ðñ÷ &× %Ó %à ‡�‡��Ð&Ô 'Û à€Ø €Ø�‰×ÑÓ×&Ñ& zÓ2€òòòòQ.ðf ˆzÒÙ…Fð r   