z(]*>z
r8 r
z z [\s\u200b\u200c\u200d\ufeff]+r. r/ z
__LINK__\d+__z __TBL__\d+__z[ \t]+\nz\n{3,}z [ \t]{2,}r &z&z AIKL vv?X]_]a]adfdhdh]hiHvvgz8244@HvvlD("$$?HvvA2xWYW[W[\H E ' KKM z2q)!!%(
fc*228SA7UC &&( DAq !QA
. 0FAs GA3b/3/A 0
QWWY ++e
C ""$ 1kk!Q M* 03kkGA3b/3/0
&&!2s
+C
&& "c
*C
C
C
&&dC
(C
&&FC
(C
&& sC
(C 5D E
YYv
ffVR
B!:; $;
Q ++e
"
"
$C ' YLGSsG,C7G3>@C ; &&r c g }t rt | d }|j dd }|r|j ddd D ] }|j d d
j }t
j j t | }|j d xs |j d d
}t | }d
} |j d }
|
r%|
j d } | r| j d
} t j d| sd
} |j ||| f |st j d| t j D ] } t
j j t | j# d }|j |t | j# d xs | j# d | j# d j f |S )z*Extract (url, title, date) from list page. html.parserdivbox_listclass_axxgklistT)rl rC rC r titler
)r lispanz^20\d{2}-\d{2}-\d{2}$zd]*href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)\s*([^<]*)r r7 )r findfind_allr r r@ rA rB LIST_URLget_textr find_parentr matchr0 finditerrP r1 )
html_textrJ rR soupboxrm rC rF ro daterp rq r3 s
r
parse_listr s E Y
6iiji1 \\#jt\D
5uuVR(..0 ,,..x>gE!**S**E#E*]]4( 776?D #}}4} 8xx 8$? D gud 34
5 E GP RT RV RV W _A ll**8QWWQZ@G LL';qwwqz/GQWWQZ#H!''RS*JZJZJ\] ^ _ Lr c d}d}d}t rt | d }|j ddt j dt j i }|r |j d rt
|d }|j ddt j dt j i }|r'|j d r|d j d d
}|j d d
}|rB|j d t j dt j }|r t | }|s t j d| }|rt
|j d }t j d| }|r"|j d j d d
}| j d } | dk7 rt j d| | d }|r| |j z }
| |
d } d} d}
t j d| D ]D }|j d j d r | dz } | dk( s.|j }
n| dz
} F |
dkD r| d |
}|||fS )z.Extract title, date, content from detail page.r rh metaname ArticleTitle)attrsrW PubDateN
ri mart)idtrs_editor_viewrk z'name="ArticleTitle"\s+content="([^"]*)"r z"name="PubDate"\s+content="([^"]*)"z id="mart"z+]*class="trs_editor_view[^"]*"[^>]*>r z]*>|z)r rt r compilerO r r r strrQ r1 startrz
startswithend)r{ rJ ro r~ content_htmlr| r3 r teidxr segdepthr mms r parse_detailr s E
DL Y
6 IIfVRZZ-N$OI P y! ) -E IIfVRZZ
BDD-I$JI K y!Y<%%',Dyy6y* 54F)MNB "2w II@) L
+E II;Y G 771:##%cr*Dnn[) "9I9UXUY?[Bbhhj( ' ++& d$d y # t $ r }
t# d
| d|
d Y d }
~
xd }
~
ww xY w# t $ r#}
t# d| d|
d |dz
}Y d }
~
d }
~
ww xY w)%Nr < )r zPRAGMA busy_timeout=60000zPRAGMA journal_mode=WALr index_z.html)r$ zutf-8r
)errorszPage z fetch error: T)flushz: found z itemszEmpty page, stop paginationz detail error z: r8 r r z skip empty: z + ( z ()g333333?g ?u 新增: u 跳过: z=== z done ===) r Sessionr updater r connectDB_PATHr rU r _PAGESrv r* rW decoder printr r2 rV r rf r r r r r r! commitcloser )r sr new_count
skip_count seen_urlspager# r( r{ r) rR rF ro r~ dhtmldtitleddater rW rY _ plain_lenoks r mainr # s AIIW ??7B /DLL,-LL*+IJIa!$ - 19C JfT!VHE2C c1%A (((CI 9c*
dV8CJ
y1#6dCa
s0 2*J=*J? J< J77J<? K+K&&K+__main__)Nr rr )(__doc__sysr< r r htmlr urllib.parser@ r bs4r ImportErrorBASE_URL LIST_PATHrv r r r r r
_MAX_PAGESrS argvr' rm r intr?
ValueErrorr2 r r r* rf r r r r __name__ r r r sq !
' ( i )
$
@O'
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA"b'J:+%\ >?3D zF a
M0
s5 C, )C9%D,C65C69D DD D