zj d Z ddlZddlZddlZddlZddlZddlZddlZddl Z ddl
m
Z
dZ dZ dZ
dZdZ ej e
d
d Zd Zd
Zd Zd Zd Zd Zd Zedk( r e yy)u
ZC 数据同步：WebDAV 下载 PDF → 解析 → 导入 ZC.db
去重方式：解析后查 project_name 是否已在库里，不再靠文件名 track file
N)datetimez,https://dav.jianguoyun.com/dav/Downloads_nucz'chniir0000@outlook.com:ah4urmcdv7s4hgpdz/mnt/data/zc_pdfsz /root/ZC.dbz/var/log/zc_sync.logT)exist_okc t j j d }t t d 5 }|j d| d| d d d d t
| y # 1 sw Y xY w)Nz%Y-%m-%d %H:%M:%Sa[z]
)r nowstrftimeopenLOGwriteprint)msgtsfs /root/gov_crawler/sync_zc.pylogr s[ !4 5B
c3 #1 !B4r#b!"# #J# #s A!!A*c t dz } t j j | dddi }t j
t j j }|j dd| t j j |d }|j j }g }t j d | D ]f }|j! d }|j# d j% d d
} t j& j) | } |j+ | h t- | S # t $ r}t d
| g cY d }~S d }~ww xY w)N/PROPFINDDepth1)methodheaders
AuthorizationBasic timeoutu WebDAV 列表失败: z ([^<]+\.pdf) ) WEBDAV_BASEurllibrequestRequestbase64 b64encodeAUTHencodedecode
add_headerurlopenread Exceptionr refinditergrouprstripsplitparseunquoteappendsorted)
urlreqcredsrespxml_dataepdfsmhreffnames
r webdav_listrB s1
C
..
Z'3
PC T[[] + 2 2 4ENN?fUG$45~~%%c2%699;%%' D
[[98
D wwqz C &&s+B/ $$U+ E
$< %aS )* s 9?E E'
E" E'"E'c t dz t j j | z }t j j | }t
j t j j }|j dd| t j j |d }|j }t j j! t" | }t% |d 5 }|j' | d d d |S # 1 sw Y |S xY w# t( $ r}t+ d| d| Y d }~y d }~ww xY w) Nr r r < r wbu 下载失败 : )r" r# r4 quoter$ r% r& r' r( r) r* r+ r, r- ospathjoinPDF_DIRr r
r. r ) filenamer8 r9 r: r; datarI r r= s r webdav_downloadrN / s
fll 0 0 :
:C
..
%C T[[] + 2 2 4ENN?fUG$45 ~~%%c2%6yy{ww||GX.
$
GGDM ohZr! -.s7 A!D 5D D DD D E'D==Ec j dd l }|j | }|j dkD r|d j nd}|j dkD r|d j nd}d}t d|j D ] }||| j dz z
} |j t j j dd dd l }|j j dd }|j j | } |j j | | j | }
|
j! | j# | |
j% d sDt& j j) | } t& j j+ | d |
d< | j- | } |
| fS ) Nr r r z/root/gov_crawlerparse_zcz /root/gov_crawler/parse_zc.py project_name)fitzr
page_countget_textrangeclosesysrI insertimportlib.utilutilspec_from_file_locationmodule_from_specloader exec_module
extract_basicupdate
extract_page2getrH basenamesplitextextract_contacts)
pdf_pathrS doctext_p1text_p2
text_contactspi importlibspecpzprA contactss
r parse_pdfrr @ sc
))H
C#&>>A#5c!foo 2G#&>>A#5c!foo 2GMAs~~& 3R))+d22
3IIKHHOOA*+ >> 1 1*>] ^D ( ( .BKKB
!AHHR
g
&' 55 * GG , ,U 3A 6.""=1H h; c D dt | j dd z dz S )N'z'')strreplace)ss r sqry W s! Q T* *S 00rs c |r$| j d|f j }|ry|r$| j d|f j }|ryy)u4 查 DB：project_id 或 project_name 是否已存在z,SELECT 1 FROM zc_projects WHERE project_id=?Tz.SELECT 1 FROM zc_projects WHERE project_name=?F)executefetchone)cpidnamers r check_existsr [ sO
IIDsf M V V X
IIF P Y Y [ rs c t j t }|j }| j dd }| j dd xs" t
j j | d }t ||| r|j yg d}t | }||d< t j |d | d < |D cg c] } t | j | d }
} d
d j | d d j |
d
} |j | |D ] } |j d|| j dd | j dd | j dd | j dd | j dd | j dd | j dd | j dd f |j! |j yc c} w )N
project_idrP rR r F)+r rR version_type publish_datephaseconstruction_periodtotal_investment project_type owner_natureindustrytopicscalequantity_scaleindustry_levelprovincecitydetail_address
building_area land_areafloorsforeign_investment
decorationsteel_structure
exterior_wallparkingelevatorair_conditioning fresh_airheatingprefab
passive_houseconstruction_contentequipment_listprocurement_equipment process_flowschedule_overviewphase_approval phase_designphase_procurementphase_constructionphase_contractor phase_statusraw_jsonrq ) ensure_asciir zINSERT INTO zc_projects (z, z
) VALUES ()zINSERT INTO zc_contacts (project_id, company, contact_name, department, position, phone, remarks, address, role) VALUES (?,?,?,?,?,?,?,?,?)company contact_name
departmentpositionphoneremarksaddressroleT)sqlite3connectDBcursorrc rH rI re r rW dictjsondumpsry rJ r{ commit)
rp rq rL connr} r~ r fieldsraw_datar valssqlcts
r insert_to_dbr h s ??2 D
A
%% b
!C 55 $ E(8(8(B1(EDAsD!
F AwH#HZJJxe new_count
skip_countrA
local_pathrg rp rq insertedr= counts r mainr sS "# =D.T 8,-IJ ,WW\\'51
77>>* %!H&u-H ,#H-KAx#Ax7HQ fU3BZL
c(m_AFGa
#,*
OOB ' '(J K T T VWX YE(9+Z
|3CE7$OP , &s Bqc* + + ,s AD.. E7EE__main__)__doc__rH rX r/ r urllib.parser# r& urllib.requestr r r" r( rK r r makedirsr rB rN rr ry r r r __name__ rs r r s
? > > > < 0
Gd #(".1
"J D zF rs