一个很naive的代码，用来做ftp的“主->从 下载，从->主 上传”。ftp可不像mysql那样还有log可以用，所以完全naive的做法：连到ftp server然后递归列出所有目录，和维护的文件列表对比，有新增文件就下载到从服务器（其实是本地PC，囧）；然后从服务器也列出目录下文件列表，和维护的文件列表比较，有新增就上传。维护一个文件列表，作用是：当主ftp server删除文件的时候，从服务器不会上传删掉的文件。
使用python自带的ftplib，感觉功能应该挺弱的，不过也没怎么发现更强大的ftp的python库。中文处理还是比较拙计，windows上的目录和文件名字都要用name.decode(\'utf-8\').encode(\'gbk\')这样丑陋的方式处理才能正确使用，一个小问题是，如果从服务器新增文件名含有中文，上传到主服务器后文件名乱码，用的时候注意。
# coding: utf-8
# author: ChrisZZ, zchrissirhcz@gmail.com
# description: 做ftp同步
# 有本地主机A，和ftp服务器C，需要从A上传图片到C服务器
# 同步策略：增加中间变量B，维护一个文件列表。C上出现过的文件都存，C上删除的则不删除；A上新增的也存
# 形式化表示为：
# B = {}
# while (true) {
# B += C - B
# A += B - A
# B += A - B
# }
# 需要做的几个模块
# 1. 定义文件列表数据结构，用于A,B,C的表示和加法、减法
# 2. ftp连接
# 3. ftp获取服务器上文件目录结构
# 4. ftp复制文件
# 5. ftp缓冲区大小，重传机制
# 编码处理
# qq=\'喜欢你\'
# charset_detect = chardet.detect(qq)
# cur_charset = charset_detect[\'encoding\']
# print charset_detect
# #qq.decode(cur_charset)
# qq=qq.decode(\'utf-8\')
# qq=qq.encode(\'gbk\')
# print qq
# step1 ftp连接
from ftplib import FTP
import sys
import os
import string
import time
# import chardet
#reload(sys)
#sys_encode = sys.getfilesystemencoding()
#print \'系统默认的编码为\', sys_encode
# print mystr.decode(\'utf-8\').encode(type)
#reload(sys)
#sys.setdefaultencoding(\'utf8\')
class Myfile(object):
def __init__(self, name, size, mtime):
self.name = name # 文件名字
self.mtime = mtime # 文件创建时间
self.is_dir = False # 是否为文件夹，默认为不是文件夹
#self.size = float(size) / (1024 * 1024) # 文件大小
size = float(size)
if size > 1024*1024:
self.size = str(\'%.2f\'%(size / (1024*1024))) + \'MB\'
elif size > 1024:
self.size = str(\'%.2f\'%(size / 1024)) + \'KB\'
else:
self.size = str(size) + \'Bytes\'
@property
def is_file(self):
return not self.is_dir
@property
def dir_property(self):
if self.is_dir==True:
return \'dir\'
return \'file\'
def show(self):
print \'[%s], [%s], [%s], [%s]\' % (self.name, self.size, self.mtime, self.dir_property)
@property
def pack(self):
"""
将myfile对象封装为一个字符串
:return:
"""
#return \'[%s][%s][%s]\'%(self.name, self.size, self.mtime)
return \'[%s][%s]\'%(self.name, self.size)
def ftpconnect():
ftp_server = \'host_ip\'
username = \'name\'
password = \'password\'
ftp=FTP(ftp_server)
ftp.login(username, password)
#ftp.set_debuglevel(2) #打开调试级别2，显示详细信息
#ftp.connect(ftp_server,21) #连接
#ftp.login(username,password) #登录，如果匿名登录则用空串代替即可
return ftp
def str_codec_std(mystr):
"""
将字符串转换编码，使得本地打印输出是正常的！
具体讲，就是从ascii编码转化为utf-8编码
:param mystr:需要转化编码的字符串
:return:转化好编码的字符串
"""
return mystr.decode(\'gbk\').encode(\'utf8\')
def filter_dir_list(mystr_list):
"""
过滤目录列表，包括转换编码，以及去除\'.\'和\'..\'目录，然后创建Myfile对象
:param mystr_list:
:return:
"""
res = []
for mystr in mystr_list:
mystr = str_codec_std(mystr)
# print "mystr is :%s" % mystr
file_info = string.split(mystr, maxsplit=8) # 一共有9列，第9列是文件名。文件名可能有空格，所以要指定maxsplit. 从0开始计数，name是第8列
# 形如[\'-rw-r--r--\', \'1\', \'48\', \'apache\', \'1018596\', \'Dec\', \'1\', \'20:08\', \'RPN_BF.pdf\']
name = file_info[8]
# print \'name = \', name
if name == \'.\' or name == \'..\':
continue
size = file_info[4]
mtime = \'%s-%s-%s\' % (file_info[5], file_info[6], file_info[7])
myfile = Myfile(name=name, size=size, mtime=mtime)
dir_info = file_info[0]
if dir_info[0] == \'d\':
myfile.is_dir = True
res.append(myfile)
return res
def list_local_dir(local_dir):
"""
列出本地目录下的内容，并将local_dir换成root路径\'/\'，并转为unix风格路径
:param local_dir:本地路径，比如\'g:/code/ftp_sync\'
:return:文件列表，是local_dir路径下递归找出的文件
"""
#local_dir = \'g:/code/ftp_sync\'
g=os.walk(local_dir)
res = []
for path,d,filelist in g:
path = path.replace("\\", "/")
if path[-1]!=\'/\':
path = path + \'/\'
prefix = path[len(local_dir):]
for filename in filelist:
filename = filename
name = os.path.join(\'/\', prefix, filename)
size = os.path.getsize(os.path.join(path, filename))
size = float(size)
if size > 1024 * 1024:
size = str(\'%.2f\' % (size / (1024 * 1024))) + \'MB\'
elif size > 1024:
size = str(\'%.2f\' % (size / 1024)) + \'KB\'
else:
size = str(size) + \'Bytes\'
mtime = os.path.getmtime(os.path.join(path, filename))
mtime = time.strftime(\'%b-%d-%H:%M\', time.localtime(mtime))
#item = \'/\'+prefix+\'/\'+filename
record = \'[%s][%s]\' % (name, size)
record = record.decode(\'gbk\').encode(\'utf-8\')
res.append(record)
return res
def list_server_dir(server_dir=None):
"""
获取ftp服务器上，指定目录下递归列出的文件列表
:param server_dir: ftp服务器上的目录，比如\'/photo\'
:return:文件列表，是server_dir路径下递归找出的文件
"""
#sys.setdefaultencoding(\'utf8\')
#sys_encode = sys.getfilesystemencoding()
# 先连接服务器
ftp = ftpconnect()
# 列出指定目录下所有文件
server_file_list = []
fuck_callback = lambda x: (server_file_list.append(x))
ftp.retrlines(\'LIST\', fuck_callback)
# 生成C
server_file_items = filter_dir_list(server_file_list)
#for server_file in server_file_items:
# server_file.show()
#for server_file in server_file_list:
# #print server_file.decode(\'gbk\').encode(\'utf-8\')
# print string.split(server_file,maxsplit=8)
# 关闭连接
ftp.quit()
def get_C(ftp, target_dir=None):
C = []
if target_dir is not None:
ftp.cwd(target_dir) # change working directory to target_dir
server_file_list = []
fuck_callback = lambda x: (server_file_list.append(x))
ftp.retrlines(\'LIST\', fuck_callback)
server_file_items = filter_dir_list(server_file_list)
for item in server_file_items:
if item.is_dir:
sub_C = get_C(ftp, item.name)
# sub_C = [\'/\'+item.name+\'/\'+cc.name for cc in sub_C]
for cc in sub_C:
cc.name = \'/\'+item.name+cc.name
C.extend(sub_C)
else:
item.name = \'/\' + item.name
C.append(item)
return C
def old_main():
#if __name__ == \'__main__\':
#dir_content = list_local_dir(\'g:/code/ftp_sync\')
#for dir_line in dir_content:
# print dir_line
# list_server_dir()
# 生成A
local_sync_dir = \'g:/code/ftp_sync/sycn\'
A = list_local_dir(local_sync_dir)
# 生成C TODO:连接失败的处理
ftp = ftpconnect()
server_file_list = []
fuck_callback = lambda x: (server_file_list.append(x))
ftp.retrlines(\'LIST\', fuck_callback)
server_file_items = filter_dir_list(server_file_list)
# C = [item.pack for item in server_file_items if item.is_file]
C = get_C(ftp)
C = [cc.name for cc in C]
for cc in C:
print cc
def list_diff(a, b):
"""
差集操作A-B
:param a:
:param b:
:return:
"""
ret = []
for i in a:
if i not in b:
ret.append(i)
return ret
#if __name__ == \'__main__\':
def future_main():
# 设定B
B = []
# 设定C
ftp = ftpconnect()
C = get_C(ftp)
#ftp.quit()
C = [cc.pack for cc in C]
# 设定A
local_sync_dir = \'g:/code/ftp_sync/sync\'
A = list_local_dir(local_sync_dir)
print \'====== Begin本地文件列表(A)\'
for aa in A:
print aa
print \'====== End本地文件列表(A)\'
# B += C - B
delta = list_diff(C, B)
B.extend(delta)
print \'====== Begin维护文件列表(B)\'
for bb in B:
print bb
print \'====== End维护文件列表(B)\'
#print \'====before====\'
#for aa in A:
# print aa
#print len(A)
# A += B - A
delta = list_diff(B, A)
print \'====== Begin服务器上新增文件列表(B-A):\'
for d in delta:
print d
# TODO: 把delta里的文件都下载下来
bufsize = 20000 * 1024
for filename in delta:
filename = filename.split(\']\')[0][1:]
filename = filename.decode(\'utf-8\').encode(\'gbk\')
#print \'filename is : \', filename
LocalFile = local_sync_dir + filename
print \'将服务器上的文件%s保存为本地文件%s\' % (filename.decode(\'gbk\').encode(\'utf-8\'), LocalFile.decode(\'gbk\').encode(\'utf-8\'))
server_dir = filename[0:filename.rindex(\'/\')+1]
#print \'server_dir:\' , server_dir
ftp.cwd(server_dir)
short_filename = filename[filename.rindex(\'/\')+1:]
#print \'short_filename is \', short_filename
#LocalFile = string.replace(LocalFile, \'/\', \'\\\')
#print \'item is:\', LocalFile
#print \'local_sync_dir is \', local_sync_dir
#local_sync_dir = string.replace(local_sync_dir, \'/\', \'\\\')
local_dir = LocalFile[0:LocalFile.rindex(\'/\')+1]
print \'local_dir is \', local_dir
if(not os.path.exists(local_dir)):
print \'创建目录%s\' % local_dir
os.makedirs(local_dir)
#LocalFile = LocalFile.decode(\'utf-8\').encode(\'gbk\')
fh = open(LocalFile, \'wb\')
ftp.retrbinary(\'RETR \'+short_filename, fh.write, bufsize)
#ftp.storbinary(\'STOR %s\' % os.path.basename(short_filename), fh.write, bufsize)
#ftp.storbinary(\'STOR %s\'%short_filename, fh, bufsize)
fh.close()
A.extend(delta)
print "==== ^_^Begin下载完毕，现在本地文件列表如下"
for aa in A:
print aa
print "==== ^_^End下载完毕，现在本地文件列表如下"
# B += A - B
delta = list_diff(A, B)
# TODO: 把delta里面的文件都上传上去
for filename in delta:
filename = filename.split(\']\')[0][1:]
#filename = filename.decode(\'utf-8\').encode(\'gbk\')
LocalFile = local_sync_dir + filename
#LocalFile = LocalFile.decode(\'utf-8\').encode(\'gbk\')
print \'将本地文件%s上传到为服务器文件%s\' % (LocalFile.decode(\'gbk\').encode(\'utf-8\'), filename.decode(\'gbk\').encode(\'utf-8\'))
server_dir = filename[0:filename.rindex(\'/\') + 1]
ftp.cwd(server_dir)
# todo:检查服务器上路径server_dir是否存在，若不存在则创建
short_filename = filename[filename.rindex(\'/\') + 1:]
#local_dir = LocalFile[0:LocalFile.rindex(\'/\') + 1]
#print \'local_dir is \', local_dir
#if (not os.path.exists(local_dir)):
# print \'创建目录%s\' % local_dir
# os.makedirs(local_dir)
LocalFile = LocalFile.decode(\'utf-8\').encode(\'gbk\')
fh = open(LocalFile, \'rb\')
ftp.storbinary(\'STOR \' + LocalFile, fh, bufsize)
fh.close()
B.extend(delta)
print "==== !!任务结束!!"
ftp.quit()
def download_example():
ftp = ftpconnect()
local_sync_dir = \'g:/code/ftp_sync/sync\'
#filename = \'test.php\'
filename = \'反卷积iccv2011.pdf\'
filename = filename.decode(\'utf-8\').encode(\'gbk\')
LocalFile = local_sync_dir + \'/\' + filename
#ftp.set_pasv(0)
fh = open(LocalFile, \'wb\')
bufsize = 1000*1024
ftp.retrbinary(\'RETR %s\'%filename, fh.write, bufsize)
fh.close()
ftp.close()
def upload_example():
ftp = ftpconnect()
local_sync_dir = \'g:/code/ftp_sync/sync\'
filename = \'test你.php\'
import chardet
#print chardet.detect(filename)
#filename = filename.decode(\'gbk\').encode(\'utf-8\')
LocalFile = local_sync_dir + \'/\' + filename
print \'LocalFile is : %s\' % LocalFile
LocalFile = LocalFile.decode(\'utf-8\').encode(\'gbk\')
# ftp.set_pasv(0)
fh = open(LocalFile, \'rb\')
bufsize = 1000 * 1024
ftp.storbinary(\'STOR %s\' % filename, fh, bufsize)
fh.close()
ftp.close()
if __name__ == \'__main__\':
#download_example()
#upload_example()
future_main()