使用 boto 将目录上传到 s3

Question

我已经连接到实例，我想将我的 python 脚本生成的文件直接上传到 S3。我试过这个：

import boto
s3 = boto.connect_s3()
bucket = s3.get_bucket('alexandrabucket')
from boto.s3.key import Key
key = bucket.new_key('s0').set_contents_from_string('some content')

但这只是在我想将目录 s0 上传到 mybucket 时创建一个具有上下文“相同内容”的新文件 s0。

我也看过 s3put，但没有得到我想要的。

Answer 1

更新了@user 923227 对 (1) 的回答，包括更新的 boto3 界面 (2) 使用 Windows 双反斜杠的细微差别 (3) 更清洁的 tqdm 进度条：

import os
from tqdm import tqdm

def upload_folder_to_s3(s3_client, s3bucket, input_dir, s3_path):
    pbar = tqdm(os.walk(input_dir))
    for path, subdirs, files in pbar:
        for file in files:
            dest_path = path.replace(input_dir, "").replace(os.sep, '/')
            s3_file = f'{s3_path}/{dest_path}/{file}'.replace('//', '/')
            local_file = os.path.join(path, file)
            s3_client.upload_file(local_file, s3bucket, s3_file)
            pbar.set_description(f'Uploaded {local_file} to {s3_file}')
    print(f"Successfully uploaded {input_dir} to S3 {s3_path}")

使用示例：

s3_client = boto3.client('s3', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
upload_folder_to_s3(s3_client, 'BUCKET-NAME', <local-directory>, <s3-directory>)

Answer 2

这个解决方案不使用 boto，但我认为它可以做 OP 想要的。

它使用 awscli 和 Python。

import os

class AwsCredentials:
    def __init__(self, access_key: str, secret_key: str):
        self.access_key = access_key
        self.secret_key = secret_key

    def to_command(self):
        credentials = f'AWS_ACCESS_KEY_ID={self.access_key} AWS_SECRET_ACCESS_KEY={self.secret_key}'
        return credentials


def sync_s3_bucket(credentials: AwsCredentials, source_path: str, bucket: str) -> None:
    command = f'{credentials.to_command()} aws s3 sync {source_path} s3://{bucket}'
    result = os.system(command)
    assert result == 0, f'The s3 sync was not successful, error code: {result}'

请考虑从文件或环境中获取 AWS 凭证。

s3 sync 命令的文档是here。

Answer 3

s3fs package 提供了很好的功能来处理这种情况

s3_file = s3fs.S3FileSystem()
local_path = "some_dir_path/some_dir_path/"
s3_path = "bucket_name/dir_path"
s3_file.put(local_path, s3_path, recursive=True 

Answer 4

这是我使用的代码，它递归地将文件从指定文件夹上传到指定的 s3 路径。只需在脚本中添加 S3 凭据和存储桶详细信息：

https://gist.github.com/hari116/4ab5ebd885b63e699c4662cd8382c314/

#!/usr/bin/python
"""Usage: Add bucket name and credentials
          script.py <source folder> <s3 destination folder >"""

import os
from sys import argv
import boto3
from botocore.exceptions import NoCredentialsError

ACCESS_KEY = ''
SECRET_KEY = ''
host = ''
bucket_name = ''

local_folder, s3_folder = argv[1:3]
walks = os.walk(local_folder)
# Function to upload to s3
def upload_to_aws(bucket, local_file, s3_file):
    """local_file, s3_file can be paths"""
    s3 = boto3.client('s3', aws_access_key_id=ACCESS_KEY,
                      aws_secret_access_key=SECRET_KEY)
    print('  Uploading ' +local_file + ' as ' + bucket + '/' +s3_file)
    try:
        s3.upload_file(local_file, bucket, s3_file)
        print('  '+s3_file + ": Upload Successful")
        print('  ---------')
        return True
    except NoCredentialsError:
        print("Credentials not available")
        return False

"""For file names"""
for source, dirs, files in walks:
    print('Directory: ' + source)
    for filename in files:
        # construct the full local path
        local_file = os.path.join(source, filename)
        # construct the full Dropbox path
        relative_path = os.path.relpath(local_file, local_folder)
        s3_file = os.path.join(s3_folder, relative_path)
        # Invoke upload function
        upload_to_aws(bucket_name, local_file, s3_file)

Answer 5

第一次提出这个问题时不存在的另一种方法是使用 python-rclone (https://github.com/ddragosd/python-rclone/blob/master/README.md)。

这需要下载 rclone 和有效的 rclone 配置。常用于 AWS (https://rclone.org/s3/)，但也可用于其他提供商。

install('python-rclone')
import rclone
cfg_path = r'(path to rclone config file here)'

with open(cfg_path) as f:
   cfg = f.read()

# Implementation
# Local file to cloud server
result = rclone.with_config(cfg).run_cmd(command="sync", extra_args=["/home/demodir/", "AWS test:dummydir/etc/"])
# Cloud server to cloud server
result = rclone.with_config(cfg).run_cmd(command="sync", extra_args=["Gdrive:test/testing/", "AWS test:dummydir/etc/"

这允许您通过读取配置文件并通过 kwargs (extra_args) 映射您的输出，在您的 python 代码中运行类似于 AWS CLI 的“同步”命令

Answer 6

我根据@JDPTET 的反馈构建了这个函数，但是，

1. 我需要删除通用的整个本地路径，以免上传到存储桶！
2. 不确定我遇到了多少路径分隔符 - 所以我不得不使用os.path.normpath

    def upload_folder_to_s3(s3bucket, inputDir, s3Path):
        print("Uploading results to s3 initiated...")
        print("Local Source:",inputDir)
        os.system("ls -ltR " + inputDir)

        print("Dest  S3path:",s3Path)

        try:
            for path, subdirs, files in os.walk(inputDir):
                for file in files:
                    dest_path = path.replace(inputDir,"")
                    __s3file = os.path.normpath(s3Path + '/' + dest_path + '/' + file)
                    __local_file = os.path.join(path, file)
                    print("upload : ", __local_file, " to Target: ", __s3file, end="")
                    s3bucket.upload_file(__local_file, __s3file)
                    print(" ...Success")
        except Exception as e:
            print(" ... Failed!! Quitting Upload!!")
            print(e)
            raise e

    s3 = boto3.resource('s3', region_name='us-east-1')
    s3bucket = s3.Bucket("<<s3bucket_name>>")
    upload_folder_to_s3(s3bucket, "<<Local Folder>>", "<<s3 Path>>")

Answer 7

为了读取文件表单文件夹，我们可以使用

import boto
from boto.s3.key import Key

keyId = 'YOUR_AWS_ACCESS_KEY_ID'
sKeyId='YOUR_AWS_ACCESS_KEY_ID'
bucketName='your_bucket_name'

conn = boto.connect_s3(keyId,sKeyId)
bucket = conn.get_bucket(bucketName)
for key in bucket.list():
    print ">>>>>"+key.name
    pathV = key.name.split('/')
    if(pathV[0] == "data"):
        if(pathV[1] != ""):
            srcFileName = key.name
            filename = key.name
            filename = filename.split('/')[1]
            destFileName = "model/data/"+filename
            k = Key(bucket,srcFileName)
            k.get_contents_to_filename(destFileName)
    elif(pathV[0] == "nlu_data"):
        if(pathV[1] != ""):
            srcFileName = key.name
            filename = key.name
            filename = filename.split('/')[1]
            destFileName = "model/nlu_data/"+filename
            k = Key(bucket,srcFileName)
            k.get_contents_to_filename(destFileName)

Answer 8

您可以执行以下操作：

import os
import boto3

s3_resource = boto3.resource("s3", region_name="us-east-1")

def upload_objects():
    try:
        bucket_name = "S3_Bucket_Name" #s3 bucket name
        root_path = 'D:/sample/' # local folder for upload

        my_bucket = s3_resource.Bucket(bucket_name)

        for path, subdirs, files in os.walk(root_path):
            path = path.replace("\\","/")
            directory_name = path.replace(root_path,"")
            for file in files:
                my_bucket.upload_file(os.path.join(path, file), directory_name+'/'+file)

    except Exception as err:
        print(err)

if __name__ == '__main__':
    upload_objects()

Answer 9

以下函数可用于通过boto将目录上传到s3。

    def uploadDirectory(path,bucketname):
        for root,dirs,files in os.walk(path):
            for file in files:
                s3C.upload_file(os.path.join(root,file),bucketname,file)

提供目录路径和存储桶名称作为输入。文件直接放入存储桶中。更改 upload_file() 函数的最后一个变量，将它们放在“目录”中。

Answer 10

boto 库本身没有任何内容可以让您上传整个目录。您可以编写自己的代码以使用 os.walk 或类似的方式遍历目录，并使用 boto 上传每个单独的文件。

boto 中有一个名为s3put 的命令行实用程序可以处理此问题，或者您可以使用AWS CLI tool，它有很多功能可以让您上传整个目录，甚至可以将 S3 存储桶与本地目录同步反之亦然。