如何通过 Lambda 重命名 S3 中的多个输出文件

Question

如何重命名 S3 存储桶中的多个输出文件。作为一个例子，我正在使用

os.path.basename(keyprefix),

'w' 写入模式文件

abc_00000.csv.gz
abc_00001.csv.gz 

我想用命名约定重命名上述文件：

 abc_{today date in YYYYMMDD format}_00.csv.gz 
 abc_{today date in YYYYMMDD format}_01.csv.gz

以下是参考代码：

    import boto3
    import os
    from smart_open import open
    import gzip
    import csv
    import io
    def lambda_handler(event, context):
    dirpath = 'output/'
    bucket = 'export'
    key = 'export/_SUCCESS'

    if '_SUCCESS' in key:
    client = boto3.client('s3')
    response = client.list_objects_v2(Bucket=bucket,Prefix=os.path.dirname(key))
    for i in response['Contents']:
        keyprefix = i['Key']
        if 'part-' in keyprefix:
            with gzip.GzipFile(fileobj=client.get_object(Bucket=bucket,Key=keyprefix)['Body']) as gzipfile, open('s3://'+bucket + '/' + dirpath + os.path.basename(keyprefix),'w') as fout :
    writer = csv.writer(fout , delimiter=',')
                writer.writerow(['test1','test1','test3','test4','test5','test6','test7'])        
                for row in csv.reader(gzipfile.read().decode('utf-8').splitlines(), delimiter=',', quotechar='"'):
                    if row[5] == 'CDE':         
                        writer.writerow(row)

Answer 1

S3 Service Resource 提供更多灵活性。

复制文件后，我正在删除旧文件，如果您想保留，可以删除删除调用。

#!/usr/bin/env python3

import boto3
import os
from datetime import datetime
prefix = 'abc_'
new_prefix = f"{prefix}{datetime.today().strftime('%Y-%m-%d')}"
suffix = 'csv.gz'
bucket_name = 'mybucketname'

def lambda_handler(event, context):
    s3 = boto3.resource('s3')
    bucket = s3.Bucket(bucket_name)
    for obj in bucket.objects.all():
        key = obj.key
        path_part = os.path.dirname(key)
        filename = os.path.basename(key)
        copy_source = {
        'Bucket': bucket_name,
        'Key': key
        }
        if filename.startswith(prefix) and filename.endswith(suffix):
            new_key = f"{new_prefix}_{key.split('_')[1]}"
            full_key_with_path = os.path.join(path_part, new_key)
            destination_bucket = s3.Bucket(bucket_name)
            print(f'copying the object with new key : {full_key_with_path}')
            destination_bucket.copy(copy_source, full_key_with_path)
            print(f'deleting old key : {key}')
            s3.Object(bucket_name, key).delete()

❯❯ python3 s3rename.py 
copying the object with new key : myinventorylist/2021-02-07T00-00Z/abc_2021-03-07_00000.csv.gz
deleting old key : myinventorylist/2021-02-07T00-00Z/abc_00000.csv.gz
copying the object with new key : myinventorylist/2021-02-07T00-00Z/abc_2021-03-07_00001.csv.gz
deleting old key : myinventorylist/2021-02-07T00-00Z/abc_00001.csv.gz