将文件从 GCP 存储推送到另一个没有 sdk 的存储桶

Question

我在层次结构中的 gcp 云存储中有文件

data/staging1 
  --/temp1.json
  --/temp2.json
  --/temp3.json
data/staging2 
  --/temp1.json
  --/temp2.json
  --/temp3.json

想要读取 staging1 中的所有文件并移动到另一个文件夹 staging2

 def getFiles(self):
        filename = list(self.bucketname.list_blobs(prefix=self.prefix))

        target_blob = filename[0]
        print(target_blob)
        filename = list(self.bucketname.list_blobs(prefix=self.prefix))
        for name in filename:
            local_tmp_path =name.name
            local_tmp_path=self.bucketname.blob(local_tmp_path)
            with open(local_tmp_path, 'r') as f:
                    target_blob.upload_from_file(f)
                    print("done")

我想在循环中读取所有文件....当我尝试打开它时显示错误

    with open(local_tmp_path, 'r') as f:
TypeError: expected str, bytes or os.PathLike object, not Blob

 for name in filename:
            local_tmp_path =name.name
            local_tmp_path=self.bucketname.blob(local_tmp_path)
            with open(local_tmp_path, 'r') as f:
                    target_blob.upload_from_file(f)

Answer 1

根据GCP Official documentation，您可以以编程方式将文件从一个存储桶移动/传输到另一个存储桶。即使桶在另一个项目中，过程也是一样的。

"""Command-line sample that creates a daily transfer from a standard
GCS bucket to a Nearline GCS bucket for objects untouched for 30 days.

This sample is used on this page:

    https://cloud.google.com/storage/transfer/create-transfer

For more information, see README.md.
"""

import argparse
import datetime
import json

import googleapiclient.discovery


def main(description, project_id, start_date, start_time, source_bucket,
         sink_bucket):
    """Create a daily transfer from Standard to Nearline Storage class."""
    storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1')

    # Edit this template with desired parameters.
    transfer_job = {
        'description': description,
        'status': 'ENABLED',
        'projectId': project_id,
        'schedule': {
            'scheduleStartDate': {
                'day': start_date.day,
                'month': start_date.month,
                'year': start_date.year
            },
            'startTimeOfDay': {
                'hours': start_time.hour,
                'minutes': start_time.minute,
                'seconds': start_time.second
            }
        },
        'transferSpec': {
            'gcsDataSource': {
                'bucketName': source_bucket
            },
            'gcsDataSink': {
                'bucketName': sink_bucket
            },
            'objectConditions': {
                'minTimeElapsedSinceLastModification': '2592000s'  # 30 days
            },
            'transferOptions': {
                'deleteObjectsFromSourceAfterTransfer': 'true'
            }
        }
    }

    result = storagetransfer.transferJobs().create(body=transfer_job).execute()
    print('Returned transferJob: {}'.format(
        json.dumps(result, indent=4)))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('description', help='Transfer description.')
    parser.add_argument('project_id', help='Your Google Cloud project ID.')
    parser.add_argument('start_date', help='Date YYYY/MM/DD.')
    parser.add_argument('start_time', help='UTC Time (24hr) HH:MM:SS.')
    parser.add_argument('source_bucket', help='Standard GCS bucket name.')
    parser.add_argument('sink_bucket', help='Nearline GCS bucket name.')

    args = parser.parse_args()
    start_date = datetime.datetime.strptime(args.start_date, '%Y/%m/%d')
    start_time = datetime.datetime.strptime(args.start_time, '%H:%M:%S')

    main(
        args.description,
        args.project_id,
        start_date,
        start_time,
        args.source_bucket,
        args.sink_bucket)