获取 s3 对象元数据然后创建流

Question

我正在从 s3 下载一个对象并从中创建一个读取流对象来处理视频：

s3.getObject(params).createReadStream()

但是，我需要从中获取元数据，当我通过访问它的“元数据”属性来获取对象时，这是可能的：

s3.getObject()

我会怎么做：

1. 通过s3.getObject()获取对象，从其元数据属性中获取元数据，然后将其变成读取流？

   var stream = fs.createReadStream(response); 不起作用 - 输入必须是字符串

-- 或者--

1. 通过s3.getObject().createReadStream()获取流，并从流中提取元数据？

   据我所知，元数据不会在流中传递。

---

如果我的假设是错误的，请告诉我，但我目前被这两个需求所困扰：

• 获取元数据
• 使其成为流

Answer 1

所以我找到了一个解决方案。这适用于 10 MB 以下的大多数文件。如果它们大于该缓冲区流在文件完成写入之前结束。我已经尝试将 bufferStream.end 放在 on finish 函数中，但是我的回调没有通过....

function download(s3Event, srcKey, cb){

  console.log('Starting download');

  s3.getObject({
      Bucket: s3Event.bucket.name,
      Key: srcKey
  }, cb);
}



function writeToFile(data, cb){

  var dlFile = path.join(tempDir, 'download');

  console.log('data = ', data);
  console.log('data.Body = ', data.Body);
  var stream = bufferStream.pipe(fs.createWriteStream(dlFile)).on('finish', function () {
     console.log('finished writing stream');
     cb(null, data);
  });

  bufferStream.end(data.Body);
}


exports.handler = function(event, context) {

// Read options from the event.

console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));

var s3Event = event.Records[0].s3;
var srcKey = decodeURIComponent(s3Event.object.key);
var keyPrefix = srcKey.replace(/\.[^/.]+$/, '');         
var dstBucket = "jump-lambda";


async.waterfall([

    function (cb){
        download(s3Event, srcKey, cb);
    },

    function (data, cb){
        writeToFile(data, cb);
    },

    function (data, cb){
        fluentffmpegProcess(data, cb);
    },

    function (data, cb){
        transform(data, cb);
    },

    function (data, buffer, cb){
        thumbnailUpload(data, buffer, dstBucket, keyPrefix, cb);
    },

    function (data, cb){
        updateParse(data, srcKey, keyPrefix, cb);
    },

],

    function (err) {
        if (err) {
            console.error(
                'Unable to convert video to scene object, with error code: ' + err.description
            );
        } else {
            console.log(
                'Successfully created scene object, updated venue, and created thumbnail'
            );
        }
    }
);
};

Answer 2

您可以通过请求的 httpHeaders 事件获取元数据。

let fs = require('fs')
let aws = require('aws-sdk')
let s3 = new aws.S3()

let request = s3.getObject({
    Bucket: 'my-bucket',
    Key: 'my-key'
})

let stream

request.on('httpHeaders', (statusCode, httpHeaders) => {
    // object metadata is represented by any header in httpHeaders starting with 'x-amz-meta-'
    // you can use the stream object that this point
    stream.pipe(fs.createWriteStream('./somepath'))
    stream.on('end', () => { 
        console.log('were done')
    })
})

stream = request.createReadStream()

您也可以调用s3.headObject 获取元数据而不下载对象，然后使用s3.getObject 下载对象