使用 Axios 进行 Amazon S3 远程文件上传

Question

我正在尝试编写一个函数：

1. 以远程 URL 作为参数，
2. 使用 axios 获取文件
3. 将流上传到 amazon s3
4. 最后，返回上传的网址

我找到了帮助 here on stackoverflow。到目前为止，我有这个：

/* 
 * Method to pipe the stream 
 */
const uploadFromStream = (file_name, content_type) => {
  const pass = new stream.PassThrough();

  const obj_key = generateObjKey(file_name);
  const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };

  s3.upload(params, function(err, data) {
    if(!err){
        return data.Location;
    } else {
        console.log(err, data);
    }
  });

  return pass;
}


/*
 * Method to upload remote file to s3
 */
const uploadRemoteFileToS3 = async (remoteAddr) => {
    axios({
        method: 'get',
        url: remoteAddr,
        responseType: 'stream'
    }).then( (response) => {
        if(response.status===200){
            const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
            const content_type = response.headers['content-type'];
            response.data.pipe(uploadFromStream(file_name, content_type));
        }
    });
}

但是uploadRemoteFileToS3 没有返回任何东西（因为它是一个异步函数）。如何获取上传的url？

更新

我进一步改进了代码并编写了一个类。这是我现在拥有的：

const config = require('../config.json');

const stream = require('stream');
const axios = require('axios');
const AWS = require('aws-sdk');

class S3RemoteUploader {
    constructor(remoteAddr){
        this.remoteAddr = remoteAddr;
        this.stream = stream;
        this.axios = axios;
        this.config = config;
        this.AWS = AWS;
        this.AWS.config.update({
            accessKeyId: this.config.api_key,
            secretAccessKey: this.config.api_secret
        });
        this.spacesEndpoint = new this.AWS.Endpoint(this.config.endpoint);
        this.s3 = new this.AWS.S3({endpoint: this.spacesEndpoint});

        this.file_name = this.remoteAddr.substring(this.remoteAddr.lastIndexOf('/')+1);
        this.obj_key = this.config.subfolder+'/'+this.file_name;
        this.content_type = 'application/octet-stream';

        this.uploadStream();
    }

    uploadStream(){
        const pass = new this.stream.PassThrough();
        this.promise = this.s3.upload({
            Bucket: this.config.bucket,
            Key: this.obj_key,
            ACL: this.config.acl,
            Body: pass,
            ContentType: this.content_type
        }).promise();
        return pass;
    }

    initiateAxiosCall() {
        axios({
            method: 'get',
            url: this.remoteAddr,
            responseType: 'stream'
        }).then( (response) => {
            if(response.status===200){
                this.content_type = response.headers['content-type'];
                response.data.pipe(this.uploadStream());
            }
        });
    }

    dispatch() {
        this.initiateAxiosCall();
    }

    async finish(){
        //console.log(this.promise); /* return Promise { Pending } */
        return this.promise.then( (r) => {
            console.log(r.Location);
            return r.Location;
        }).catch( (e)=>{
            console.log(e);
        });
    }

    run() {
        this.dispatch();
        this.finish();
    }
}

但仍然不知道如何在 promise 被解决时捕获结果。到目前为止，我尝试了这些：

testUpload = new S3RemoteUploader('https://avatars2.githubusercontent.com/u/41177');
testUpload.run();
//console.log(testUpload.promise); /* Returns Promise { Pending } */
testUpload.promise.then(r => console.log); // does nothing

但以上都不起作用。我有一种感觉，我错过了一些非常微妙的东西。有什么线索吗？

Answer 1

上传后，您可以调用 s3 sdk 中的 getsignedurl 函数来获取 url，您还可以在其中指定 url 的到期时间。您需要传递该功能的密钥。现在旅行将在稍后更新示例。

生成一个简单的预签名 URL，允许任何用户查看 您拥有的存储桶中的私有对象的内容，您可以使用 在调用 getSignedUrl() 之后：

 var s3 = new AWS.S3(); 
 var params = {Bucket: 'myBucket', Key: 'myKey'}; 
 s3.getSignedUrl('getObject', params, function (err, url) {  
   console.log("The URL is", url); 
 });

官方文档链接 http://docs.amazonaws.cn/en_us/AWSJavaScriptSDK/guide/node-examples.html

代码必须是这样的

function uploadFileToS3AndGenerateUrl(cb) {
const pass = new stream.PassThrough();//I have generated streams from file. Using this since this is what you have used. Must be a valid one.
var params = {
            Bucket: "your-bucket", // required
            Key: key , // required
            Body: pass,
            ContentType: 'your content type',

        };
s3.upload(params, function(s3Err, data) {
    if (s3Err) {
        cb(s3Err)
    }
    console.log(`File uploaded successfully at ${data.Location}`)

    const params = {
        Bucket: 'your-bucket',
        Key: data.key,
        Expires: 180
    };
    s3.getSignedUrl('getObject', params, (urlErr, urlData) => {
        if (urlErr) {

            console.log('There was an error getting your files: ' + urlErr);
            cb(urlErr);

        } else {
            console.log(`url: ${urlData}`);
            cb(null, urlData);

        }
    })
})
}

Answer 2

请检查我是否已更新您的代码可能会对您有所帮助。

    /*
         * Method to upload remote file to s3
         */
        const uploadRemoteFileToS3 = async (remoteAddr) => {
            const response = await axios({
                method: 'get',
                url: remoteAddr,
                responseType: 'stream'
            })
               if(response.status===200){
                    const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
                    const content_type = response.headers['content-type'];
                    response.data.pipe(uploadFromStream(file_name, content_type));
                }
                return new Promise((resolve, reject) => {
                    response.data.on('end', (response) => {
                      console.log(response)
                      resolve(response)
                    })

                    response.data.on('error', () => {
                      console.log(response);
                      reject(response)
                    })
              })
        };

       * 
     * Method to pipe the stream 
     */
    const uploadFromStream = (file_name, content_type) => {
       return new Promise((resolve, reject) => {
          const pass = new stream.PassThrough();
          const obj_key = generateObjKey(file_name);
          const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };
          s3.upload(params, function(err, data) {
            if(!err){
                console.log(data)
                return resolve(data.Location);
            } else {
                console.log(err)
                return reject(err);
            }
          });
       });
    }

//call uploadRemoteFileToS3
    uploadRemoteFileToS3(remoteAddr)
      .then((finalResponse) => {
            console.log(finalResponse)
       })
       .catch((err) => {
         console.log(err);
    });