如何在 Cloud Functions for Firebase（multer、busboy）上使用 express 执行 HTTP 文件上传

Question

我正在尝试将文件上传到 Cloud Functions，使用 Express 在那里处理请求，但我没有成功。我创建了一个在本地工作的版本：

服务器端 js

const express = require('express');
const cors = require('cors');
const fileUpload = require('express-fileupload');

const app = express();
app.use(fileUpload());
app.use(cors());

app.post('/upload', (req, res) => {
    res.send('files: ' + Object.keys(req.files).join(', '));
});

客户端 js

const formData = new FormData();
Array.from(this.$refs.fileSelect.files).forEach((file, index) => {
    formData.append('sample' + index, file, 'sample');
});

axios.post(
    url,
    formData, 
    {
        headers: { 'Content-Type': 'multipart/form-data' },
    }
);

此完全相同的代码在部署到未定义 req.files 的 Cloud Functions 时似乎会中断。有谁知道这里发生了什么？

编辑 我也尝试过使用multer，它在本地运行良好，但是一旦上传到 Cloud Functions，我得到了一个空数组（相同的客户端代码）：

const app = express();
const upload = multer();
app.use(cors());

app.post('/upload', upload.any(), (req, res) => {
    res.send(JSON.stringify(req.files));
});

Answer 1

我尝试过 Dougs 的回答，但从未触发完成，所以我稍微调整了代码并得到了适合我的：

// It's very crucial that the file name matches the name attribute in your html
app.post('/', (req, res) => {
  const busboy = new Busboy({ headers: req.headers })
  // This object will accumulate all the uploaded files, keyed by their name
  const uploads = {}

  // This callback will be invoked for each file uploaded
  busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
    console.log(`File [${fieldname}] filename: ${filename}, encoding: ${encoding}, mimetype: ${mimetype}`)
    // Note that os.tmpdir() is an in-memory file system, so should only
    // be used for files small enough to fit in memory.
    const filepath = path.join(os.tmpdir(), filename)
    uploads[fieldname] = { file: filepath }
    console.log(`Saving '${fieldname}' to ${filepath}`)
    const stream = fs.createWriteStream(filepath)
    stream.on('open', () => file.pipe(stream))
  })

  // This callback will be invoked after all uploaded files are saved.
  busboy.on('finish', () => {
    console.log('look im firing!')
// Do whatever you want here
    res.end()
  })

  // The raw bytes of the upload will be in req.rawBody.  Send it to busboy, and get
  // a callback when it's finished.
  busboy.end(req.rawBody)
})

Answer 2

如果您只想从请求中获取单个上传的文件，请使用busboy 将文件作为可读流获取：

const express = require('express')
const Busboy = require('busboy')

express().post('/', (req, res) => {
  const busboy = new Busboy({ headers: req.headers })

  busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
    // Do something with `file`, e.g. pipe it to an output stream.
    // file.pipe(fs.createWriteStream('upload.pdf')
  })

  // The original input was moved to `req.rawBody`
  busboy.write(req.rawBody)
})

Answer 3

我今天遇到了这个问题，查看here了解更多关于如何处理谷歌云上的文件（基本上你不需要multer）。

这是我用来提取文件的中间件。这会将您的所有文件保留在 request.files 上，并将其他表单字段保留在 request.body 上，用于所有具有 multipart/form-data 内容类型的 POST。它会让你的其他中间件处理其他所有事情。

// multiparts.js
const { createWriteStream } = require('fs')
const { tmpdir } = require('os')
const { join } = require('path')
const BusBoy = require('busboy')

exports.extractFiles = async(req, res, next) => {
  const multipart = req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')
  if (!multipart) return next()
  //
  const busboy = new BusBoy({ headers: req.headers })
  const incomingFields = {}
  const incomingFiles = {}
  const writes = []
  // Process fields
  busboy.on('field', (name, value) => {
    try {
      // This will keep a field created like so form.append('product', JSON.stringify(product)) intact
      incomingFields[name] = JSON.parse(value)
    } catch (e) {
      // Numbers will still be strings here (i.e 1 will be '1')
      incomingFields[name] = value
    }
  })
  // Process files
  busboy.on('file', (field, file, filename, encoding, contentType) => {
    // Doing this to not have to deal with duplicate file names
    // (i.e. TIMESTAMP-originalName. Hmm what are the odds that I'll still have dups?)
    const path = join(tmpdir(), `${(new Date()).toISOString()}-${filename}`)
    // NOTE: Multiple files could have same fieldname (which is y I'm using arrays here)
    incomingFiles[field] = incomingFiles[field] || []
    incomingFiles[field].push({ path, encoding, contentType })
    //
    const writeStream = createWriteStream(path)
    //
    writes.push(new Promise((resolve, reject) => {
      file.on('end', () => { writeStream.end() })
      writeStream.on('finish', resolve)
      writeStream.on('error', reject)
    }))
    //
    file.pipe(writeStream)
  })
  //
  busboy.on('finish', async () => {
    await Promise.all(writes)
    req.files = incomingFiles
    req.body = incomingFields
    next()
  })
  busboy.end(req.rawBody)
}

现在在您的函数中，确保这是您使用的第一个中间件。

// index.js
const { onRequest } = require('firebase-functions').https
const bodyParser = require('body-parser')
const express = require('express')
const cors = require('cors')
const app = express()
// First middleware I'm adding
const { extractFiles } = require('./multiparts')
app.use(extractFiles)
app.use(bodyParser.urlencoded({ extended: true }))
app.use(bodyParser.json())
app.use(cors({ origin: true }))

app.use((req) => console.log(req.originalUrl))

exports.MyFunction = onRequest(app);

Answer 4

云函数在进一步传递 request 对象之前对其进行预处理。因此，原来的 multer 中间件不起作用。此外，使用busboy 级别太低，您需要自己处理所有事情，这并不理想。相反，您可以使用 forked version of multer 中间件在云函数上处理 multipart/form-data。

这是你可以做的。

1. 安装前叉

npm install --save emadalam/multer#master

1. 使用startProcessing 配置自定义处理由云功能添加的req.rawBody。

const express = require('express')
const multer = require('multer')

const SIZE_LIMIT = 10 * 1024 * 1024 // 10MB
const app = express()

const multipartFormDataParser = multer({
  storage: multer.memoryStorage(),
  // increase size limit if needed
  limits: {fieldSize: SIZE_LIMIT},
  // support firebase cloud functions
  // the multipart form-data request object is pre-processed by the cloud functions
  // currently the `multer` library doesn't natively support this behaviour
  // as such, a custom fork is maintained to enable this by adding `startProcessing`
  // https://github.com/emadalam/multer
  startProcessing(req, busboy) {
    req.rawBody ? busboy.end(req.rawBody) : req.pipe(busboy)
  },
})

app.post('/some_route', multipartFormDataParser.any(), function (req, res, next) {
  // req.files is array of uploaded files
  // req.body will contain the text fields
})

Answer 5

我在使用 firebase 功能部署我的应用程序时遇到了同样的问题。我正在使用 multer 将图像上传到亚马逊 s3。我通过使用 Cristóvão 创建的上述 npm https://stackoverflow.com/a/48648805/5213790 解决了这个问题。

  const { mimetype, buffer, } = req.files[0]

  let s3bucket = new aws.S3({
     accessKeyId: functions.config().aws.access_key,
     secretAccessKey: functions.config().aws.secret_key,
  });

  const config = {
     Bucket: functions.config().aws.bucket_name,
     ContentType: mimetype,
     ACL: 'public-read',
     Key: Date.now().toString(),
     Body: buffer,    
   }

   s3bucket.upload(config, (err, data) => {
     if(err) console.log(err)

     req.file = data;
     next()
  })

请注意，这是针对单个文件图像上传的。 下一个中间件将具有从 s3 返回的对象

{ 
  ETag: '"cacd6d406f891e216f9946911a69aac5"',
  Location:'https://react-significant.s3.us-west1.amazonaws.com/posts/1567282665593',
  key: 'posts/1567282665593',
  Key: 'posts/1567282665593',
  Bucket: 'react-significant' 
}

在这种情况下，在将数据保存到数据库之前，您可能需要 Location url。

Answer 6

请注意，除了在服务器上使用 Busboy 并解析 rawReq 之外，您可能还需要将以下配置添加到您的 Axios 请求中：

{ headers: { 'content-type': `multipart/form-data; boundary=${formData._boundary}` }};

如果您只指定content-type 而不是边界，您会在服务器上收到Boundary not found 错误。相反，如果您完全删除标头，Busboy 将无法正确解析这些字段。 见：Firebase Cloud Functions and Busboy not parsing fields or files

Answer 7

感谢大家对这个帖子的帮助。我浪费了一整天的时间尝试所有可能的组合和所有这些不同的库......只是在用尽所有其他选项后才发现这一点。

结合上述一些解决方案，在此处创建一个支持 TypeScript 和中间件的脚本：

https://gist.github.com/jasonbyrne/8dcd15701f686a4703a72f13e3f800c0

Answer 8

感谢answers above，我为此构建了一个 npm 模块 (github)

它适用于谷歌云功能，只需安装它 (npm install --save express-multipart-file-parser) 并像这样使用它：

const fileMiddleware = require('express-multipart-file-parser')

...
app.use(fileMiddleware)
...

app.post('/file', (req, res) => {
  const {
    fieldname,
    filename,
    encoding,
    mimetype,
    buffer,
  } = req.files[0]
  ...
})

Answer 9

我能够将 Brian 和 Doug 的回答结合起来。这是我的中间件，它最终模仿了 multer 中的 req.files，因此不会对您的其余代码进行重大更改。

module.exports = (path, app) => {
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({ extended: true }))
app.use((req, res, next) => {
    if(req.rawBody === undefined && req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')){
        getRawBody(req, {
            length: req.headers['content-length'],
            limit: '10mb',
            encoding: contentType.parse(req).parameters.charset
        }, function(err, string){
            if (err) return next(err)
            req.rawBody = string
            next()
        })
    } else {
        next()
    }
})

app.use((req, res, next) => {
    if (req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')) {
        const busboy = new Busboy({ headers: req.headers })
        let fileBuffer = new Buffer('')
        req.files = {
            file: []
        }

        busboy.on('field', (fieldname, value) => {
            req.body[fieldname] = value
        })

        busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
            file.on('data', (data) => {
                fileBuffer = Buffer.concat([fileBuffer, data])
            })

            file.on('end', () => {
                const file_object = {
                    fieldname,
                    'originalname': filename,
                    encoding,
                    mimetype,
                    buffer: fileBuffer
                }

                req.files.file.push(file_object)
            })
        })

        busboy.on('finish', () => {
            next()
        })


        busboy.end(req.rawBody)
        req.pipe(busboy)
    } else {
        next()
    }
})}

Answer 10

我修复了一些错误 G. Rodriguez 的回复。我为 Busboy 添加了 'field' 和 'finish' 事件，并在 'finish' 事件中执行 next()。这对我来说是工作。如下：

    module.exports = (path, app) => {
    app.use(bodyParser.json())
    app.use(bodyParser.urlencoded({ extended: true }))
    app.use((req, res, next) => {
        if(req.rawBody === undefined && req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')){
            getRawBody(req, {
                length: req.headers['content-length'],
                limit: '10mb',
                encoding: contentType.parse(req).parameters.charset
            }, function(err, string){
                if (err) return next(err)
                req.rawBody = string
                next()
            })
        } else {
            next()
        }
    })

    app.use((req, res, next) => {
        if (req.method === 'POST' && req.headers['content-type'].startsWith('multipart/form-data')) {
            const busboy = new Busboy({ headers: req.headers })
            let fileBuffer = new Buffer('')
            req.files = {
                file: []
            }

            busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
                file.on('data', (data) => {
                    fileBuffer = Buffer.concat([fileBuffer, data])
                })

                file.on('end', () => {
                    const file_object = {
                        fieldname,
                        'originalname': filename,
                        encoding,
                        mimetype,
                        buffer: fileBuffer
                    }

                    req.files.file.push(file_object)
                })
            })

            busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
              console.log('Field [' + fieldname + ']: value: ' + inspect(val));
            });

            busboy.on('finish', function() {
              next()
            });

            busboy.end(req.rawBody)
            req.pipe(busboy);
        } else {
            next()
        }
    })}

Answer 11

要添加到官方 Cloud Function 团队的答案，您可以通过执行以下操作在本地模拟此行为（显然，将此中间件添加到高于他们发布的 busboy 代码的位置）

const getRawBody = require('raw-body');
const contentType = require('content-type');

app.use(function(req, res, next){
    if(req.rawBody === undefined && req.method === 'POST' && req.headers['content-type'] !== undefined && req.headers['content-type'].startsWith('multipart/form-data')){
        getRawBody(req, {
            length: req.headers['content-length'],
            limit: '10mb',
            encoding: contentType.parse(req).parameters.charset
        }, function(err, string){
            if (err) return next(err);
            req.rawBody = string;
            next();
        });
    }
    else{
        next();
    }
});

Answer 12

触发此问题的 Cloud Functions 设置确实存在重大变化。它与应用于所有用于提供 HTTPS 功能的 Express 应用程序（包括默认应用程序）的中间件的工作方式有关。基本上，Cloud Functions 将解析请求的主体并决定如何处理它，将主体的原始内容留在req.rawBody 的缓冲区中。您可以使用它直接解析您的多部分内容，但您不能使用中间件（如 multer）来解析。

相反，您可以使用名为busboy 的模块直接处理原始正文内容。它可以接受rawBody 缓冲区并用它找到的文件给你回电。这是一些示例代码，它将迭代所有上传的内容，将它们保存为文件，然后删除它们。你显然想做一些更有用的事情。

const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');

exports.upload = functions.https.onRequest((req, res) => {
    if (req.method === 'POST') {
        const busboy = new Busboy({ headers: req.headers });
        // This object will accumulate all the uploaded files, keyed by their name
        const uploads = {}

        // This callback will be invoked for each file uploaded
        busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
            console.log(`File [${fieldname}] filename: ${filename}, encoding: ${encoding}, mimetype: ${mimetype}`);
            // Note that os.tmpdir() is an in-memory file system, so should only 
            // be used for files small enough to fit in memory.
            const filepath = path.join(os.tmpdir(), fieldname);
            uploads[fieldname] = { file: filepath }
            console.log(`Saving '${fieldname}' to ${filepath}`);
            file.pipe(fs.createWriteStream(filepath));
        });

        // This callback will be invoked after all uploaded files are saved.
        busboy.on('finish', () => {
            for (const name in uploads) {
                const upload = uploads[name];
                const file = upload.file;
                res.write(`${file}\n`);
                fs.unlinkSync(file);
            }
            res.end();
        });

        // The raw bytes of the upload will be in req.rawBody.  Send it to busboy, and get
        // a callback when it's finished.
        busboy.end(req.rawBody);
    } else {
        // Client error - only support POST
        res.status(405).end();
    }
})

请记住，保存到临时空间的文件会占用内存，因此它们的总大小应限制在 10MB 以内。对于较大的文件，您应该将它们上传到 Cloud Storage 并使用存储触发器对其进行处理。

还请记住，Cloud Functions 添加的中间件的默认选择当前未通过firebase serve 添加到本地模拟器。因此，在这种情况下，此示例将不起作用（rawBody 将不可用）。

团队正在努力更新文档，以便更清楚地了解与标准 Express 应用不同的 HTTPS 请求期间发生的一切。

Answer 13

几天来我一直在遭受同样的问题，事实证明，firebase 团队已将 multipart/form-data 的原始主体与他们的中间件一起放入 req.body 中。如果您在使用 multer 处理您的请求之前尝试 console.log(req.body.toString())，您将看到您的数据。由于 multer 创建了一个新的 req.body 对象，该对象覆盖了生成的 req，因此数据消失了，我们只能得到一个空的 req.body。希望 Firebase 团队能尽快纠正这个问题。