如何在 go 中将文件作为流上传？

Question

有很多关于在 go 中使用 http.Request 发布文件的教程，但几乎都是这样开始的：

file, err := os.Open(path)
if err != nil {
    return nil, err
}
fileContents, err := ioutil.ReadAll(file)

也就是说，您将整个文件读入内存，然后将其转换为Buffer 并将其传递给请求，如下所示：

func send(client *http.Client, file *os.File, endpoint string) {
    body := &bytes.Buffer{}
    io.Copy(body, file)
    req, _ := http.NewRequest("POST", endpoint, body)
    resp, _ := client.Do(req)
}

如果您想发布一个庞大的文件并避免将其读入内存，而是将文件分块存储...您会怎么做？

Answer 1

如果需要设置Content-Length，可以手动设置。下面的sn -p是上传文件和额外参数作为流的例子（代码基于Buffer-less Multipart POST in Golang）

//NOTE: for simplicity, error check is omitted
func uploadLargeFile(uri, filePath string, chunkSize int, params map[string]string) {
    //open file and retrieve info
    file, _ := os.Open(filePath)
    fi, _ := file.Stat()
    defer file.Close()    

    //buffer for storing multipart data
    byteBuf := &bytes.Buffer{}

    //part: parameters
    mpWriter := multipart.NewWriter(byteBuf)
    for key, value := range params {
        _ = mpWriter.WriteField(key, value)
    }

    //part: file
    mpWriter.CreateFormFile("file", fi.Name())
    contentType := mpWriter.FormDataContentType()

    nmulti := byteBuf.Len()
    multi := make([]byte, nmulti)
    _, _ = byteBuf.Read(multi)    

    //part: latest boundary
    //when multipart closed, latest boundary is added
    mpWriter.Close()
    nboundary := byteBuf.Len()
    lastBoundary := make([]byte, nboundary)
    _, _ = byteBuf.Read(lastBoundary)

    //calculate content length
    totalSize := int64(nmulti) + fi.Size() + int64(nboundary)
    log.Printf("Content length = %v byte(s)\n", totalSize)

    //use pipe to pass request
    rd, wr := io.Pipe()
    defer rd.Close()

    go func() {
        defer wr.Close()

        //write multipart
        _, _ = wr.Write(multi)

        //write file
        buf := make([]byte, chunkSize)
        for {
            n, err := file.Read(buf)
            if err != nil {
                break
            }
            _, _ = wr.Write(buf[:n])
        }        
        //write boundary
        _, _ = wr.Write(lastBoundary)        
    }()

    //construct request with rd
    req, _ := http.NewRequest("POST", uri, rd)
    req.Header.Set("Content-Type", contentType)
    req.ContentLength = totalSize

    //process request
    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        log.Fatal(err)
    } else {
        log.Println(resp.StatusCode)
        log.Println(resp.Header)

        body := &bytes.Buffer{}
        _, _ = body.ReadFrom(resp.Body)
        resp.Body.Close()
        log.Println(body)
    }
}

Answer 2

事实证明，您实际上可以将*File（或任何类似流的）对象直接传递给NewRequest。

但请注意，NewRequest（如下所示：https://golang.org/src/net/http/request.go?s=21674:21746#L695）实际上不会设置 ContentLength，除非流明确是以下之一：

• *bytes.Buffer
• *bytes.Reader
• *strings.Reader

由于*File不是其中之一，除非您手动设置，否则请求将在没有内容长度的情况下发送，这可能会导致某些服务器丢弃传入请求的正文，从而导致正文为@987654326 @ 在服务器上似乎已从 go 端正确发送。

Answer 3

如果请求必须有Content-Length 标头（大多数文件主机拒绝没有它的上传请求），并且您想将文件作为流上传（不将所有内容加载到内存中），标准库不会帮助您，你必须自己计算。

这是一个最小的工作示例（没有错误检查），它使用io.MultiReader 将os.File 与其他字段连接起来，同时保留请求大小的选项卡。

它支持常规字段（带有字符串内容）和文件字段，并计算总请求正文大小。只需添加一个新的case 分支，就可以轻松地使用其他值类型对其进行扩展。

import (
    "crypto/rand"
    "fmt"
    "io"
    "io/fs"
    "mime"
    "path/filepath"
    "strings"
)

type multipartPayload struct {
    headers map[string]string
    body    io.Reader
    size    int64
}

func randomBoundary() string {
    var buf [8]byte
    _, err := io.ReadFull(rand.Reader, buf[:])
    if err != nil {
        panic(err)
    }
    return fmt.Sprintf("%x", buf[:])
}

// Multipart request has the following structure:
//  POST /upload HTTP/1.1
//  Other-Headers: ...
//  Content-Type: multipart/form-data; boundary=$boundary
//  \r\n
//  --$boundary\r\n    ? request body starts here 
//  Content-Disposition: form-data; name="field1"\r\n
//  Content-Type: text/plain; charset=utf-8\r\n
//  Content-Length: 4\r\n
//  \r\n
//  $content\r\n
//  --$boundary\r\n
//  Content-Disposition: form-data; name="field2"\r\n
//  ...
//  --$boundary--\r\n
func prepareMultipartPayload(fields map[string]interface{}) (*multipartPayload, error) {
    boundary := randomBoundary()
    headers := make(map[string]string)
    totalSize := 0
    headers["Content-Type"] = fmt.Sprintf("multipart/form-data; boundary=%s", boundary)

    parts := make([]io.Reader, 0)
    CRLF := "\r\n"

    fieldBoundary := "--" + boundary + CRLF

    for k, v := range fields {
        parts = append(parts, strings.NewReader(fieldBoundary))
        totalSize += len(fieldBoundary)
        if v == nil {
            continue
        }
        switch v.(type) {
        case string:
            header := fmt.Sprintf(`Content-Disposition: form-data; name="%s"`, k)
            parts = append(
                parts,
                strings.NewReader(header+CRLF+CRLF),
                strings.NewReader(v.(string)),
                strings.NewReader(CRLF),
            )
            totalSize += len(header) + 2*len(CRLF) + len(v.(string)) + len(CRLF)
            continue
        case fs.File:
            stat, _ := v.(fs.File).Stat()
            contentType := mime.TypeByExtension(filepath.Ext(stat.Name()))
            header := strings.Join([]string{
                fmt.Sprintf(`Content-Disposition: form-data; name="%s"; filename="%s"`, k, stat.Name()),
                fmt.Sprintf(`Content-Type: %s`, contentType),
                fmt.Sprintf(`Content-Length: %d`, stat.Size()),
            }, CRLF)
            parts = append(
                parts,
                strings.NewReader(header+CRLF+CRLF),
                v.(fs.File),
                strings.NewReader(CRLF),
            )
            totalSize += len(header) + 2*len(CRLF) + int(stat.Size()) + len(CRLF)
            continue
        }
    }
    finishBoundary := "--" + boundary + "--" + CRLF
    parts = append(parts, strings.NewReader(finishBoundary))
    totalSize += len(finishBoundary)

    headers["Content-Length"] = fmt.Sprintf("%d", totalSize)

    return &multipartPayload{headers, io.MultiReader(parts...), int64(totalSize)}, nil
}

然后准备请求，设置内容长度并发送：

file, err := os.Open("/path/to/file.ext")
if err != nil {
    return nil, err
}
defer file.Close()

up, err := prepareMultipartPayload(map[string]interface{}{
    "a_string":      "field",
    "another_field": "yep",
    "file":          file,  // you can have multiple file fields
})
r, _ := http.NewRequest("POST", "https://example.com/upload", up.body)
for k, v := range up.headers {
    r.Header.Set(k, v)
}
r.ContentLength = up.size
c := http.Client{}
res, err := c.Do(r)