Python Flask块数据上传不起作用

Question

我正在尝试从客户端（使用 Python request.post）上传一个大文件（比如 ~1GB）到烧瓶服务器。

当客户端以 1024 的块向服务器发送请求时，服务器不会读取整个文件并保存到服务器 0kb。

你能帮我调试一下我在这里到底犯了什么错误吗？

服务器 - 烧瓶代码：

from flask import Flask, request, jsonify
from werkzeug.utils import secure_filename
import os

app = Flask(__name__)

app.config['UPLOAD_FOLDER'] = 'uploads/'

@app.route("/upload/<filename>", methods=["POST", "PUT"])
def upload_process(filename):
    filename = secure_filename(filename)
    fileFullPath = os.path.join(app.config['UPLOAD_FOLDER'], filename)

    with open(fileFullPath, "wb") as f:
        chunk_size = 1024
        chunk = request.stream.read(chunk_size)
        f.write(chunk)
    return jsonify({'filename': filename})


if __name__ == '__main__':
    app.run(host="0.0.0.0", port=int("8080"),debug=True)

客户端 - 请求代码

import os
import requests


def read_in_chunks(file_object, chunk_size=1024):
    while True:
        data = file_object.read(chunk_size)
        if not data:
            break
        yield data


def main(fname, url):
    content_path = os.path.abspath(fname)
    with open(content_path, 'r') as f:
        try:
            r = requests.post(url, data=read_in_chunks(f))
            print "r: {0}".format(r)
        except Exception, e:
            print e


if __name__ == '__main__':
    filename = 'bigfile.zip'  # ~1GB
    url = 'http://localhost:8080/upload/{0}'.format(filename)
    main(filename, url)

Answer 1

Flask 依赖 werkzeug 来处理流，以及 werkzeug demands a content length for a stream。 here 上有一个帖子，但目前没有真正的解决方案，除了采用另一种框架方法。

Answer 2

请使用“file.stream.read(chunk_size)”而不是 request.stream.read(chunk_size)。它对我有用...！

Answer 3

下面的这个例子应该很适合你们。如果您使用 Redis，您还可以在另一个 API 中发布/订阅进度条正在处理的块。

from flask import Flask, request, jsonify
@app.route("/submit_vdo", methods=['POST'])
def submit_vdo():

@copy_current_request_context
def receive_chunk(stream, full_file_path):
    if full_file_path is None:
        tmpfile = tempfile.NamedTemporaryFile('wb+', prefix=str(uuid.uuid4())+"_")
        full_file_path = tmpfile.name

    print ('Write temp to ', full_file_path)
    with open(full_file_path, "wb") as f:
        max_chunk_size = settings.VIDEO_MAX_SIZE_CHUNK # config.MAX_UPLOAD_BYTE_LENGHT
        count_chunks = 0
        total_uploaded = 0
        try:
            while True:
                print ('Chunk ', count_chunks)
                chunk = stream.read(max_chunk_size)
                if chunk is not None and len(chunk)>0:
                    total_uploaded += len(chunk)
                    count_chunks += 1
                    f.write(chunk)   
                    temp = {}
                    temp ['chunk_counts'] = count_chunks
                    temp ['total_bytes']  = total_uploaded
                    temp ['status'] = 'uploading...'
                    temp ['success'] = True
                    db_apn_logging.set(user_id+"@CHUNK_DOWNLOAD", json.dumps(temp), ex=5)
                    print (temp)
                else:
                    f.close()
                    temp = {}
                    temp ['chunk_counts'] = count_chunks
                    temp ['total_bytes']  = total_uploaded
                    temp ['status'] = 'DONE'
                    temp ['success'] = True
                    db_apn_logging.set(user_id+"@CHUNK_DOWNLOAD", json.dumps(temp), ex=5)
                    break
        except Exception as e:
            temp = {}
            temp ['chunk_counts'] = count_chunks
            temp ['total_bytes']  = total_uploaded
            temp ['status'] = e
            temp ['success'] = False
            db_apn_logging.set(user_id+"@CHUNK_DOWNLOAD", json.dumps(temp), ex=5)
            return  None

        return full_file_path

    stream = flask.request.files['file']
    stream.seek(0)
    full_file_path = receive_chunk(stream, full_file_path)

    return "DONE !"

Answer 4

老帖子，但我正在寻找类似的东西，所以我还是会在这里发帖。

服务器以write 模式读取文件，该模式将覆盖每个块。首选append模式：

with open(fileFullPath, "ab") as f:

客户端需要以字节模式读取文件：

with open(content_path, "rb") as f:

最后，生成器read_in_chunks在传递给请求之前需要循环使用：

def main(fname, url):
    content_path = os.path.abspath(fname)
    with open(content_path, "rb") as f:
        try:
            for data in read_in_chunks(f):
                r = requests.post(url, data=data)
                print("r: {0}".format(r))
        except Exception as e:
            print(e)

那么你就有了 2 个文件

服务器

from flask import Flask, request, jsonify
from werkzeug.utils import secure_filename
import os

app = Flask(__name__)

app.config["UPLOAD_FOLDER"] = "uploads/"


@app.route("/upload/<filename>", methods=["POST", "PUT"])
def upload_process(filename):
    filename = secure_filename(filename)
    fileFullPath = os.path.join(app.config["UPLOAD_FOLDER"], filename)

    with open(fileFullPath, "ab") as f:
        chunk_size = 1024
        chunk = request.stream.read(chunk_size)
        f.write(chunk)
    return jsonify({"filename": filename})


if __name__ == "__main__":
    app.run(host="0.0.0.0", port=int("8080"), debug=True)

客户

import os
import requests


def read_in_chunks(file_object, chunk_size=1024):
    while True:
        data = file_object.read(chunk_size)
        if not data:
            break
        yield data


def main(fname, url):
    content_path = os.path.abspath(fname)
    with open(content_path, "rb") as f:
        try:
            for data in read_in_chunks(f):
                r = requests.post(url, data=data)
                print("r: {0}".format(r))
        except Exception as e:
            print(e)


if __name__ == "__main__":
    filename = "bigfile.zip"  # ~1GB
    url = "http://localhost:8080/upload/{0}".format(filename)
    main(filename, url)

请注意，发布 un 块通常需要块的总数和文件的哈希来验证上传。