在 Android 中合并多个 Azure 的云块 blob

Question

我是 Azure 的 Blob 云的新手。我想基本上将视频文件从我的 android 应用程序上传到 Azure 云，但我不能，因为一旦大小达到 32MB，它就会停止抛出 OutOfMemory 异常。因此，我对如何解决此问题进行了一些研究，并提出了将文件分解为字节并将其作为多个 blob 上传的解决方案。最后将它们编译成一个blob。但我不知道该怎么做。我尝试使用 commitBlockList，但我无法获得每个 blob 的 ID。

try {
        // Setup the cloud storage account.
        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
        int maxSize = 64 * Constants.MB;

        // Create a blob service client
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
        blobClient.getDefaultRequestOptions().setSingleBlobPutThresholdInBytes(maxSize);
        CloudBlobContainer container = blobClient.getContainerReference("testing");
        container.createIfNotExists();
        BlobContainerPermissions containerPermissions = new     BlobContainerPermissions();
        containerPermissions.setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
        container.uploadPermissions(containerPermissions);
        CloudBlockBlob finalFile = container.getBlockBlobReference("1.jpg");
        CloudBlob b = container.getBlockBlobReference("temp");
        String Lease = b.getSnapshotID();
        b.uploadFromFile(URL);
        List<BlockEntry> blockEntryIterator = new ArrayList<>();
        blockEntryIterator.add(new BlockEntry(Lease));
        finalFile.commitBlockList(blockEntryIterator);
    } catch (Throwable t) {

    }

~~~更新~~~

我试图将文件分成几部分，但现在出现此错误“指定的 blob 或块内容无效”。 public void splitTest(String URL) throws IOException, URISyntaxException, InvalidKeyException, StorageException {

    new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"STARTED");
    CloudBlockBlob blob = null;
    List<BlockEntry> blockList = null;
    try{
        // get file reference
        FileInputStream fs = new FileInputStream( URL );
        File sourceFile = new File( URL);

        // set counters
        long fileSize = sourceFile.length();
        int blockSize = 3 * (1024 * 1024); // 256K
        int blockCount = (int)((float)fileSize / (float)blockSize) + 1;
        long bytesLeft = fileSize;
        int blockNumber = 0;
        long bytesRead = 0;

        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
        CloudBlobContainer container = blobClient.getContainerReference("testing");
        String title = "Android_" + getFileNameFromUrl(URL);
        // get ref to the blob we are creating while uploading
        blob = container.getBlockBlobReference(title);
        blob.deleteIfExists();

        // list of all block ids we will be uploading - need it for the commit at the end
        blockList = new ArrayList<BlockEntry>();

        // loop through the file and upload chunks of the file to the blob
        while( bytesLeft > 0 ) {

            blockNumber++;
            // how much to read (only last chunk may be smaller)
            int bytesToRead = 0;
            if ( bytesLeft >= (long)blockSize ) {
                bytesToRead = blockSize;
            } else {
                bytesToRead = (int)bytesLeft;
            }

            // trace out progress
            float pctDone = ((float)blockNumber / (float)blockCount) * (float)100;


            // save block id in array (must be base64)
            String x = "";
            if(blockNumber<=9) {
                traceLine( "blockid: 000" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid000" + blockNumber;
            }
            else if(blockNumber>=10 && blockNumber<=99){
                traceLine( "blockid: 00" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid00" + blockNumber;
            }
            else if(blockNumber>=100 && blockNumber<=999){
                traceLine( "blockid0: " + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid0" + blockNumber;
            }
            else if(blockNumber>=1000 && blockNumber<=9999){
                traceLine( "blockid: " + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid" + blockNumber;
            }
            String blockId = Base64.encodeToString(x.getBytes(),Base64.DEFAULT).replace("\n","").toLowerCase();
            traceLine( "Base 64["+x+"] -> " + blockId);
            BlockEntry block = new BlockEntry(blockId);
            blockList.add(block);

            // upload block chunk to Azure Storage
            blob.uploadBlock( blockId, fs, (long)bytesToRead);

            // increment/decrement counters
            bytesRead += bytesToRead;
            bytesLeft -= bytesToRead;

        }
        fs.close();
        traceLine( "CommitBlockList. BytesUploaded: " + bytesRead);
        blob.commitBlockList(blockList);
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"UPLOAD COMPLETE");
        return;
    }
    catch (StorageException storageException) {
        traceLine("StorageException encountered: ");
        traceLine(storageException.getMessage());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        assert blockList != null;
        blob.commitBlockList(blockList);
        return;
    } catch( IOException ex ) {
        traceLine( "IOException: " + ex );
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        assert blockList != null;
        blob.commitBlockList(blockList);
        return;
    } catch (Exception e) {
        traceLine("Exception encountered: ");
        traceLine(e.getMessage());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        assert blockList != null;
        blob.commitBlockList(blockList);
        return;
    }
}

~~~工作更新~~~

对于任何想要重用此方法打开文件>并逐字节读取它的人都可以使用它。我能够上传 ~1GB 的文件，但之后它给了我错误 500 并崩溃。如果有人有任何解决方案，请告诉我。

public boolean splitTest(String URL) throws IOException, URISyntaxException, InvalidKeyException, StorageException {
    new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"STARTED");
    CloudBlockBlob blob = null;
    List<BlockEntry> blockList = null;
    try{
        // get file reference
        FileInputStream fs = new FileInputStream(URL);
        File sourceFile = new File(URL);

        // set counters
        long fileSize = sourceFile.length();
        int blockSize = 512 * 1024; // 256K
        //int blockSize = 1 * (1024 * 1024); // 256K
        int blockCount = (int)((float)fileSize / (float)blockSize) + 1;
        long bytesLeft = fileSize;
        int blockNumber = 0;
        long bytesRead = 0;

        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
        CloudBlobContainer container = blobClient.getContainerReference("testing");
        String title = "Android/Android_" + getFileNameFromUrl(URL).replace("\n","").replace(" ","_").replace("-","").toLowerCase();
        // get ref to the blob we are creating while uploading
        blob = container.getBlockBlobReference(title);
        traceLine("Title of blob -> " + title);

        if(blob.exists())
            blob.deleteIfExists();

        blob.setStreamWriteSizeInBytes(blockSize);
        // list of all block ids we will be uploading - need it for the commit at the end
        blockList = new ArrayList<>();

        // loop through the file and upload chunks of the file to the blob
        while( bytesLeft > 0 ) {
            // how much to read (only last chunk may be smaller)
            int bytesToRead = 0;
            if ( bytesLeft >= (long)blockSize ) {
                bytesToRead = blockSize;
            } else {
                bytesToRead = (int)bytesLeft;
            }

            // trace out progress
            float pctDone = ((float)blockNumber / (float)blockCount) * (float)100;


            // save block id in array (must be base64)
            String x = "";
            if(blockNumber<=9) {
                traceLine( "tempblobid0000" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid0000" + blockNumber;
            }
            else if(blockNumber>=10 && blockNumber<=99){
                traceLine( "tempblobid000" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid000" + blockNumber;
            }
            else if(blockNumber>=100 && blockNumber<=999){
                traceLine( "tempblobid00" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid00" + blockNumber;
            }
            else if(blockNumber>=1000 && blockNumber<=9999){
                traceLine( "tempblobid0" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid0" + blockNumber;
            }
            else if(blockNumber>=10000 && blockNumber<=99999){
                traceLine( "tempblobid" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid" + blockNumber;
            }
            String blockId = Base64.encodeToString(x.getBytes(),Base64.NO_WRAP).replace("\n","").toLowerCase();
            traceLine( "Base 64["+ x +"] -> " + blockId);
            BlockEntry block = new BlockEntry(blockId);
            blockList.add(block);
            // upload block chunk to Azure Storage
            blob.uploadBlock( blockId, fs, (long)bytesToRead);
            notification2(a,pctDone);
            //a.update(pctDone);
            // increment/decrement counters
            bytesRead += bytesToRead;
            bytesLeft -= bytesToRead;
            blockNumber++;
        }
        fs.close();
        traceLine( "CommitBlockList. BytesUploaded: " + bytesRead + "\t total bytes -> " + fileSize + "\tBytes Left -> " + bytesLeft);
        blob.commitBlockList(blockList);
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"UPLOAD COMPLETE");
        return true;
    }
    catch (StorageException storageException) {
        traceLine("StorageException encountered: ");
        traceLine(storageException.getMessage());
        traceLine("HTTP Status code -> " + storageException.getHttpStatusCode());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        if (blob != null) {
            blob.commitBlockList(blockList);
        }
        return false;
    } catch( IOException ex ) {
        traceLine( "IOException: " + ex );
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        if (blob != null) {
            blob.commitBlockList(blockList);
        }
        return false;
    } catch (Exception e) {
        traceLine("Exception encountered: ");
        traceLine(e.getMessage());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        if (blob != null) {
            blob.commitBlockList(blockList);
        }
        return false;
    }

}

Answer 1

这取决于您使用的是哪种手机。如果您的内存不足 32MB，那么您是在使用非常小的手机或使用许多其他进程。查看您的手机以及您有多少可用内存，就像 Gaurav 提到的和我的其他答案提到的那样，将阈值降低到那个水平。将自己分块并不能真正帮助您尝试做的事情。

您要查看的两个设置是 blob 本身的 singleBlobPutThresholdInBytes 和 setStreamWriteSizeInBytes。 singleBlobPutThresholdInBytes 影响我们何时开始分块与放置整个 blob 和 setStreamWriteSizeInBytes 如果发生分块会影响块的大小。 put 阈值默认为 64MB，写入大小默认为 4MB。请注意，如果您减小写入大小，您可能无法获得最大块 blob 大小，因为块 blob 被限制为 50k 个块。尝试将 singleBlobPutThreshold 减少到您可以处理的内存量，直到 4MB - 如果它小于 4MB，那么您真的有麻烦并且还需要减少流写入大小。