从 IDirect3DSurface 到 SoftwareBitmap 的无效转换

Question

我尝试使用 UWP API 实现从网络摄像头到 WPF 应用程序的处理帧。 有文章如何使用 MediaCapture 和 MediaFrameReader： https://docs.microsoft.com/en-us/windows/uwp/audio-video-camera/process-media-frames-with-mediaframereader#handle-the-frame-arrived-event

如果我将 MemoryPreference 设置为 cpu，SoftwareBitmaps 会在事件中初始化为 null。当我放置 Auto 时，我可以看到 IDirect3DSurface 对象在事件中，但在转换为 SoftwareBitmap 时出现异常“指定的转换无效”。被提升了。

如何将 IDirect3DSurface 转换为 SoftwareBitmap？

private async void MediaCaptureExample()
{
    var frameSourceGroups = await MediaFrameSourceGroup.FindAllAsync();

    MediaFrameSourceGroup selectedGroup = null;
    MediaFrameSourceInfo colorSourceInfo = null;

    foreach (var sourceGroup in frameSourceGroups)
    {
        foreach (var sourceInfo in sourceGroup.SourceInfos)
        {
            if (sourceInfo.MediaStreamType == MediaStreamType.VideoRecord && sourceInfo.SourceKind == MediaFrameSourceKind.Color)
            {
                colorSourceInfo = sourceInfo;
                break;
            }
        }
        if (colorSourceInfo != null)
        {
            selectedGroup = sourceGroup;
            break;
        }
    }

    capture = new MediaCapture();

    var settings = new MediaCaptureInitializationSettings()
    {
        SourceGroup = selectedGroup,
        SharingMode = MediaCaptureSharingMode.ExclusiveControl,
        MemoryPreference = MediaCaptureMemoryPreference.Auto,
        StreamingCaptureMode = StreamingCaptureMode.Video
    };

    await capture.InitializeAsync(settings);

    var colorFrameSource = capture.FrameSources[colorSourceInfo.Id];

    var preferredFormat = colorFrameSource.SupportedFormats.Where(format =>
    {
        return format.VideoFormat.Width >= 1080
        && String.Compare(format.Subtype, MediaEncodingSubtypes.Mjpg, true) == 0;

    }).FirstOrDefault();

    if (preferredFormat == null)
    {
        // Our desired format is not supported
        return;
    }

    await colorFrameSource.SetFormatAsync(preferredFormat);

    mediaFrameReader = await capture.CreateFrameReaderAsync(colorFrameSource);
    mediaFrameReader.FrameArrived += MediaFrameReader_FrameArrived;
    var result = await mediaFrameReader.StartAsync();
    Console.WriteLine("Result = " + result.ToString());
}

private void MediaFrameReader_FrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args)
{
    try
    {
        var mediaFrameReference = sender.TryAcquireLatestFrame();
        var videoMediaFrame = mediaFrameReference?.VideoMediaFrame;
        var softwareBitmap = videoMediaFrame?.SoftwareBitmap;
        var direct3DSurface = videoMediaFrame?.Direct3DSurface;

        if (direct3DSurface != null)
        {
            var softwareBitmapTask = SoftwareBitmap.CreateCopyFromSurfaceAsync(mediaFrameReference.VideoMediaFrame.Direct3DSurface).AsTask();
            softwareBitmap = softwareBitmapTask.Result;
        }

        if (softwareBitmap != null)
        {
            using (var stream = new Windows.Storage.Streams.InMemoryRandomAccessStream())
            {
                var encoderTask = BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, stream).AsTask();
                encoderTask.Wait();
                var encoder = encoderTask.Result;
                encoder.SetSoftwareBitmap(softwareBitmap);
                Task t = encoder.FlushAsync().AsTask();
                t.Wait();
                var image = new System.Windows.Media.Imaging.BitmapImage();

                image.BeginInit();
                image.StreamSource = stream.AsStream();
                image.CacheOption = System.Windows.Media.Imaging.BitmapCacheOption.OnLoad;
                image.EndInit();

                imageElement.Source = image;
            }
        }
    } 
    catch(Exception e)
    {
        Console.WriteLine(e.Message);
    }
}

Answer 1

问题在于格式子类型。我将格式从 Mjpg 更改为 Nv12，一切都开始正常工作（即使是 MediaCaptureMemoryPreference.Auto）：

var preferredFormat = colorFrameSource.SupportedFormats.Where(format =>
{
    return format.VideoFormat.Width >= 1080 && String.Compare(format.Subtype, MediaEncodingSubtypes.Nv12, true) == 0;
}).FirstOrDefault();