如何有效地绘制波形

Question

我正在尝试显示音频文件的波形。我希望在 ffmpeg 处理文件时逐步绘制波形，而不是在完成后立即绘制。虽然我已经达到了这个效果，但它真的很慢；像痛苦的慢。它开始时非常快，但速度会下降到需要几分钟才能抽取样本。

我觉得必须有一种方法可以更有效地做到这一点，因为我使用了一个程序来做到这一点，我只是不知道怎么做。另一个程序可以接收超过 10 小时的音频，并逐渐显示波形而不会降低速度。我已将 ffmpeg 设置为以 500 个样本/秒的速度处理文件，但其他程序以 1000 个/秒的速度采样，它的运行速度仍然比我写的要快。另一个程序的波形显示只需要大约 120MB 的 RAM 和 10 小时的文件，而我的需要 1.5GB 的 10 分钟文件。

我相当肯定缓慢是由所有 UI 更新引起的，而 RAM 使用来自所有正在创建的矩形对象。当我禁用绘制波形时，异步流完成得非常快； 10 小时的文件不到 1 分钟。

这是我能想到的实现我想要的唯一方法。我欢迎任何帮助以改进我所写的内容或任何关于以不同方式完成它的建议。

附带说明，这不是我想要展示的全部内容。我最终会想要添加一个背景网格来帮助判断时间，并添加可拖动的线注释来标记波形中的特定位置。

MainWindow.xaml

<ItemsControl x:Name="AudioDisplayItemsControl"
              DockPanel.Dock="Top"
              Height="100"
              ItemsSource="{Binding Samples}">
    <ItemsControl.Resources>
        <DataTemplate DataType="{x:Type poco:Sample}">
            <Rectangle Width="{Binding Width}"
                       Height="{Binding Height}"
                       Fill="ForestGreen"/>
        </DataTemplate>
    </ItemsControl.Resources>
    <ItemsControl.ItemsPanel>
        <ItemsPanelTemplate>
            <Canvas Background="Black"
                    Width="500"/>
        </ItemsPanelTemplate>
    </ItemsControl.ItemsPanel>
    <ItemsControl.ItemContainerStyle>
        <Style TargetType="ContentPresenter">
            <Setter Property="Canvas.Top" Value="{Binding Top}"/>
            <Setter Property="Canvas.Left" Value="{Binding Left}"/>
        </Style>
    </ItemsControl.ItemContainerStyle>
</ItemsControl>

MainWindow.xaml.cs

private string _audioFilePath;
public string AudioFilePath
{
    get => _audioFilePath;
    set
    {
        if (_audioFilePath != value)
        {
            _audioFilePath = value;
            NotifyPropertyChanged();
        }
    }
}

private ObservableCollection<IShape> _samples;
public ObservableCollection<IShape> Samples
{
    get => _samples;
    set
    {
        if (_samples != value)
        {
            _samples = value;
            NotifyPropertyChanged();
        }
    }
}

//Eventhandler that starts this whole process
private async void GetGetWaveform_Click(object sender, RoutedEventArgs e)
{
    ((Button)sender).IsEnabled = false;
    await GetWaveformClickAsync();
    ((Button)sender).IsEnabled = true;
}


private async Task GetWaveformClickAsync()
{
    Samples.Clear();
    double left = 0;
    double width = .01;
    double top = 0;
    double height = 0;
    await foreach (var sample in FFmpeg.GetAudioWaveform(AudioFilePath).ConfigureAwait(false))
    {
        // Map {-32,768, 32,767} (pcm_16le) to {-50, 50} (height of sample display)
        // I don't this this mapping is correct, but that's not important right now 
        height = ((sample + 32768) * 100 / 65535) - 50;

        // "0" pcm values are not drawn in order to save on UI updates,
        // but draw position is still advanced
        if (height==0)
        {
            left += width;
            continue;
        }
        // Positive pcm values stretch "height" above the canvas center line
        if (height > 0)
        top = 50 - height;
        // Negative pcm values stretch "height" below the centerline
        else
        {
            top = 50;
            height = -height;
        }

        Samples.Add(new Sample
        {
            Height = height,
            Width = width,
            Top = top,
            Left = left,
            ZIndex = 1
            });
        left += width;
    }
}

用于定义样本的类

public interface IShape
{
    double Top { get; set; }
    double Left { get; set; }
}

public abstract class Shape : IShape
{
    public double Top { get; set; }
    public double Left { get; set; }
    public int ZIndex { get; set; }
}

public class Sample : Shape
{
    public double Width { get; set; }
    public double Height { get; set; }
}

FFMpeg.cs

public static class FFmpeg
{
    public static async IAsyncEnumerable<short> GetAudioWaveform(string filename)
    {
        var args = GetFFmpegArgs(FFmpegTasks.GetWaveform, filename);

        await foreach (var sample in RunFFmpegAsyncStream(args))
        {
            yield return sample;
        }
    }



    /// <summary>
    /// Streams raw results of running ffmpeg.exe with given arguments string
    /// </summary>
    /// <param name="args">CLI argument string used for ffmpeg.exe</param>
    private static async IAsyncEnumerable<short> RunFFmpegAsyncStream(string args)
    {
        using (var process = new Process())
        {
            process.StartInfo.FileName = @"External\ffmpeg\bin\x64\ffmpeg.exe";
            process.StartInfo.Arguments = args;
            process.StartInfo.UseShellExecute = false;
            process.StartInfo.RedirectStandardError = true;
            process.StartInfo.RedirectStandardOutput = true;
            process.StartInfo.CreateNoWindow = true;

            process.Start();
            process.BeginErrorReadLine();

            var buffer = new byte[2];
            while (true)
            {
                // Asynchronously read a pcm16_le value from ffmpeg.exe output
                var r = await process.StandardOutput.BaseStream.ReadAsync(buffer, 0, 2);

                if (r == 0)
                break;

                yield return BitConverter.ToInt16(buffer);
            }
        }
    }

FFmpegTasks 只是一个枚举。 GetFFmpegArgs 使用 FFmpegTasks 上的 switch 参数为 ffmpeg.exe 返回适当的 CLI 参数。

我尝试使用以下类而不是标准的 ObservableCollection，因为我希望减少 UI 更新会加快速度，但实际上它会使绘制波形变慢。

RangeObservableCollection.cs

public class RangeObservableCollection<T> : ObservableCollection<T>
{
    private bool _suppressNotification = false;

    protected override void OnCollectionChanged(NotifyCollectionChangedEventArgs e)
    {
        if (!_suppressNotification)
        base.OnCollectionChanged(e);
    }

    public void AddRange(IEnumerable<T> list)
    {
        if (list == null)
        throw new ArgumentNullException("list");

        _suppressNotification = true;

        foreach (T item in list)
        {
            Add(item);
        }
        _suppressNotification = false;
        OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset));
    }
}

Answer 1

您可以尝试手动绘制Path。我用它在应用程序中绘制图像的直方图：

/// <summary>
/// Converts a histogram to a <see cref="PathGeometry"/>.
/// This is used by converters to draw a path.
/// It is easiest to use the default Canvas width and height and then use a ViewBox.
/// </summary>
/// <param name="histogram">The counts of each value.</param>
/// <param name="CanvasWidth">Width of the canvas</param>
/// <param name="CanvasHeight">Height of the canvas.</param>=
/// <returns>A path geometry. This value can be bound to a <see cref="Path"/>'s Data.</returns>
public static PathGeometry HistogramToPathGeometry(int[] histogram, double CanvasWidth = 100.0, double CanvasHeight = 100.0)
{
    double xscale = CanvasWidth / histogram.Length;
    double histMax = histogram.Max();

    double yscale = CanvasHeight / histMax;

    List<LineSegment> segments = new List<LineSegment>();
    for (int i = 0; i < histogram.Length; i++)
    {
        double X = i * xscale;
        double Y1 = histogram[i] * yscale;

        double Y = CanvasHeight - Y1;
        if (Y == double.PositiveInfinity) Y = CanvasHeight;
        segments.Add(new LineSegment(new Point(X, Y), true));
    }
    segments.Add(new LineSegment(new Point(CanvasWidth, CanvasHeight), true));
    PathGeometry geometry = new PathGeometry();
    PathFigure figure = new PathFigure(new Point(0, CanvasHeight), segments, true);

    geometry.Figures = new PathFigureCollection
    {
        figure
    };
    return geometry;
}

然后在你的 Xaml 中有这个：

<Canvas Width="100" Height="100">
    <Path Data="{Binding ConvertedPathGeometry}" />
</Canvas>

您可以对其进行更改，以便它在数据进入时对其进行处理。我不确定它在处理大量点时的效果如何，但您只能在出现一些新点后更新视图中。我已经尝试在显示器中绘制许多矩形并且遇到了同样的问题。

Answer 2

正如所承诺的，这里是绘制我的波形所涉及的所有代码。我可能添加了一些不必要的内容，但确实如此。我希望在处理波形时画出它，但现在放弃了这个想法。我希望这对那里的人有所帮助，因为我总共花了 2 周的时间来解决这个问题。欢迎评论。

仅供参考，在我的电脑上处理约 10 小时的 m4b 文件大约需要 35 秒，显示图像需要

此外，它采用 little-endian 系统。我没有检查大端系统，因为我不在其中。如果你是，Buffer.BlockCopy 需要注意。

Mainwindow.xaml

<Border Background="Black"
        Height="100"
        Width="720"
        BorderThickness="0">
    <Image x:Name="WaveformImg" Source="{Binding Waveform, Mode=OneWay}"
    Height="100"
    Width="{Binding ImageWidth, Mode=OneWayToSource}"
    Stretch="Fill" />
</Border>

<Button Content="Get Waveform"
        Padding="5 0"
        Margin="5 0"
        Click="GetGetWaveform_Click" />

Mainwindow.xaml.cs

private async void GetGetWaveform_Click (object sender, RoutedEventArgs e)
{

    ((Button)sender).IsEnabled = false;
    await ProcessAudiofile();
    await GetWaveformClickAsync(0, AudioData!.TotalSamples - 1);
    ((Button)sender).IsEnabled = true;
}

private async Task ProcessAudiofile ()
{
    var milliseconds = FFmpeg.GetAudioDurationSeconds(AudioFilePath);
    AudioData = new AudioData(await milliseconds, FFmpeg.SampleRate, new Progress<ProgressStatus>(ReportProgress));

    await AudioData.ProcessStream(FFmpeg.GetAudioWaveform(AudioFilePath)).ConfigureAwait(false);
}

private const int _IMAGE_WIDTH = 720;
private async Task GetWaveformClickAsync (int min, int max)
{
    var sw = new Stopwatch();
    sw.Start();

    int color_ForestGreen = 0xFF << 24 | 0x22 << 16 | 0x8c << 8 | 0x22 << 0; //

    Waveform = new WriteableBitmap(_IMAGE_WIDTH, 100, 96, 96, PixelFormats.Bgra32, null);

    int col = 0;
    int currSample = 0;
    int row;
    var sampleTop = 0;
    var sampleBottom = 0;
    var sampleHeight = 0;

    //I thought this would draw the wave form line by line but it blocks the UI and draws the whole waveform at once
    foreach (var sample in AudioData.GetSamples(_IMAGE_WIDTH, min, max))
    {
        sampleBottom = 50 + (int)(sample.min * (double)50 / short.MinValue);
        sampleTop = 50 - (int)(sample.max * (double)50 / short.MaxValue);
        sampleHeight = sampleBottom - sampleTop;

        try
        {
            Waveform.Lock();

            DrawLine(col, sampleTop, sampleHeight, color_ForestGreen);

            col++;
        }
        finally
        {
            Waveform.Unlock();
        }
    }

    sw.Stop();
    Debug.WriteLine($"Image Creation: {sw.Elapsed}");
}


private void DrawLine (int column, int top, int height, int color)
{
    unsafe
    {

        IntPtr pBackBuffer = Waveform.BackBuffer;
        for (int i = 0; i < height; i++)
        {
            pBackBuffer = Waveform.BackBuffer;                    // Backbuffer start address
            pBackBuffer += (top + i) * Waveform.BackBufferStride; // Move to address or desired row
            pBackBuffer += column * 4;                            // Move to address of desired column

            *((int*)pBackBuffer) = color;
        }

    }

    try
    {
        Waveform.AddDirtyRect(new Int32Rect(column, top, 1, height));
    }
    catch (Exception) { } // I know this isn't a good way to deal with exceptions, but its what i did.
}

AudioData.cs

public class AudioData
    {
        private List<short> _amp;           // pcm_s16le amplitude values
        private int _expectedTotalSamples;  // Number of samples expected to be returned by FFMpeg
        private int _totalSamplesRead;      // Current total number of samples read from the file

        private IProgress<ProgressStatus>? _progressIndicator;  // Communicates progress to the progress bar
        private ProgressStatus _progressStatus;                 // Progress status to be passed to progress bar

        /// <summary>
        /// Total number of samples obtained from the audio file
        /// </summary>
        public int TotalSamples
        {
            get => _amp.Count;
        }   

        /// <summary>
        /// Length of audio file in seconds
        /// </summary>
        public double Duration
        {
            get => _duration;
            private set
            {
                _duration = value < 0 ? 0 : value;
            }
        }
        private double _duration;

        /// <summary>
        /// Number of data points per second of audio
        /// </summary>
        public int SampleRate
        {
            get => _sampleRate;
            private set
            {
                _sampleRate = value < 0 ? 0 : value;
            }
        }
        private int _sampleRate;

        /// <summary>Update the window's size.</summary>
        /// <param name = "duration" > How long the audio file is in milliseconds.</param>
        /// <param name = "sampleRate" > Number of times per second to sample the audio file</param>
        /// <param name = "progressIndicator" >Used to report progress back to a progress bar</param>
        public AudioData (double duration, int sampleRate, IProgress<ProgressStatus>? progressIndicator = null)
        {
            Duration = duration;
            SampleRate = sampleRate;
            _progressIndicator = progressIndicator;

            _expectedTotalSamples = (int)Math.Ceiling(Duration * SampleRate);

            _amp = new List<short>();
            _progressStatus = new ProgressStatus();
        }


        /// <summary>
        /// Get values from an async pcm_s16le stream from FFMpeg
        /// </summary>
        /// <param name = "sampleStream" >FFMpeg samples stream</param>
        public async Task ProcessStream (IAsyncEnumerable<(int read, short[] samples)> sampleStream)
        {
            _totalSamplesRead = 0;

            _progressStatus = new ProgressStatus
            {
                Label = "Started",
            };

            await foreach ((int read, short[] samples) in sampleStream)
            {
                _totalSamplesRead += read;
                _amp.AddRange(samples[..read]);     // Only add the number of samples that where read this iteration
                
                UpdateProgress();
            }

            Duration = _amp.Count() / SampleRate;   // update duration to the correct value; incase duration reported by FFMpeg was wrong
            UpdateProgress(true);
        }

        /// <summary>
        /// Report progress back to the UI
        /// </summary>
        /// <param name="finished">Is FFmpeg done processing the file</param>
        private void UpdateProgress (bool finished = false)
        {
            int percent = (int)(100 * (double)_totalSamplesRead / _expectedTotalSamples);

            // Calculate progress update interval; once every 1%
            if (percent == _progressStatus.Value)
                return;

            // update progress status bar object
            if (finished)
            {
                _progressStatus.Label = "Done";
                _progressStatus.Value = 100;
            }
            else
            {
                _progressStatus.Label = $"Running ({_totalSamplesRead} / {_expectedTotalSamples})";
                _progressStatus.Value = percent;
            }


            _progressIndicator?.Report(_progressStatus);
        }

        /// <summary>
        /// Get evenly spaced sample subsets of the entire audio file samples
        /// </summary>
        /// <param name="count">Number of samples to be returned</param>
        /// <returns>An IEnumerable tuple containg the minimum and maximum amplitudes of a range of samples</returns>
        public IEnumerable<(short min, short max)> GetSamples (int count)
        {
            foreach (var sample in GetSamples(count, 0, -_amp.Count - 1))
                yield return sample;
        }

        /// <summary>
        /// Get evenly spaced sample subsets of a section of the audio file samples
        /// </summary>
        /// <param name="count">number of data points to return</param>
        /// <param name="min">inclusive starting index</param>
        /// <param name="max">inclusive ending index</param>
        /// <returns>An IEnumerable tuple containing the minimum and maximum amplitudes of a range of samples</returns>
        public IEnumerable<(short min, short max)> GetSamples (int count, int min, int max)
        {
            // Protect from out of range exception
            max = max >= _amp.Count ? _amp.Count - 1 : max;
            max = max < 1 ? 1 : max;
            min = min >= _amp.Count - 1 ? _amp.Count - 2 : min;
            min = min < 0 ? 0 : min;

            
            double sampleSize = (max - min) / (double)count;  // Number of samples to inspect for return value
            short rMin;         // Minimum return value
            short rMax;         // Maximum return value
            int ssOffset;
            int ssLength;
            for (int n = 0; n < count; n++)
            {
                // Calculate offset; no account for min
                ssOffset = (int)(n * sampleSize);

                // Determine how many samples to get, with a minimum of 1
                ssLength = sampleSize <= 1 ? 1 : (int)((n + 1) * sampleSize) - ssOffset;
                
                //shift offset to account for min
                ssOffset += min;   
                
                // Double check that ssLength wont take us out of bounds
                ssLength = ssOffset + ssLength >= _amp.Count ? _amp.Count - ssOffset : ssLength;

                // Get the minimum and maximum amplitudes in this sample range
                rMin = _amp.GetRange(ssOffset, ssLength).Min();
                rMax = _amp.GetRange(ssOffset, ssLength).Max();

                // In case this sample range has no (-) values, make the lowest one zero. This makes the rendered waveform look better.
                rMin = rMin > 0 ? (short)0 : rMin;
                rMax = rMax < 0 ? (short)0 : rMax;

                yield return (rMin, rMax);
            }
        }
    }

public class ProgressStatus
{
    public int Value { get; set; }
    public string Label { get; set; }
}

FFMpeg.cs

private const int _BUFER_READ_SIZE = 500;  // Number of bytes to read from process.StandardOutput.BaseStream. Must be a multiple of 2
public const int SampleRate = 1000;

public static async IAsyncEnumerable<(int, short[])> GetAudioWaveform (string filename)
{
    using (var process = new Process())
    {
        process.StartInfo = FFMpegStartInfo(FFmpegTasks.GetWaveform, filename);
        process.Start();
        process.BeginErrorReadLine();

        await Task.Delay(1000);         // Give process.StandardOutput a chance to build up values in BaseStream
        var bBuffer = new byte[_BUFER_READ_SIZE];    // BaseStream.ReadAsync buffer
        var sBuffer = new short[_BUFER_READ_SIZE / 2];    // Return value buffer; bBuffer.length
        int read = 1;  // ReadAsync returns 0 when BaseStream is empty
        while (true)
        {
            read = await process.StandardOutput.BaseStream.ReadAsync(bBuffer, 0, _BUFER_READ_SIZE);

            if (read == 0)
                break;


            Buffer.BlockCopy(bBuffer, 0, sBuffer, 0, read);
            yield return (read / 2, sBuffer);
        }
    }
}

private static ProcessStartInfo FFMpegStartInfo (FFmpegTasks task, string inputFile1, string inputFile2 = "", string outputFile = "", bool overwriteOutput = true)
{
    if (string.IsNullOrWhiteSpace(inputFile1))
        throw new ArgumentException(nameof(inputFile1), "Path to input file is null or empty");
    if (!File.Exists(inputFile1))
        throw new FileNotFoundException($"No file found at: {inputFile1}", nameof(inputFile1));

    var args = task switch
    {
        // TODO: Set appropriate sample rate
        // TODO: remove -t xx
        FFmpegTasks.GetWaveform => $@" -i ""{inputFile1}"" -ac 1 -filter:a aresample={SampleRate} -map 0:a -c:a pcm_s16le -f data -",
        FFmpegTasks.DetectSilence => throw new NotImplementedException(),
        _ => throw new NotImplementedException(),
    };

    return new ProcessStartInfo()
    {
        FileName = @"External\ffmpeg\bin\x64\ffmpeg.exe",
        Arguments = args,
        UseShellExecute = false,
        RedirectStandardError = true,
        RedirectStandardOutput = true,
        CreateNoWindow = true
    };

    public enum FFmpegTasks
    {
        GetWaveform
    }
}