从由 C++ 程序写出的 C# 文件中读取二进制对象

Question

我正在尝试从包含填充结构的非常大的文件中读取对象，这些填充结构由 C++ 进程写入其中。我正在使用一个示例对大文件进行内存映射并尝试将数据反序列化为一个对象，但我现在可以看到它不会以这种方式工作。

如何从文件中提取所有对象以在 C# 中使用？我可能很遥远，但我已经提供了代码。对象有一个 8 字节的毫秒成员，后跟 21 个 16 位整数，需要 6 字节的填充才能与 8 字节的边界对齐。

[Serializable]
unsafe public struct DataStruct
{
    public UInt64 milliseconds;
    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 21)]
    public fixed Int16 data[21];
    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 3)]
    public fixed Int16 padding[3];

};

[Serializable]
public class DataArray
{
    public DataStruct[] samples;
}

public static class Helper
{
    public static Int16[] GetData(this DataStruct data)
    {
        unsafe
        {
            Int16[] output = new Int16[21];
            for (int index = 0; index < 21; ++index)
                output[index] = data.data[index];
            return output;
        }
    }
}

class FileThreadSupport
{
    struct DataFileInfo
    {
        public string path;
        public UInt64 start;
        public UInt64 stop;
        public UInt64 elements;
    };

    // Create our epoch timestamp
    private static readonly DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);

    // Output TCP client
    private Support.AsyncTcpClient output;

    // Directory which contains our data
    private string replay_directory;

    // Files to be read from
    private DataFileInfo[] file_infos;

    // Current timestamp of when the process was started
    UInt64 process_start = 0;

    // Object from current file
    DataArray current_file_data;

    // Offset into current files
    UInt64 current_file_index = 0;

    // Offset into current files
    UInt64 current_file_offset = 0;

    // Run flag
    bool run = true;

    public FileThreadSupport(ref Support.AsyncTcpClient output, ref Engine.A.Information info, ref Support.Configuration configuration)
    {
        // Set our output directory
        replay_directory = configuration.getString("replay_directory");
        if (replay_directory.Length == 0)
        {
            Console.WriteLine("Configuration does not provide a replay directory");
            return;
        }

        // Check the directory for playable files
        if(!loadDataDirectory(replay_directory))
        {
            Console.WriteLine("Replay directory {} did not have any valid files", replay_directory);
        }

        // Set the output TCP client
        this.output = output;
    }

    private bool loadDataDirectory(string directory)
    {
        string[] files = Directory.GetFiles(directory, "*.*", SearchOption.TopDirectoryOnly);
        file_infos = new DataFileInfo[files.Length];
        int index = 0;
        foreach (string file in files)
        {
            string[] parts = file.Split('\\');
            string name = parts.Last();
            parts = name.Split('.');
            if (parts.Length != 2)
                continue;
            UInt64 start, stop = 0;
            if (!UInt64.TryParse(parts[0], out start) || !UInt64.TryParse(parts[1], out stop))
                continue;

            long size = new System.IO.FileInfo(file).Length;

            // Add to our file info array
            file_infos[index] = new DataFileInfo
            {
                path = file,
                start = start,
                stop = stop,
                elements = (ulong)(new System.IO.FileInfo(file).Length / 56 
                /*System.Runtime.InteropServices.Marshal.SizeOf(typeof(DataStruct))*/)
            };
            ++index;
        }

        // Sort the array
        Array.Sort(file_infos, delegate (DataFileInfo x, DataFileInfo y) { return x.start.CompareTo(y.start); });

        // Return whether or not there were files found
        return (files.Length > 0);
    }

    public void start()
    {
        process_start = (ulong)DateTime.Now.ToUniversalTime().Subtract(epoch).TotalMilliseconds;
        UInt64 num_samples = 0;

        while(run)
        {
            // Get our samples and add it to the sample
            DataStruct[] result = getData(100);
            Engine.A.A message = new Engine.A.A();
            for (int i = 0; i < result.Length; ++i)
            {
                Engine.A.Data sample = new Engine.A.Data();
                sample.Time = process_start + num_samples * 4;
                Int16[] signal_data = Helper.GetData(result[i]);
                for(int e = 0; e < signal_data.Length; ++e)
                    sample.Value[e] = signal_data[e];
                message.Signal.Add(sample);
                ++num_samples;
            }

            // Send out the websocket
            this.output.SendAsync(message.ToByteArray());

            // Sleep 100 milliseconds
            Thread.Sleep(100);
        }
    }

    public void stop()
    {
        run = false;
    }

    private DataStruct[] getData(UInt64 milliseconds)
    {
        if (file_infos.Length == 0)
            return new DataStruct[0];

        if (current_file_data == null)
        {
            current_file_data = ReadObjectFromMMF(file_infos[current_file_index].path) as DataArray;
            if(current_file_data.samples.Length == 0)
                return new DataStruct[0];
        }

        UInt64 elements_to_read = (UInt64) milliseconds / 4;
        DataStruct[] result = new DataStruct[elements_to_read];
        Array.Copy(current_file_data.samples, (int)current_file_offset, result, 0, (int) Math.Min(elements_to_read, file_infos[current_file_index].elements - current_file_offset));
        while((UInt64) result.Length != elements_to_read)
        {
            current_file_index = (current_file_index + 1) % (ulong) file_infos.Length;
            current_file_data = ReadObjectFromMMF(file_infos[current_file_index].path) as DataArray;
            if (current_file_data.samples.Length == 0)
                return new DataStruct[0];
            current_file_offset = 0;
            Array.Copy(current_file_data.samples, (int)current_file_offset, result, result.Length, (int)Math.Min(elements_to_read, file_infos[current_file_index].elements - current_file_offset));
        }
        return result;
    }

    private object ByteArrayToObject(byte[] buffer)
    {
        BinaryFormatter binaryFormatter = new BinaryFormatter(); // Create new BinaryFormatter
        MemoryStream memoryStream = new MemoryStream(buffer);    // Convert buffer to memorystream
        return binaryFormatter.Deserialize(memoryStream);        // Deserialize stream to an object
    }

    private object ReadObjectFromMMF(string file)
    {
        // Get a handle to an existing memory mapped file
        using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(file, FileMode.Open))
        {
            // Create a view accessor from which to read the data
            using (MemoryMappedViewAccessor mmfReader = mmf.CreateViewAccessor())
            {
                // Create a data buffer and read entire MMF view into buffer
                byte[] buffer = new byte[mmfReader.Capacity];
                mmfReader.ReadArray<byte>(0, buffer, 0, buffer.Length);

                // Convert the buffer to a .NET object
                return ByteArrayToObject(buffer);
            }
        }
    }

Answer 1

一方面，您根本没有很好地使用该内存映射文件，您只是在缓冲区中按顺序读取它，这既不必要地低效，而且比您简单地打开文件以正常读取要慢得多.内存映射文件的卖点是由操作系统的虚拟内存分页支持的重复随机访问和随机更新。

而且您绝对不需要读取内存中的整个文件，因为您的数据结构非常强大。您确切知道要为一条记录读取多少字节：Marshal.SizeOf<DataStruct>()。

然后你需要摆脱所有的序列化噪音。同样，您的数据是强类型的，只需阅读它。摆脱那些fixed 数组并使用常规数组，您已经在指导编组器如何使用MarshalAs 属性读取它们（好）。这也摆脱了只是出于某种未知原因复制数组的辅助函数。

您的读取循环非常简单：读取一个条目的正确字节数，使用Marshal.PtrToStructure 将其转换为可读结构并将其添加到列表以在最后返回。如果您可以使用 .Net Core 和 Unsafe.As 或 Unsafe.Cast，则可以加分。

编辑：不要使用object返回，你知道你要返回什么，把它写下来。