在 Android 上使用 OpenCV 和 Unity

Question

我在 Android 上将 OpenCV 与 Unity 结合使用时遇到问题。我使用OpenCV库原生c++插件（.so文件）我想检测魔方和魔方的颜色。

所以我将数据从 Unity 的 webcamtexture 传递到 OpenCV。当 webcamtexture 的宽度和高度很小（320x240）时，它工作得很好。但是当 webcamtexture 的宽度和高度很大（480x640）时会出现问题。而当 webcamtexture 的分辨率设置为全屏（在本例中我设置为 800x1280）时，应用会停止。

我尝试调试。当我通过使用 Marshal.Copy 函数将数据从 OpenCV 复制到 Unity 时，它会停止。我该如何解决这个问题？

enter image description here

（其分辨率为 320x240）

[案例1（分辨率为320x240）]

using UnityEngine;
using UnityEngine.UI;

using System.Runtime.InteropServices;
using System;

public class Controller : MonoBehaviour {
    public float RotateSpeed = 0.5f;

    public RawImage InImage;
    public RawImage OutImage;

    WebCamTexture wct;
    Texture2D outTexture;

    void Awake()
    {
#if UNITY_EDITOR
        int width = 1280;
        int height = 720;
#else
        int width = 320;
        int height = 240;
#endif

        NativeLibAdapter.InitCV(width, height);

        outTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);

        wct = new WebCamTexture(width, height);
        wct.Play();
        // Ignore it.
        Debug.LogWarning("Foo Value in C++ is " + NativeLibAdapter.FooTest());
    }

    void Update()
    {

        if (wct.width > 100 && wct.height > 100)
        {
            Color32[] pixels = wct.GetPixels32();
            GCHandle pixelHandle = GCHandle.Alloc(pixels, GCHandleType.Pinned);

            IntPtr results = NativeLibAdapter.SubmitFrame(wct.width, wct.height, pixelHandle.AddrOfPinnedObject());
            int bufferSize = wct.width * wct.height * 4;
            byte[] rawData = new byte[bufferSize];

            if (results != IntPtr.Zero)
            {
                Marshal.Copy(results, rawData, 0, bufferSize);

                outTexture.LoadRawTextureData(rawData);
                outTexture.Apply();
            }

            OutImage.texture = outTexture;

            rawData = null;
            pixelHandle.Free();
        }
    }
}

enter image description here （它的分辨率是480x640。有很多噪点。）

[案例2（分辨率为480x640）]

using UnityEngine;
using UnityEngine.UI;

using System.Runtime.InteropServices;
using System;

public class Controller : MonoBehaviour {
    public float RotateSpeed = 0.5f;

    public RawImage InImage;
    public RawImage OutImage;

    WebCamTexture wct;
    Texture2D outTexture;

    void Awake()
    {
#if UNITY_EDITOR
        int width = 1280;
        int height = 720;
#else
        int width = 480;
        int height = 640;
#endif

        NativeLibAdapter.InitCV(width, height);

        outTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);

        wct = new WebCamTexture(width, height);
        wct.Play();
        // Ignore it.
        Debug.LogWarning("Foo Value in C++ is " + NativeLibAdapter.FooTest());
    }

    void Update()
    {

        if (wct.width > 100 && wct.height > 100)
        {
            Color32[] pixels = wct.GetPixels32();
            GCHandle pixelHandle = GCHandle.Alloc(pixels, GCHandleType.Pinned);

            IntPtr results = NativeLibAdapter.SubmitFrame(wct.width, wct.height, pixelHandle.AddrOfPinnedObject());
            int bufferSize = wct.width * wct.height * 4;
            byte[] rawData = new byte[bufferSize];

            if (results != IntPtr.Zero)
            {
                Marshal.Copy(results, rawData, 0, bufferSize);

                outTexture.LoadRawTextureData(rawData);
                outTexture.Apply();
            }

            OutImage.texture = outTexture;

            rawData = null;
            pixelHandle.Free();
        }
    }
}

它只是改变宽度和高度变量。

[案例3（设备全屏分辨率）]

using UnityEngine;
using UnityEngine.UI;

using System.Runtime.InteropServices;
using System;

public class Controller : MonoBehaviour {
    public float RotateSpeed = 0.5f;

    public RawImage InImage;
    public RawImage OutImage;

    WebCamTexture wct;
    Texture2D outTexture;

    void Awake()
    {
#if UNITY_EDITOR
        int width = 1280;
        int height = 720;
#else
        int width = 480;
        int height = 640;
#endif

        NativeLibAdapter.InitCV(Screen.width, Screen.height);

        outTexture = new Texture2D(Screen.width, Screen.height, TextureFormat.RGBA32, false);

        wct = new WebCamTexture(Screen.width, Screen.height);
        wct.Play();

        // Ignore it.
        Debug.LogWarning("Foo Value in C++ is " + NativeLibAdapter.FooTest());
    }

    void Update()
    {

        if (wct.width > 100 && wct.height > 100)
        {
            Color32[] pixels = wct.GetPixels32();
            GCHandle pixelHandle = GCHandle.Alloc(pixels, GCHandleType.Pinned);

            IntPtr results = NativeLibAdapter.SubmitFrame(wct.width, wct.height, pixelHandle.AddrOfPinnedObject());
            int bufferSize = wct.width * wct.height * 4;
            byte[] rawData = new byte[bufferSize];

            if (results != IntPtr.Zero)
            {
                // here! App stops. I don't know why App stops.
                Marshal.Copy(results, rawData, 0, bufferSize);

                outTexture.LoadRawTextureData(rawData);
                outTexture.Apply();
            }

            OutImage.texture = outTexture;

            rawData = null;
            pixelHandle.Free();
        }
    }
}

我正在使用：

• IDE
• 团结 2018.1.19f
• Android Studio 3.4.2
• OpenCV 3.4.3

Answer 1

您有以下代码初始化您的 CV 缓冲区：

#if UNITY_EDITOR
        int width = 1280;
        int height = 720;
#else
        int width = 480;
        int height = 640;
#endif

        NativeLibAdapter.InitCV(Screen.width, Screen.height);

您对 CV 缓冲区的初始化不足以处理 800x1280，因此您可能内存不足。预计性能会随着大小的增加而下降，因为内存消耗更大（640x480 的数据量是 320x240 的 4 倍，因此每帧的垃圾收集和数据处理量要多得多）。在将数据输入 CV 之前，您需要增加宽度和高度以包含最大分辨率，或者限制从 GetPixels() 检索到的像素。