Kinect 导致 Unity3D 中的帧速率限制

Question

我目前正在开发一个同时使用 Oculus Rift 和 Kinect 的 Unity3D 项目。然而，Kinect 将帧速率限制为 30 fps，而 Rift 需要 60 fps 才能获得流畅的体验。我用的是官方的，绝对不是性能问题。

我将this 包装器与官方 Kinect SDK 一起使用。

我已将原因缩小到下面的这段代码。我怀疑 getSkeleton() 函数会阻塞主线程，直到它从 Kinect 接收到数据。由于 Kinect 仅以 30 fps 的速度运行，因此应用程序的其余部分无法运行得比这更快。

    public bool pollSkeleton () {
    if (!updatedSkeleton)
    {
        updatedSkeleton = true; 
        if (kinect.pollSkeleton())
        {
            newSkeleton = true;
            System.Int64 cur = kinect.getSkeleton().liTimeStamp;
            System.Int64 diff = cur - ticks;
            ticks = cur;
            deltaTime = diff / (float)1000;
            processSkeleton();
        }
    }
    return newSkeleton;
}

也许我可以运行一个单独的线程，但由于我没有多线程编程经验，我想知道是否有人有更简单的解决方案？

我的猜测是，很多 Oculus Rift 开发人员将使用 Kinect+Unity3D 组​​合，因此会遇到与我相同的限制。任何帮助将不胜感激！

编辑： 我创建了一个单独的线程来从 kinect 轮询骨架信息：这是修改后的 SkeletonWrapper.cs 的完整代码。希望它能帮助你们中的一些人在同样的问题上挣扎。

using UnityEngine;
using System.Collections;
using Kinect;
using System.Threading;

public class SkeletonWrapper : MonoBehaviour {

public DeviceOrEmulator devOrEmu;
private Kinect.KinectInterface kinect;

private bool updatedSkeleton = false;
private bool newSkeleton = false;

[HideInInspector]
public Kinect.NuiSkeletonTrackingState[] players;
[HideInInspector]
public int[] trackedPlayers;
[HideInInspector]
public Vector3[,] bonePos;
[HideInInspector]
public Vector3[,] rawBonePos;
[HideInInspector]
public Vector3[,] boneVel;
[HideInInspector]
public Quaternion[,] boneLocalOrientation;
[HideInInspector]
public Quaternion[,] boneAbsoluteOrientation;

public Kinect.NuiSkeletonPositionTrackingState[,] boneState;    
private System.Int64 ticks;
private float deltaTime;

private Matrix4x4 kinectToWorld;
public Matrix4x4 flipMatrix;

private Thread thread = null;
private bool isThreadRunning = false;

// Use this for initialization
void Start () {
    kinect = devOrEmu.getKinect();
    players = new Kinect.NuiSkeletonTrackingState[Kinect.Constants.NuiSkeletonCount];
    trackedPlayers = new int[Kinect.Constants.NuiSkeletonMaxTracked];
    trackedPlayers[0] = -1;
    trackedPlayers[1] = -1;
    bonePos = new Vector3[2,(int)Kinect.NuiSkeletonPositionIndex.Count];
    rawBonePos = new Vector3[2,(int)Kinect.NuiSkeletonPositionIndex.Count];
    boneVel = new Vector3[2,(int)Kinect.NuiSkeletonPositionIndex.Count];

    boneState = new Kinect.NuiSkeletonPositionTrackingState[2,(int)Kinect.NuiSkeletonPositionIndex.Count];
    boneLocalOrientation = new Quaternion[2, (int)Kinect.NuiSkeletonPositionIndex.Count];
    boneAbsoluteOrientation = new Quaternion[2, (int)Kinect.NuiSkeletonPositionIndex.Count];

    //create the transform matrix that converts from kinect-space to world-space
    Matrix4x4 trans = new Matrix4x4();
    trans.SetTRS( new Vector3(-kinect.getKinectCenter().x,
                              kinect.getSensorHeight()-kinect.getKinectCenter().y,
                              -kinect.getKinectCenter().z),
                 Quaternion.identity, Vector3.one );
    Matrix4x4 rot = new Matrix4x4();
    Quaternion quat = new Quaternion();
    double theta = Mathf.Atan((kinect.getLookAt().y+kinect.getKinectCenter().y-kinect.getSensorHeight()) / (kinect.getLookAt().z + kinect.getKinectCenter().z));
    float kinectAngle = (float)(theta * (180 / Mathf.PI));
    quat.eulerAngles = new Vector3(-kinectAngle, 0, 0);
    rot.SetTRS( Vector3.zero, quat, Vector3.one);

    //final transform matrix offsets the rotation of the kinect, then translates to a new center
    kinectToWorld = flipMatrix*trans*rot;

    thread = new Thread(ThreadUpdate);
    thread.Start(); 
}

void OnDestroy()
{
    if (isThreadRunning)
    {
        isThreadRunning = false;
        thread.Abort();
        thread = null;
    }
}

// Update is called once per frame
void Update () {

}

void LateUpdate () {
    updatedSkeleton = false;
    newSkeleton = false;
}

private void ThreadUpdate()
{
    isThreadRunning = true;

    while (isThreadRunning)
    {
        // This function is capping the FPS to 30. 
        if (kinect.pollSkeleton())
        {
            System.Int64 cur = kinect.getSkeleton().liTimeStamp;
            System.Int64 diff = cur - ticks;
            ticks = cur;
            deltaTime = diff / (float)1000;
            processSkeleton();
            newSkeleton = true;
        }
    }
}

/// <summary>
/// First call per frame checks if there is a new skeleton frame and updates,
/// returns true if there is new data
/// Subsequent calls do nothing have the same return as the first call.
/// </summary>
/// <returns>
/// A <see cref="System.Boolean"/>
/// </returns>
public bool pollSkeleton () {
    //if (!updatedSkeleton)
    //{
    //    updatedSkeleton = true;
    //    //this function is capping the FPS to 30. 
    //    //It might be blocking the main thread because it waits for the kinects skeleton input which only runs 30 fps
    //    //possible solution: run function in seperate thread
    //    if (kinect.pollSkeleton())
    //    {
    //        newSkeleton = true;
    //        System.Int64 cur = kinect.getSkeleton().liTimeStamp;
    //        System.Int64 diff = cur - ticks;
    //        ticks = cur;
    //        deltaTime = diff / (float)1000;
    //        processSkeleton();
    //    }
    //}
    return newSkeleton;
}

private void processSkeleton () {
    int[] tracked = new int[Kinect.Constants.NuiSkeletonMaxTracked];
    tracked[0] = -1;
    tracked[1] = -1;
    int trackedCount = 0;
    //update players
    for (int ii = 0; ii < Kinect.Constants.NuiSkeletonCount; ii++)
    {
        players[ii] = kinect.getSkeleton().SkeletonData[ii].eTrackingState;
        if (players[ii] == Kinect.NuiSkeletonTrackingState.SkeletonTracked)
        {
            tracked[trackedCount] = ii;
            trackedCount++;
        }
    }
    //this should really use trackingID instead of index, but for now this is fine
    switch (trackedCount)
    {
    case 0:
        trackedPlayers[0] = -1;
        trackedPlayers[1] = -1;
        break;
    case 1:
        //last frame there were no players: assign new player to p1
        if (trackedPlayers[0] < 0 && trackedPlayers[1] < 0)
            trackedPlayers[0] = tracked[0];
        //last frame there was one player, keep that player in the same spot
        else if (trackedPlayers[0] < 0) 
            trackedPlayers[1] = tracked[0];
        else if (trackedPlayers[1] < 0)
            trackedPlayers[0] = tracked[0];
        //there were two players, keep the one with the same index (if possible)
        else
        {
            if (tracked[0] == trackedPlayers[0])
                trackedPlayers[1] = -1;
            else if (tracked[0] == trackedPlayers[1])
                trackedPlayers[0] = -1;
            else
            {
                trackedPlayers[0] = tracked[0];
                trackedPlayers[1] = -1;
            }
        }
        break;
    case 2:
        //last frame there were no players: assign new players to p1 and p2
        if (trackedPlayers[0] < 0 && trackedPlayers[1] < 0)
        {
            trackedPlayers[0] = tracked[0];
            trackedPlayers[1] = tracked[1];
        }
        //last frame there was one player, keep that player in the same spot
        else if (trackedPlayers[0] < 0)
        {
            if (trackedPlayers[1] == tracked[0])
                trackedPlayers[0] = tracked[1];
            else{
                trackedPlayers[0] = tracked[0];
                trackedPlayers[1] = tracked[1];
            }
        }
        else if (trackedPlayers[1] < 0)
        {
            if (trackedPlayers[0] == tracked[1])
                trackedPlayers[1] = tracked[0];
            else{
                trackedPlayers[0] = tracked[0];
                trackedPlayers[1] = tracked[1];
            }
        }
        //there were two players, keep the one with the same index (if possible)
        else
        {
            if (trackedPlayers[0] == tracked[1] || trackedPlayers[1] == tracked[0])
            {
                trackedPlayers[0] = tracked[1];
                trackedPlayers[1] = tracked[0];
            }
            else
            {
                trackedPlayers[0] = tracked[0];
                trackedPlayers[1] = tracked[1];
            }
        }
        break;
    }

    //update the bone positions, velocities, and tracking states)
    for (int player = 0; player < 2; player++)
    {
        //print(player + ", " +trackedPlayers[player]);
        if (trackedPlayers[player] >= 0)
        {
            for (int bone = 0; bone < (int)Kinect.NuiSkeletonPositionIndex.Count; bone++)
            {
                Vector3 oldpos = bonePos[player,bone];

                bonePos[player,bone] = kinectToWorld.MultiplyPoint3x4(kinect.getSkeleton().SkeletonData[trackedPlayers[player]].SkeletonPositions[bone]);
                //bonePos[player,bone] = kinectToWorld.MultiplyPoint3x4(bonePos[player, bone]);
                rawBonePos[player, bone] = kinect.getSkeleton().SkeletonData[trackedPlayers[player]].SkeletonPositions[bone];


                Kinect.NuiSkeletonBoneOrientation[] or = kinect.getBoneOrientations(kinect.getSkeleton().SkeletonData[trackedPlayers[player]]);
                boneLocalOrientation[player,bone] = or[bone].hierarchicalRotation.rotationQuaternion.GetQuaternion();
                boneAbsoluteOrientation[player,bone] = or[bone].absoluteRotation.rotationQuaternion.GetQuaternion();

                //print("index " + bone + ", start" + (int)or[bone].startJoint + ", end" + (int)or[bone].endJoint);

                boneVel[player,bone] = (bonePos[player,bone] - oldpos) / deltaTime;
                boneState[player,bone] = kinect.getSkeleton().SkeletonData[trackedPlayers[player]].eSkeletonPositionTrackingState[bone];
                //print(kinect.getSkeleton().SkeletonData[player].Position.z);
            }
        }
    }
}

}

Answer 1

我不太了解 Kinect SDK 的工作原理。它可能已经在管理一个正在执行骨骼跟踪的后台线程，并且 pollSkeleton 方法只是阻塞，直到下一帧可用。

这似乎是一个合理的假设，因为 SDK 支持轮询和基于事件的通知，这意味着如果您不进行轮询，其他东西将触发获取下一帧并向您发送事件。

如果是这种情况，那么您可以通过简单地扫描线程中可用的骨架数据上的时间戳来解决您的问题...

System.Int64 lastSkeletonTime = 0;
public bool pollSkeleton () 
{
  if (kinect.getSkeleton().liTimeStamp > lastSkeletonTime) {
    updatedSkeleton = true; 
    newSkeleton = true;
    System.Int64 cur = kinect.getSkeleton().liTimeStamp;
    System.Int64 diff = cur - lastSkeletonTime;
    deltaTime = diff / (float)1000;
    lastSkeletonTime = cur;
    processSkeleton();
  }
  return newSkeleton;
}

如果这不起作用，那么您很可能需要启动后台线程，或切换到处理事件。