如何在 Unity 中通过 Agora.io 使用场景相机

Question

在 Unity 中，我集成了 Agora.io，以便在我的虚拟现实应用程序中，我可以将视频通话连接到网页上的外部用户。 VR 用户可以看到网站用户，但网站用户看不到 VR 用户，因为没有可用的物理相机可用。有没有办法为 Agora 视频源使用场景摄像机？这意味着网站用户将能够看到 VR 用户的世界

Answer 1

是的。虽然我之前没有做过 VR 项目，但是这个概念应该是有的。您可以使用外部视频源来发送视频的任何帧，就好像它是从物理相机发送的一样。对于场景相机，您可以使用 RenderTexture 来输出相机馈送，并从 RenderTexture 中提取原始数据。所以步骤是：

1. 将相机设置为输出到 RenderTexture（如果需要，还可以在本地某处显示此 RenderTexture 的逻辑。）

2. 还要确保在设置 Agora RTC 引擎时，使用此调用启用外部视频源：

   mRtcEngine.SetExternalVideoSource(true, false);

3. 在每一帧，从 RenderTexture 中提取原始图像数据

4. 将原始帧数据发送到 SDK 函数 rtc.pushVideoFrame()

您可以在此处找到最后一步的代码 https://gist.github.com/icywind/92053d0983e713515c64d5c532ebee21

Answer 2

我修改了 Agora io 编辑的共享屏幕代码以提取渲染纹理。问题是我的接收器上只有白色或黑色屏幕，而我的渲染纹理是深度凸轮视频流。

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using agora_gaming_rtc;
using UnityEngine.UI;
using System.Globalization;
using System.Runtime.InteropServices;
using System;
public class ShareScreen : MonoBehaviour
{
    Texture2D mTexture;
    Rect mRect;
    [SerializeField]
    private string appId = "Your_AppID";
    [SerializeField]
    private string channelName = "agora";
    public IRtcEngine mRtcEngine;
    int i = 100;
    public RenderTexture depthMap;
    void Start()
    {
        Debug.Log("ScreenShare Activated");
        mRtcEngine = IRtcEngine.getEngine(appId);

        mRtcEngine.SetLogFilter(LOG_FILTER.DEBUG | LOG_FILTER.INFO | LOG_FILTER.WARNING | LOG_FILTER.ERROR | LOG_FILTER.CRITICAL);

    mRtcEngine.SetParameters("{\"rtc.log_filter\": 65535}");

    mRtcEngine.SetExternalVideoSource(true, false);

    mRtcEngine.EnableVideo();

    mRtcEngine.EnableVideoObserver();

    mRtcEngine.JoinChannel(channelName, null, 0);

    mRect = new Rect(0, 0, depthMap.width, depthMap.height); 

    mTexture = new Texture2D((int)mRect.width, (int)mRect.height, TextureFormat.RGBA32, false);
}
void Update()
{
    //Start the screenshare Coroutine
    StartCoroutine(shareScreen());
}
//Screen Share
IEnumerator shareScreen()
{
    yield return new WaitForEndOfFrame();
    //FB activate automaticaly the render texture for the copy
    RenderTexture.active = depthMap;
    //Read the Pixels inside the Rectangle
    mTexture.ReadPixels(mRect, 0, 0);
    //Apply the Pixels read from the rectangle to the texture
    mTexture.Apply();


    // Get the Raw Texture data from the the from the texture and apply it to an array of bytes
    byte[] bytes = mTexture.GetRawTextureData();
    // Make enough space for the bytes array
    int size = Marshal.SizeOf(bytes[0]) * bytes.Length;
    // Check to see if there is an engine instance already created
    IRtcEngine rtc = IRtcEngine.QueryEngine();
    //if the engine is present
    if (rtc != null)
    {
        //Create a new external video frame
        ExternalVideoFrame externalVideoFrame = new ExternalVideoFrame();
        //Set the buffer type of the video frame
        externalVideoFrame.type = ExternalVideoFrame.VIDEO_BUFFER_TYPE.VIDEO_BUFFER_RAW_DATA;
        // Set the video pixel format
        externalVideoFrame.format = ExternalVideoFrame.VIDEO_PIXEL_FORMAT.VIDEO_PIXEL_BGRA;
        //apply raw data you are pulling from the rectangle you created earlier to the video frame
        externalVideoFrame.buffer = bytes;
        //Set the width of the video frame (in pixels)
        externalVideoFrame.stride = (int)mRect.width;
        //Set the height of the video frame
        externalVideoFrame.height = (int)mRect.height;
        //Remove pixels from the sides of the frame
        externalVideoFrame.cropLeft = 0;
        externalVideoFrame.cropTop = 0;
        externalVideoFrame.cropRight = 0;
        externalVideoFrame.cropBottom = 0;
        //Rotate the video frame (0, 90, 180, or 270)
        externalVideoFrame.rotation = 180;
        // increment i with the video timestamp
        externalVideoFrame.timestamp = i++;
        //Push the external video frame with the frame we just created
        int a = rtc.PushVideoFrame(externalVideoFrame);
        Debug.Log(" pushVideoFrame =       " + a);
    }
}

}