ARKit 3.5 – 如何使用 LiDAR 从新 iPad Pro 导出 OBJ？

Question

如何将最新 iPad Pro 上新的SceneReconstruction API 生成的ARMeshGeometry 导出到.obj 文件？

这是SceneReconstruction 文档。

Answer 1

此代码 sn-p 允许您将 LiDAR 的几何图形保存为 USD 并通过 AirDrop 将其发送到 Mac 计算机。您不仅可以导出.usd，还可以导出.usda、.usdc、.obj、.stl、.abc和.ply文件格式。

此外，您可以使用 SceneKit 的 write(to:options:delegate:progressHandler:) 方法来保存 .usdz 版本的文件。

import RealityKit
import ARKit
import MetalKit
import ModelIO

@IBOutlet var arView: ARView!
var saveButton: UIButton!
let rect = CGRect(x: 50, y: 50, width: 100, height: 50)

override func viewDidLoad() {
    super.viewDidLoad()

    let tui = UIControl.Event.touchUpInside
    saveButton = UIButton(frame: rect)
    saveButton.setTitle("Save", for: [])
    saveButton.addTarget(self, action: #selector(saveButtonTapped), for: tui)
    self.view.addSubview(saveButton)
}

@objc func saveButtonTapped(sender: UIButton) {        
    print("Saving is executing...")
    
    guard let frame = arView.session.currentFrame
    else { fatalError("Can't get ARFrame") }
            
    guard let device = MTLCreateSystemDefaultDevice()
    else { fatalError("Can't create MTLDevice") }
    
    let allocator = MTKMeshBufferAllocator(device: device)        
    let asset = MDLAsset(bufferAllocator: allocator)       
    let meshAnchors = frame.anchors.compactMap { $0 as? ARMeshAnchor }
    
    for ma in meshAnchors {
        let geometry = ma.geometry
        let vertices = geometry.vertices
        let faces = geometry.faces
        let vertexPointer = vertices.buffer.contents()
        let facePointer = faces.buffer.contents()
        
        for vtxIndex in 0 ..< vertices.count {
            
            let vertex = geometry.vertex(at: UInt32(vtxIndex))                
            var vertexLocalTransform = matrix_identity_float4x4
            
            vertexLocalTransform.columns.3 = SIMD4<Float>(x: vertex.0,
                                                          y: vertex.1,
                                                          z: vertex.2,
                                                          w: 1.0)
            
            let vertexWorldTransform = (ma.transform * vertexLocalTransform).position                
            let vertexOffset = vertices.offset + vertices.stride * vtxIndex               
            let componentStride = vertices.stride / 3
            
            vertexPointer.storeBytes(of: vertexWorldTransform.x,
                           toByteOffset: vertexOffset,
                                     as: Float.self)
            
            vertexPointer.storeBytes(of: vertexWorldTransform.y,
                           toByteOffset: vertexOffset + componentStride,
                                     as: Float.self)
            
            vertexPointer.storeBytes(of: vertexWorldTransform.z,
                           toByteOffset: vertexOffset + (2 * componentStride),
                                     as: Float.self)
        }
        
        let byteCountVertices = vertices.count * vertices.stride            
        let byteCountFaces = faces.count * faces.indexCountPerPrimitive * faces.bytesPerIndex
        
        let vertexBuffer = allocator.newBuffer(with: Data(bytesNoCopy: vertexPointer, 
                                                                count: byteCountVertices, 
                                                          deallocator: .none), type: .vertex)
        
        let indexBuffer = allocator.newBuffer(with: Data(bytesNoCopy: facePointer, 
                                                               count: byteCountFaces, 
                                                         deallocator: .none), type: .index)
        
        let indexCount = faces.count * faces.indexCountPerPrimitive            
        let material = MDLMaterial(name: "material", 
                     scatteringFunction: MDLPhysicallyPlausibleScatteringFunction())
        
        let submesh = MDLSubmesh(indexBuffer: indexBuffer, 
                                  indexCount: indexCount, 
                                   indexType: .uInt32, 
                                geometryType: .triangles, 
                                    material: material)
        
        let vertexFormat = MTKModelIOVertexFormatFromMetal(vertices.format)
        
        let vertexDescriptor = MDLVertexDescriptor()
        
        vertexDescriptor.attributes[0] = MDLVertexAttribute(name: MDLVertexAttributePosition, 
                                                          format: vertexFormat, 
                                                          offset: 0, 
                                                     bufferIndex: 0)
        
        vertexDescriptor.layouts[0] = MDLVertexBufferLayout(stride: ma.geometry.vertices.stride)
        
        let mesh = MDLMesh(vertexBuffer: vertexBuffer, 
                            vertexCount: ma.geometry.vertices.count, 
                             descriptor: vertexDescriptor, 
                              submeshes: [submesh])

        asset.add(mesh)
    }

    let filePath = FileManager.default.urls(for: .documentDirectory, 
                                             in: .userDomainMask).first!
    
    let usd: URL = filePath.appendingPathComponent("model.usd")

    if MDLAsset.canExportFileExtension("usd") {
        do {
            try asset.export(to: usd)
            
            let controller = UIActivityViewController(activityItems: [usd],
                                              applicationActivities: nil)
            controller.popoverPresentationController?.sourceView = sender
            self.present(controller, animated: true, completion: nil)

        } catch let error {
            fatalError(error.localizedDescription)
        }
    } else {
        fatalError("Can't export USD")
    }
}

按下Save 按钮后，在Activity View Controller 中选择More 并通过AirDrop 将准备好的模型发送到Mac 的Downloads 文件夹。

附：在这里您可以找到有关capturing real-world texture 的额外信息。

Answer 2

从 Apple 的 Visualising Scene Scemantics 示例应用程序开始，您可以从框架中的第一个锚点检索 ARMeshGeometry 对象。

导出数据最简单的方法是首先将其转换为 MDLMesh：

extension ARMeshGeometry {
    func toMDLMesh(device: MTLDevice) -> MDLMesh {
        let allocator = MTKMeshBufferAllocator(device: device);

        let data = Data.init(bytes: vertices.buffer.contents(), count: vertices.stride * vertices.count);
        let vertexBuffer = allocator.newBuffer(with: data, type: .vertex);

        let indexData = Data.init(bytes: faces.buffer.contents(), count: faces.bytesPerIndex * faces.count * faces.indexCountPerPrimitive);
        let indexBuffer = allocator.newBuffer(with: indexData, type: .index);

        let submesh = MDLSubmesh(indexBuffer: indexBuffer,
                                 indexCount: faces.count * faces.indexCountPerPrimitive,
                                 indexType: .uInt32,
                                 geometryType: .triangles,
                                 material: nil);

        let vertexDescriptor = MDLVertexDescriptor();
        vertexDescriptor.attributes[0] = MDLVertexAttribute(name: MDLVertexAttributePosition,
                                                            format: .float3,
                                                            offset: 0,
                                                            bufferIndex: 0);
        vertexDescriptor.layouts[0] = MDLVertexBufferLayout(stride: vertices.stride);

        return MDLMesh(vertexBuffer: vertexBuffer,
                       vertexCount: vertices.count,
                       descriptor: vertexDescriptor,
                       submeshes: [submesh]);
    }
}

拥有 MDLMesh 后，导出到 OBJ 文件是一件轻而易举的事：

    @IBAction func exportMesh(_ button: UIButton) {
        let meshAnchors = arView.session.currentFrame?.anchors.compactMap({ $0 as? ARMeshAnchor });

        DispatchQueue.global().async {

            let directory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0];
            let filename = directory.appendingPathComponent("MyFirstMesh.obj");

            guard let device = MTLCreateSystemDefaultDevice() else {
                print("metal device could not be created");
                return;
            };

            let asset = MDLAsset();

            for anchor in meshAnchors! {
                let mdlMesh = anchor.geometry.toMDLMesh(device: device);
                asset.add(mdlMesh);
            }

            do {
                try asset.export(to: filename);
            } catch {
                print("failed to write to file");
            }
        }
    }

Answer 3

@swiftcoder 的回答效果很好。但是在有多个锚点的情况下，您需要根据锚点变换将顶点坐标转换为世界坐标系。在相反的情况下，所有网格都将放置在零位置，您将一团糟。

更新后的代码如下所示：

extension ARMeshGeometry {
    func toMDLMesh(device: MTLDevice, transform: simd_float4x4) -> MDLMesh {
        let allocator = MTKMeshBufferAllocator(device: device)

        let data = Data.init(bytes: transformedVertexBuffer(transform), count: vertices.stride * vertices.count)
        let vertexBuffer = allocator.newBuffer(with: data, type: .vertex)

        let indexData = Data.init(bytes: faces.buffer.contents(), count: faces.bytesPerIndex * faces.count * faces.indexCountPerPrimitive)
        let indexBuffer = allocator.newBuffer(with: indexData, type: .index)

        let submesh = MDLSubmesh(indexBuffer: indexBuffer,
                                 indexCount: faces.count * faces.indexCountPerPrimitive,
                                 indexType: .uInt32,
                                 geometryType: .triangles,
                                 material: nil)

        let vertexDescriptor = MDLVertexDescriptor()
        vertexDescriptor.attributes[0] = MDLVertexAttribute(name: MDLVertexAttributePosition,
                                                            format: .float3,
                                                            offset: 0,
                                                            bufferIndex: 0)
        vertexDescriptor.layouts[0] = MDLVertexBufferLayout(stride: vertices.stride)

        return MDLMesh(vertexBuffer: vertexBuffer,
                       vertexCount: vertices.count,
                       descriptor: vertexDescriptor,
                       submeshes: [submesh])
    }

    func transformedVertexBuffer(_ transform: simd_float4x4) -> [Float] {
        var result = [Float]()
        for index in 0..<vertices.count {
            let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index)
            let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee
            var vertextTransform = matrix_identity_float4x4
            vertextTransform.columns.3 = SIMD4<Float>(vertex.0, vertex.1, vertex.2, 1)
            let position = (transform * vertextTransform).position
            result.append(position.x)
            result.append(position.y)
            result.append(position.z)
        }
        return result
    }
}

extension simd_float4x4 {
    var position: SIMD3<Float> {
        return SIMD3<Float>(columns.3.x, columns.3.y, columns.3.z)
    }
}

extension Array where Element == ARMeshAnchor {
    func save(to fileURL: URL, device: MTLDevice) throws {
        let asset = MDLAsset()
        self.forEach {
            let mesh = $0.geometry.toMDLMesh(device: device, transform: $0.transform)
            asset.add(mesh)
        }
        try asset.export(to: fileURL)
    }
}

我不是 ModelIO 方面的专家，也许有更简单的方法来转换顶点缓冲区 :) 但是这段代码对我有用。