SpatialUnderstandingCustomMesh.cs

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.

using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using HoloToolkit.Unity.SpatialMapping;

#if UNITY_WSA
#if UNITY_2017_2_OR_NEWER
using UnityEngine.XR.WSA;
#else
using UnityEngine.VR.WSA;
#endif
#endif

namespace HoloToolkit.Unity
{
    /// <summary>
    /// Handles the custom meshes generated by the understanding DLL. The meshes
    /// are generated during the scanning phase and once more on scan finalization.
    /// The meshes can be used to visualize the scanning progress.
    /// </summary>
    public class SpatialUnderstandingCustomMesh : SpatialMappingSource
    {
        // Config
        [Tooltip("Indicate the time in seconds between mesh imports, during the scanning phase. A value of zero will disable pulling meshes from the DLL")]
        public float ImportMeshPeriod = 1.0f;

        [SerializeField]
        [Tooltip("Material used to render the custom mesh generated by the DLL")]
        private Material meshMaterial;

        public Material MeshMaterial
        {
            get
            {
                return meshMaterial;
            }
            set
            {
                meshMaterial = value;

                if (spatialUnderstanding.ScanState == SpatialUnderstanding.ScanStates.Done)
                {
                    for (int i = 0; i < SurfaceObjects.Count; ++i)
                    {
                        SurfaceObjects[i].Renderer.sharedMaterial = meshMaterial;
                    }
                }
            }
        }

        /// <summary>
        /// Used to keep our processing from exceeding our frame budget.
        /// </summary>
        [Tooltip("Max time per frame in milliseconds to spend processing the mesh")]
        public float MaxFrameTime = 5.0f;
        private float MaxFrameTimeInSeconds
        {
            get { return (MaxFrameTime / 1000); }
        }

        /// <summary>
        ///  Whether to create mesh colliders. If unchecked, mesh colliders will be empty and disabled.
        /// </summary>
        public bool CreateMeshColliders = true;

        private bool drawProcessedMesh = true;

        // Properties
        /// <summary>
        /// Controls rendering of the mesh. This can be set by the user to hide or show the mesh.
        /// </summary>
        public bool DrawProcessedMesh
        {
            get
            {
                return drawProcessedMesh;
            }
            set
            {
                drawProcessedMesh = value;
                for (int i = 0; i < SurfaceObjects.Count; ++i)
                {
                    SurfaceObjects[i].Renderer.enabled = drawProcessedMesh;
                }
            }
        }

        /// <summary>
        /// Have any mesh sectors been scanned
        /// </summary>
        public bool HasMeshSectors { get { return meshSectors != null && meshSectors.Count > 0; } }

        /// <summary>
        /// Indicates if the previous import is still being processed.
        /// </summary>
        public bool IsImportActive { get; private set; }

        /// <summary>
        /// The material to use if rendering.
        /// </summary>
        protected override Material RenderMaterial { get { return MeshMaterial; } }

        /// <summary>
        /// To prevent us from importing too often, we keep track of the last import.
        /// </summary>
        private float timeLastImportedMesh = 0;

        /// <summary>
        /// For a cached SpatialUnderstanding.Instance.
        /// </summary>
        private SpatialUnderstanding spatialUnderstanding;

        /// <summary>
        /// The spatial understanding mesh will be split into pieces so that we don't have to
        /// render the whole mesh, rather just the parts that are visible to the user.
        /// </summary>
        private Dictionary<Vector3, MeshData> meshSectors = new Dictionary<Vector3, MeshData>();

        /// <summary>
        /// A data structure to manage collecting triangles as we
        /// subdivide the spatial understanding mesh into smaller sub meshes.
        /// </summary>
        private class MeshData
        {
            /// <summary>
            /// Lists of vertices/triangles that describe the mesh geometry.
            /// </summary>
            private readonly List<Vector3> verts = new List<Vector3>();
            private readonly List<int> tris = new List<int>();

            /// <summary>
            /// The mesh object based on the triangles passed in.
            /// </summary>
            public readonly Mesh MeshObject = new Mesh();

            /// <summary>
            /// The MeshCollider with which this mesh is associated. Must be set even if
            /// no collision mesh will be created.
            /// </summary>
            public MeshCollider SpatialCollider = null;

            /// <summary>
            /// Whether to create collision mesh. If false, the MeshCollider attached to this
            /// object will also be disabled when Commit() is called.
            /// </summary>
            public bool CreateMeshCollider = false;

            /// <summary>
            /// Clears the geometry, but does not clear the mesh.
            /// </summary>
            public void Reset()
            {
                verts.Clear();
                tris.Clear();
            }

            /// <summary>
            /// Commits the new geometry to the mesh.
            /// </summary>
            public void Commit()
            {
                MeshObject.Clear();
                if (verts.Count > 2)
                {
                    MeshObject.SetVertices(verts);
                    MeshObject.SetTriangles(tris, 0);
                    MeshObject.RecalculateNormals();
                    MeshObject.RecalculateBounds();
                    // The null assignment is required by Unity in order to pick up the new mesh
                    SpatialCollider.sharedMesh = null;
                    if (CreateMeshCollider)
                    {
                        SpatialCollider.sharedMesh = MeshObject;
                        SpatialCollider.enabled = true;
                    }
                    else
                    {
                        SpatialCollider.enabled = false;
                    }
                }
            }

            /// <summary>
            /// Adds a triangle composed of the specified three points to our mesh.
            /// </summary>
            /// <param name="point1">First point on the triangle.</param>
            /// <param name="point2">Second point on the triangle.</param>
            /// <param name="point3">Third point on the triangle.</param>
            public void AddTriangle(Vector3 point1, Vector3 point2, Vector3 point3)
            {
                // Currently spatial understanding in the native layer voxellizes the space
                // into ~2000 voxels per cubic meter.  Even in a degenerate case we
                // will use far fewer than 65000 vertices, this check should not fail
                // unless the spatial understanding native layer is updated to have more
                // voxels per cubic meter.
                if (verts.Count < 65000)
                {
                    tris.Add(verts.Count);
                    verts.Add(point1);

                    tris.Add(verts.Count);
                    verts.Add(point2);

                    tris.Add(verts.Count);
                    verts.Add(point3);
                }
                else
                {
                    Debug.LogError("Mesh would have more vertices than Unity supports");
                }
            }
        }

        private void Start()
        {
            spatialUnderstanding = SpatialUnderstanding.Instance;
#if UNITY_WSA
            if (gameObject.GetComponent<WorldAnchor>() == null)
            {
                gameObject.AddComponent<WorldAnchor>();
            }
#endif
        }

        private void Update()
        {
            Update_MeshImport();
        }

        /// <summary>
        /// Adds a triangle with the specified points to the specified sector.
        /// </summary>
        /// <param name="sector">The sector to add the triangle to.</param>
        /// <param name="point1">First point of the triangle.</param>
        /// <param name="point2">Second point of the triangle.</param>
        /// <param name="point3">Third point of the triangle.</param>
        private void AddTriangleToSector(Vector3 sector, Vector3 point1, Vector3 point2, Vector3 point3)
        {
            // Grab the mesh container we are using for this sector.
            MeshData nextSectorData;
            if (!meshSectors.TryGetValue(sector, out nextSectorData))
            {
                nextSectorData = new MeshData();
                nextSectorData.CreateMeshCollider = CreateMeshColliders;

                int surfaceObjectIndex = SurfaceObjects.Count;

                SurfaceObject surfaceObject = CreateSurfaceObject(
                  mesh: nextSectorData.MeshObject,
                  objectName: string.Format("SurfaceUnderstanding Mesh-{0}", surfaceObjectIndex),
                  parentObject: transform,
                  meshID: surfaceObjectIndex,
                  drawVisualMeshesOverride: DrawProcessedMesh);

                nextSectorData.SpatialCollider = surfaceObject.Collider;

                AddSurfaceObject(surfaceObject);

                // Or make it if this is a new sector.
                meshSectors.Add(sector, nextSectorData);
            }

            // Add the vertices to the sector's mesh container.
            nextSectorData.AddTriangle(point1, point2, point3);
        }

        /// <summary>
        /// Imports the custom mesh from the DLL. This a a coroutine which will take multiple frames to complete.
        /// </summary>
        /// <returns></returns>
        public IEnumerator Import_UnderstandingMesh()
        {
            var stopwatch = System.Diagnostics.Stopwatch.StartNew();
            int startFrameCount = Time.frameCount;

            if (!spatialUnderstanding.AllowSpatialUnderstanding || IsImportActive)
            {
                yield break;
            }

            IsImportActive = true;

            SpatialUnderstandingDll dll = spatialUnderstanding.UnderstandingDLL;

            Vector3[] meshVertices = null;
            Vector3[] meshNormals = null;
            Int32[] meshIndices = null;

            // Pull the mesh - first get the size, then allocate and pull the data
            int vertCount;
            int idxCount;

            if ((SpatialUnderstandingDll.Imports.GeneratePlayspace_ExtractMesh_Setup(out vertCount, out idxCount) > 0) &&
                (vertCount > 0) &&
                (idxCount > 0))
            {
                meshVertices = new Vector3[vertCount];
                IntPtr vertPos = dll.PinObject(meshVertices);
                meshNormals = new Vector3[vertCount];
                IntPtr vertNorm = dll.PinObject(meshNormals);
                meshIndices = new Int32[idxCount];
                IntPtr indices = dll.PinObject(meshIndices);

                SpatialUnderstandingDll.Imports.GeneratePlayspace_ExtractMesh_Extract(vertCount, vertPos, vertNorm, idxCount, indices);
            }

            // Wait a frame
            stopwatch.Stop();
            yield return null;
            stopwatch.Start();

            // Create output meshes
            if ((meshVertices != null) &&
                (meshVertices.Length > 0) &&
                (meshIndices != null) &&
                (meshIndices.Length > 0))
            {
                // first get all our mesh data containers ready for meshes.
                foreach (MeshData meshdata in meshSectors.Values)
                {
                    meshdata.Reset();
                }

                float startTime = Time.realtimeSinceStartup;
                // first we need to split the playspace up into segments so we don't always
                // draw everything.  We can break things up in to cubic meters.
                for (int index = 0; index < meshIndices.Length; index += 3)
                {
                    Vector3 firstVertex = meshVertices[meshIndices[index]];
                    Vector3 secondVertex = meshVertices[meshIndices[index + 1]];
                    Vector3 thirdVertex = meshVertices[meshIndices[index + 2]];

                    // The triangle may belong to multiple sectors.  We will copy the whole triangle
                    // to all of the sectors it belongs to.  This will fill in seams on sector edges
                    // although it could cause some amount of visible z-fighting if rendering a wireframe.
                    Vector3 firstSector = VectorToSector(firstVertex);

                    AddTriangleToSector(firstSector, firstVertex, secondVertex, thirdVertex);

                    // If the second sector doesn't match the first, copy the triangle to the second sector.
                    Vector3 secondSector = VectorToSector(secondVertex);
                    if (secondSector != firstSector)
                    {
                        AddTriangleToSector(secondSector, firstVertex, secondVertex, thirdVertex);
                    }

                    // If the third sector matches neither the first nor second sector, copy the triangle to the
                    // third sector.
                    Vector3 thirdSector = VectorToSector(thirdVertex);
                    if (thirdSector != firstSector && thirdSector != secondSector)
                    {
                        AddTriangleToSector(thirdSector, firstVertex, secondVertex, thirdVertex);
                    }

                    // Limit our run time so that we don't cause too many frame drops.
                    // Only checking every few iterations or so to prevent losing too much time to checking the clock.
                    if ((index % 30 == 0) && ((Time.realtimeSinceStartup - startTime) > MaxFrameTimeInSeconds))
                    {
                        //  Debug.LogFormat("{0} of {1} processed", index, meshIndices.Length);
                        stopwatch.Stop();
                        yield return null;
                        stopwatch.Start();
                        startTime = Time.realtimeSinceStartup;
                    }
                }

                startTime = Time.realtimeSinceStartup;

                // Now we have all of our triangles assigned to the correct mesh, we can make all of the meshes.
                // Each sector will have its own mesh.
                foreach (MeshData meshData in meshSectors.Values)
                {
                    // Construct the mesh.
                    meshData.Commit();

                    // Make sure we don't build too many meshes in a single frame.
                    if ((Time.realtimeSinceStartup - startTime) > MaxFrameTimeInSeconds)
                    {
                        stopwatch.Stop();
                        yield return null;
                        stopwatch.Start();
                        startTime = Time.realtimeSinceStartup;
                    }
                }
            }

            // Wait a frame
            stopwatch.Stop();
            yield return null;
            stopwatch.Start();

            // All done - can free up marshal pinned memory
            dll.UnpinAllObjects();

            // Done
            IsImportActive = false;

            // Mark the timestamp
            timeLastImportedMesh = Time.time;

            stopwatch.Stop();
            int deltaFrameCount = (Time.frameCount - startFrameCount + 1);

            if (stopwatch.Elapsed.TotalSeconds > 0.75)
            {
                Debug.LogWarningFormat("Import_UnderstandingMesh took {0:N0} frames ({1:N3} ms)",
                    deltaFrameCount,
                    stopwatch.Elapsed.TotalMilliseconds
                    );
            }
        }

        /// <summary>
        /// Basically floors the Vector so we can use it to subdivide our spatial understanding mesh into parts based
        /// on their position in the world.
        /// </summary>
        /// <param name="vector">The vector to floor.</param>
        /// <returns>A floored vector</returns>
        private Vector3 VectorToSector(Vector3 vector)
        {
            return new Vector3(Mathf.FloorToInt(vector.x), Mathf.FloorToInt(vector.y), Mathf.FloorToInt(vector.z));
        }

        /// <summary>
        /// Updates the mesh import process. This function will kick off the import
        /// coroutine at the requested internal.
        /// </summary>
        private void Update_MeshImport()
        {
            // Only update every so often
            if (IsImportActive || (ImportMeshPeriod <= 0.0f) ||
                ((Time.time - timeLastImportedMesh) < ImportMeshPeriod) ||
                (spatialUnderstanding.ScanState != SpatialUnderstanding.ScanStates.Scanning))
            {
                return;
            }

            StartCoroutine(Import_UnderstandingMesh());
        }

        private void OnDestroy()
        {
            Cleanup();
        }
    }
}