Improve external texture image API

The current API used a void* in Texture::setExternalImage to pass
the platform specific image. This was a problem because on some
platform like Android, there could be several types (EGLImageKHR and
AHardwareBuffer), and with the void* there was no way to distinguish
them on the backend / platform side.

We now have a new Texture API which takes a
Platform::ExternalImageHandle, which is a reference-counted handle
to a Platform::ExternalImage. Platform::ExternalImage is an opaque
type only known by the platform.

This type must be created using a concrete Platform specific API, such
as:

PlatformEGL::createExternalImage
PlatformEGLAndroid::createExternalImage
PlatformCocoaGL::createExternalImage
PlatformCocoaTouchGL::createExternalImage

New types can easily be added in the future.

There is no "destroy" call because the Platform::ExternalImageHandle is
reference-counted and the client can let go of it as soon as it
doesn't need it any longer (typically after calling
Texture::setExternalImage). Platform::ExternalImage is a small
structure that just anonymously wraps the concrete types.

New internal driver APIs have been added to handle this functionality.

NEW_RELEASE_NOTES.md

@@ -9,3 +9,4 @@ appropriate header in [RELEASE_NOTES.md](./RELEASE_NOTES.md).
 ## Release notes for next branch cut
 
 - matdbg: Add support for debugging ESSL 1.0 shaders
+- backend: New platform API to better handle external textures [⚠️ **New Material Version**]

filament/backend/include/backend/Platform.h

@@ -25,6 +25,8 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include <atomic>
+
 namespace filament::backend {
 
 class Driver;
@@ -41,6 +43,46 @@ class UTILS_PUBLIC Platform {
     struct Fence {};
     struct Stream {};
 
+    class ExternalImageHandle;
+
+    class ExternalImage {
+        friend class ExternalImageHandle;
+        std::atomic_uint32_t mRefCount{0};
+    protected:
+        virtual ~ExternalImage() noexcept;
+    };
+
+    class ExternalImageHandle {
+        ExternalImage* UTILS_NULLABLE mTarget = nullptr;
+        static void incref(ExternalImage* UTILS_NULLABLE p) noexcept;
+        static void decref(ExternalImage* UTILS_NULLABLE p) noexcept;
+
+    public:
+        ExternalImageHandle() noexcept;
+        ~ExternalImageHandle() noexcept;
+        explicit ExternalImageHandle(ExternalImage* UTILS_NULLABLE p) noexcept;
+        ExternalImageHandle(ExternalImageHandle const& rhs) noexcept;
+        ExternalImageHandle(ExternalImageHandle&& rhs) noexcept;
+        ExternalImageHandle& operator=(ExternalImageHandle const& rhs) noexcept;
+        ExternalImageHandle& operator=(ExternalImageHandle&& rhs) noexcept;
+
+        explicit operator bool() const noexcept { return mTarget != nullptr; }
+
+        ExternalImage* UTILS_NULLABLE get() noexcept { return mTarget; }
+        ExternalImage const* UTILS_NULLABLE get() const noexcept { return mTarget; }
+
+        ExternalImage* UTILS_NULLABLE operator->() noexcept { return mTarget; }
+        ExternalImage const* UTILS_NULLABLE operator->() const noexcept { return mTarget; }
+
+        ExternalImage& operator*() noexcept { return *mTarget; }
+        ExternalImage const& operator*() const noexcept { return *mTarget; }
+
+        void clear() noexcept;
+        void reset(ExternalImage* UTILS_NULLABLE p) noexcept;
+    };
+
+    using ExternalImageHandleRef = ExternalImageHandle const&;
+
     /**
      * The type of technique for stereoscopic rendering. (Note that the materials used will need to
      * be compatible with the chosen technique.)

filament/backend/include/backend/platforms/OpenGLPlatform.h

@@ -51,10 +51,9 @@ class OpenGLPlatform : public Platform {
     ~OpenGLPlatform() noexcept override;
 
 public:
-
     struct ExternalTexture {
-        unsigned int target;            // GLenum target
-        unsigned int id;                // GLuint id
+        unsigned int target; // GLenum target
+        unsigned int id; // GLuint id
     };
 
     /**
@@ -324,36 +323,45 @@ class OpenGLPlatform : public Platform {
      * Destroys an external texture handle and associated data.
      * @param texture a pointer to the handle to destroy.
      */
-    virtual void destroyExternalImage(ExternalTexture* UTILS_NONNULL texture) noexcept;
+    virtual void destroyExternalImageTexture(ExternalTexture* UTILS_NONNULL texture) noexcept;
 
     // called on the application thread to allow Filament to take ownership of the image
 
     /**
      * Takes ownership of the externalImage. The externalImage parameter depends on the Platform's
-     * concrete implementation. Ownership is released when destroyExternalImage() is called.
+     * concrete implementation. Ownership is released when destroyExternalImageTexture() is called.
      *
      * WARNING: This is called synchronously from the application thread (NOT the Driver thread)
      *
      * @param externalImage A token representing the platform's external image.
      * @see destroyExternalImage
+     * @{
      */
     virtual void retainExternalImage(void* UTILS_NONNULL externalImage) noexcept;
 
+    virtual void retainExternalImage(ExternalImageHandleRef externalImage) noexcept;
+    /** @}*/
+
     /**
      * Called to bind the platform-specific externalImage to an ExternalTexture.
      * ExternalTexture::id is guaranteed to be bound when this method is called and ExternalTexture
      * is updated with new values for id/target if necessary.
      *
      * WARNING: this method is not allowed to change the bound texture, or must restore the previous
-     * binding upon return. This is to avoid problem with a backend doing state caching.
+     * binding upon return. This is to avoid a problem with a backend doing state caching.
      *
      * @param externalImage The platform-specific external image.
      * @param texture an in/out pointer to ExternalTexture, id and target can be updated if necessary.
      * @return true on success, false on error.
+     * @{
      */
     virtual bool setExternalImage(void* UTILS_NONNULL externalImage,
             ExternalTexture* UTILS_NONNULL texture) noexcept;
 
+    virtual bool setExternalImage(ExternalImageHandleRef externalImage,
+            ExternalTexture* UTILS_NONNULL texture) noexcept;
+    /** @}*/
+
     /**
      * The method allows platforms to convert a user-supplied external image object into a new type
      * (e.g. HardwareBuffer => EGLImage). The default implementation returns source.

filament/backend/include/backend/platforms/PlatformCocoaGL.h

@@ -34,12 +34,14 @@ class PlatformCocoaGL : public OpenGLPlatform {
     PlatformCocoaGL();
     ~PlatformCocoaGL() noexcept override;
 
+    ExternalImageHandle createExternalImage(void* cvPixelBuffer) noexcept;
+
 protected:
     // --------------------------------------------------------------------------------------------
     // Platform Interface
 
     Driver* createDriver(void* sharedContext,
-            const Platform::DriverConfig& driverConfig) noexcept override;
+            const DriverConfig& driverConfig) noexcept override;
 
     // Currently returns 0
     int getOSVersion() const noexcept override;
@@ -59,10 +61,12 @@ class PlatformCocoaGL : public OpenGLPlatform {
     void destroySwapChain(SwapChain* swapChain) noexcept override;
     bool makeCurrent(ContextType type, SwapChain* drawSwapChain, SwapChain* readSwapChain) noexcept override;
     void commit(SwapChain* swapChain) noexcept override;
-    OpenGLPlatform::ExternalTexture* createExternalImageTexture() noexcept override;
-    void destroyExternalImage(ExternalTexture* texture) noexcept override;
+    ExternalTexture* createExternalImageTexture() noexcept override;
+    void destroyExternalImageTexture(ExternalTexture* texture) noexcept override;
     void retainExternalImage(void* externalImage) noexcept override;
     bool setExternalImage(void* externalImage, ExternalTexture* texture) noexcept override;
+    void retainExternalImage(ExternalImageHandleRef externalImage) noexcept override;
+    bool setExternalImage(ExternalImageHandleRef externalImage, ExternalTexture* texture) noexcept override;
 
 private:
     PlatformCocoaGLImpl* pImpl = nullptr;

filament/backend/include/backend/platforms/PlatformCocoaTouchGL.h

@@ -32,11 +32,13 @@ class PlatformCocoaTouchGL : public OpenGLPlatform {
     PlatformCocoaTouchGL();
     ~PlatformCocoaTouchGL() noexcept override;
 
+    ExternalImageHandle createExternalImage(void* cvPixelBuffer) noexcept;
+
     // --------------------------------------------------------------------------------------------
     // Platform Interface
 
     Driver* createDriver(void* sharedGLContext,
-            const Platform::DriverConfig& driverConfig) noexcept override;
+            const DriverConfig& driverConfig) noexcept override;
 
     int getOSVersion() const noexcept final { return 0; }
 
@@ -56,10 +58,12 @@ class PlatformCocoaTouchGL : public OpenGLPlatform {
     bool makeCurrent(ContextType type, SwapChain* drawSwapChain, SwapChain* readSwapChain) noexcept override;
     void commit(SwapChain* swapChain) noexcept override;
 
-    OpenGLPlatform::ExternalTexture* createExternalImageTexture() noexcept override;
-    void destroyExternalImage(ExternalTexture* texture) noexcept override;
+    ExternalTexture* createExternalImageTexture() noexcept override;
+    void destroyExternalImageTexture(ExternalTexture* texture) noexcept override;
     void retainExternalImage(void* externalImage) noexcept override;
     bool setExternalImage(void* externalImage, ExternalTexture* texture) noexcept override;
+    void retainExternalImage(ExternalImageHandleRef externalImage) noexcept override;
+    bool setExternalImage(ExternalImageHandleRef externalImage, ExternalTexture* texture) noexcept override;
 
 private:
     PlatformCocoaTouchGLImpl* pImpl = nullptr;

filament/backend/include/backend/platforms/PlatformEGL.h

@@ -47,6 +47,11 @@ class PlatformEGL : public OpenGLPlatform {
     // Return true if we're on an OpenGL platform (as opposed to OpenGL ES). false by default.
     virtual bool isOpenGL() const noexcept;
 
+    /**
+     * Creates an ExternalImage from a EGLImageKHR
+     */
+    ExternalImageHandle createExternalImage(EGLImageKHR eglImage) noexcept;
+
 protected:
     // --------------------------------------------------------------------------------------------
     // Helper for EGL configs and attributes parameters
@@ -118,9 +123,10 @@ class PlatformEGL : public OpenGLPlatform {
     void destroyFence(Fence* fence) noexcept override;
     FenceStatus waitFence(Fence* fence, uint64_t timeout) noexcept override;
 
-    OpenGLPlatform::ExternalTexture* createExternalImageTexture() noexcept override;
-    void destroyExternalImage(ExternalTexture* texture) noexcept override;
+    ExternalTexture* createExternalImageTexture() noexcept override;
+    void destroyExternalImageTexture(ExternalTexture* texture) noexcept override;
     bool setExternalImage(void* externalImage, ExternalTexture* texture) noexcept override;
+    bool setExternalImage(ExternalImageHandleRef externalImage, ExternalTexture* texture) noexcept override;
 
     /**
      * Logs glGetError() to slog.e
@@ -188,6 +194,12 @@ class PlatformEGL : public OpenGLPlatform {
 
     void initializeGlExtensions() noexcept;
 
+    struct ExternalImageEGL : public ExternalImage {
+        EGLImageKHR eglImage = EGL_NO_IMAGE;
+    protected:
+        ~ExternalImageEGL() override;
+    };
+
 protected:
     EGLConfig findSwapChainConfig(uint64_t flags, bool window, bool pbuffer) const;
 

filament/backend/include/backend/platforms/PlatformEGLAndroid.h

@@ -57,6 +57,11 @@ class PlatformEGLAndroid : public PlatformEGL {
     Driver* createDriver(void* sharedContext,
             const Platform::DriverConfig& driverConfig) noexcept override;
 
+    /**
+     * Creates an ExternalImage from a EGLImageKHR
+     */
+    ExternalImageHandle createExternalImage(AHardwareBuffer const *buffer, bool sRGB) noexcept;
+
     // --------------------------------------------------------------------------------------------
     // OpenGLPlatform Interface
 
@@ -89,6 +94,15 @@ class PlatformEGLAndroid : public PlatformEGL {
      */
     AcquiredImage transformAcquiredImage(AcquiredImage source) noexcept override;
 
+    bool setExternalImage(ExternalImageHandleRef externalImage, ExternalTexture* texture) noexcept override;
+
+    struct ExternalImageEGLAndroid : public ExternalImageEGL {
+        AHardwareBuffer* aHardwareBuffer = nullptr;
+        bool sRGB = false;
+    protected:
+        ~ExternalImageEGLAndroid() override;
+    };
+
 protected:
     bool makeCurrent(ContextType type,
             SwapChain* drawSwapChain,

filament/backend/include/private/backend/DriverAPI.inc

@@ -215,6 +215,14 @@ DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureViewSwizzle,
         backend::TextureSwizzle, b,
         backend::TextureSwizzle, a)
 
+DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureExternalImage2,
+        backend::SamplerType, target,
+        backend::TextureFormat, format,
+        uint32_t, width,
+        uint32_t, height,
+        backend::TextureUsage, usage,
+        backend::Platform::ExternalImageHandleRef, image)
+
 DECL_DRIVER_API_R_N(backend::TextureHandle, createTextureExternalImage,
         backend::SamplerType, target,
         backend::TextureFormat, format,
@@ -355,6 +363,7 @@ DECL_DRIVER_API_SYNCHRONOUS_0(bool, isDepthClampSupported)
 DECL_DRIVER_API_SYNCHRONOUS_0(uint8_t, getMaxDrawBuffers)
 DECL_DRIVER_API_SYNCHRONOUS_0(size_t, getMaxUniformBufferSize)
 DECL_DRIVER_API_SYNCHRONOUS_0(math::float2, getClipSpaceParams)
+DECL_DRIVER_API_SYNCHRONOUS_N(void, setupExternalImage2, backend::Platform::ExternalImageHandleRef, image)
 DECL_DRIVER_API_SYNCHRONOUS_N(void, setupExternalImage, void*, image)
 DECL_DRIVER_API_SYNCHRONOUS_N(backend::TimerQueryResult, getTimerQueryValue, backend::TimerQueryHandle, query, uint64_t*, elapsedTime)
 DECL_DRIVER_API_SYNCHRONOUS_N(bool, isWorkaroundNeeded, backend::Workaround, workaround)

filament/backend/src/Platform.cpp

@@ -16,8 +16,93 @@
 
 #include <backend/Platform.h>
 
+#include <utils/compiler.h>
+
+#include <atomic>
+#include <utility>
+
+#include <stddef.h>
+#include <stdint.h>
+
 namespace filament::backend {
 
+void Platform::ExternalImageHandle::incref(ExternalImage* p) noexcept {
+    if (p) {
+        // incrementing the ref-count doesn't acquire or release anything
+        p->mRefCount.fetch_add(1, std::memory_order_relaxed);
+    }
+}
+
+void Platform::ExternalImageHandle::decref(ExternalImage* p) noexcept {
+    if (p) {
+        // When decrementing the ref-count, unless it reaches zero, there is no need to acquire data; we need to
+        // release all previous writes though so they can be visible to the thread that will actually delete the
+        // object.
+        if (p->mRefCount.fetch_sub(1, std::memory_order_release) == 1) {
+            // if we reach zero, we're about to delete the object, we need to acquire all previous writes from other
+            // threads (i.e.: the memory from other threads prior to the decref() need to be visible now.
+            std::atomic_thread_fence(std::memory_order_acquire);
+            delete p;
+        }
+    }
+}
+
+Platform::ExternalImageHandle::ExternalImageHandle() noexcept = default;
+
+Platform::ExternalImageHandle::~ExternalImageHandle() noexcept {
+    decref(mTarget);
+}
+
+Platform::ExternalImageHandle::ExternalImageHandle(ExternalImage* p) noexcept
+        : mTarget(p) {
+    incref(mTarget);
+}
+
+Platform::ExternalImageHandle::ExternalImageHandle(ExternalImageHandle const& rhs) noexcept
+        : mTarget(rhs.mTarget) {
+    incref(mTarget);
+}
+
+Platform::ExternalImageHandle::ExternalImageHandle(ExternalImageHandle&& rhs) noexcept
+        : mTarget(rhs.mTarget) {
+    rhs.mTarget = nullptr;
+}
+
+Platform::ExternalImageHandle& Platform::ExternalImageHandle::operator=(ExternalImageHandle const& rhs) noexcept {
+    if (UTILS_LIKELY(this != &rhs)) {
+        incref(rhs.mTarget);
+        decref(mTarget);
+        mTarget = rhs.mTarget;
+    }
+    return *this;
+}
+
+Platform::ExternalImageHandle& Platform::ExternalImageHandle::operator=(ExternalImageHandle&& rhs) noexcept {
+    if (UTILS_LIKELY(this != &rhs)) {
+        decref(mTarget);
+        mTarget = rhs.mTarget;
+        rhs.mTarget = nullptr;
+    }
+    return *this;
+}
+
+void Platform::ExternalImageHandle::clear() noexcept {
+    decref(mTarget);
+    mTarget = nullptr;
+}
+
+void Platform::ExternalImageHandle::reset(ExternalImage* p) noexcept {
+    incref(p);
+    decref(mTarget);
+    mTarget = p;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+Platform::ExternalImage::~ExternalImage() noexcept = default;
+
+// --------------------------------------------------------------------------------------------------------------------
+
 Platform::Platform() noexcept = default;
 
 // this generates the vtable in this translation unit