Add a way to query the transform matrix of a surface texture from the driver (#8489)

* Add a way to query the transform matrix of a surface texture from the driver

BUGS=[399959254]

Author: ajmalk

filament/backend/include/backend/platforms/OpenGLPlatform.h

@@ -27,6 +27,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <math/mat3.h>
 
 namespace filament::backend {
 
@@ -319,6 +320,13 @@ class OpenGLPlatform : public Platform {
     virtual void updateTexImage(Stream* UTILS_NONNULL stream,
             int64_t* UTILS_NONNULL timestamp) noexcept;
 
+    /**
+     * Returns the transform matrix of the texture attached to the stream.
+     * @param stream Stream to get the transform matrix from
+     * @param uvTransform Output parameter: Transform matrix of the image bound to the texture. Returns identity if not supported.
+     */
+    virtual math::mat3f getTransformMatrix(Stream* UTILS_NONNULL stream) noexcept;
+
 
     // --------------------------------------------------------------------------------------------
     // External Image support

filament/backend/include/backend/platforms/PlatformEGLAndroid.h

@@ -86,6 +86,7 @@ class PlatformEGLAndroid : public PlatformEGL {
     void attach(Stream* stream, intptr_t tname) noexcept override;
     void detach(Stream* stream) noexcept override;
     void updateTexImage(Stream* stream, int64_t* timestamp) noexcept override;
+    math::mat3f getTransformMatrix(Stream* stream) noexcept override;
 
     /**
      * Converts a AHardwareBuffer to EGLImage

filament/backend/src/opengl/OpenGLDriver.cpp

@@ -58,6 +58,7 @@
 
 #include <math/vec2.h>
 #include <math/vec3.h>
+#include <math/mat3.h>
 
 #include <algorithm>
 #include <chrono>
@@ -2163,6 +2164,14 @@ int64_t OpenGLDriver::getStreamTimestamp(Handle<HwStream> sh) {
     return 0;
 }
 
+math::mat3f OpenGLDriver::getStreamTransformMatrix(Handle<HwStream> sh) {
+    if (sh) {
+        GLStream* s = handle_cast<GLStream*>(sh);        
+        return mPlatform.getTransformMatrix(s->stream);
+    }
+    return math::mat3f();
+}
+
 void OpenGLDriver::destroyFence(Handle<HwFence> fh) {
     if (fh) {
         GLFence* f = handle_cast<GLFence*>(fh);

filament/backend/src/opengl/OpenGLDriver.h

@@ -367,6 +367,7 @@ class OpenGLDriver final : public OpenGLDriverBase {
     void attachStream(GLTexture* t, GLStream* stream) noexcept;
     void detachStream(GLTexture* t) noexcept;
     void replaceStream(GLTexture* t, GLStream* stream) noexcept;
+    math::mat3f getStreamTransformMatrix(Handle<HwStream> sh);
 
 #ifndef FILAMENT_SILENCE_NOT_SUPPORTED_BY_ES2
     // tasks executed on the main thread after the fence signaled

filament/backend/src/opengl/OpenGLPlatform.cpp

@@ -141,6 +141,10 @@ void OpenGLPlatform::updateTexImage(
         UTILS_UNUSED int64_t* timestamp) noexcept {
 }
 
+math::mat3f OpenGLPlatform::getTransformMatrix(
+    UTILS_UNUSED Stream* stream) noexcept {
+return math::mat3f();
+}
 
 OpenGLPlatform::ExternalTexture* OpenGLPlatform::createExternalImageTexture() noexcept {
     return nullptr;

filament/backend/src/opengl/platforms/ExternalStreamManagerAndroid.cpp

@@ -38,6 +38,7 @@ typedef struct ASurfaceTexture ASurfaceTexture;
 #include <stdint.h>
 
 #include <new>
+#include <math/mat4.h>
 
 using namespace utils;
 
@@ -73,6 +74,7 @@ JNIEnv* ExternalStreamManagerAndroid::getEnvironmentSlow() noexcept {
     mSurfaceTextureClass_attachToGLContext = env->GetMethodID(SurfaceTextureClass, "attachToGLContext", "(I)V");
     mSurfaceTextureClass_detachFromGLContext = env->GetMethodID(SurfaceTextureClass, "detachFromGLContext", "()V");
     mSurfaceTextureClass_getTimestamp = env->GetMethodID(SurfaceTextureClass, "getTimestamp", "()J");
+    mSurfaceTextureClass_getTransformMatrix = env->GetMethodID(SurfaceTextureClass, "getTransformMatrix", "([F)V");
     return env;
 }
 
@@ -167,4 +169,25 @@ void ExternalStreamManagerAndroid::updateTexImage(Stream* handle, int64_t* times
     }
 }
 
+math::mat3f ExternalStreamManagerAndroid::getTransformMatrix(Stream* handle) noexcept {
+    EGLStream const* stream = static_cast<EGLStream*>(handle);
+    math::mat4f out = math::mat4f();
+    if (__builtin_available(android 28, *)) {
+        ASurfaceTexture_getTransformMatrix(stream->nSurfaceTexture, &out[0][0]);
+    } else {
+        JNIEnv* const env = getEnvironment();
+        assert_invariant(env); // we should have called attach() by now
+        auto jout = env->NewFloatArray(16);
+        env->CallVoidMethod(stream->jSurfaceTexture, mSurfaceTextureClass_getTransformMatrix, jout);
+        env->GetFloatArrayRegion(jout, 0, 16, &out[0][0]);
+        env->DeleteLocalRef(jout);
+        VirtualMachineEnv::handleException(env);
+    }
+    return math::mat3f {
+        out[0][0], out[0][1], out[0][3],
+        out[1][0], out[1][1], out[1][3],
+        out[3][0], out[3][1], out[3][3],
+    };
+}
+
 } // namespace filament::backend

filament/backend/src/opengl/platforms/ExternalStreamManagerAndroid.h

@@ -33,6 +33,7 @@ typedef struct ASurfaceTexture ASurfaceTexture;
 #include <jni.h>
 
 #include <stdint.h>
+#include <math/mat3.h>
 
 namespace filament::backend {
 
@@ -63,6 +64,9 @@ class ExternalStreamManagerAndroid {
     // must be called on GLES context thread, updates the stream content
     void updateTexImage(Stream* stream, int64_t* timestamp) noexcept;
 
+    // must be called on GLES context thread, returns the transform matrix
+    math::mat3f getTransformMatrix(Stream* stream) noexcept;
+
 private:
     ExternalStreamManagerAndroid() noexcept;
     ~ExternalStreamManagerAndroid() noexcept;
@@ -88,6 +92,7 @@ class ExternalStreamManagerAndroid {
 
     jmethodID mSurfaceTextureClass_updateTexImage{};
     jmethodID mSurfaceTextureClass_getTimestamp{};
+    jmethodID mSurfaceTextureClass_getTransformMatrix{};
     jmethodID mSurfaceTextureClass_attachToGLContext{};
     jmethodID mSurfaceTextureClass_detachFromGLContext{};
 };

filament/backend/src/opengl/platforms/PlatformEGLAndroid.cpp

@@ -283,6 +283,10 @@ void PlatformEGLAndroid::updateTexImage(Stream* stream, int64_t* timestamp) noex
     mExternalStreamManager.updateTexImage(stream, timestamp);
 }
 
+math::mat3f PlatformEGLAndroid::getTransformMatrix(Stream* stream) noexcept {
+    return mExternalStreamManager.getTransformMatrix(stream);
+}
+
 int PlatformEGLAndroid::getOSVersion() const noexcept {
     return mOSVersion;
 }