Pixar batch renderer refactoring and performance improvements

lib/mayaUsd/render/px_vp20/utils.cpp

@@ -763,8 +763,8 @@ px_vp20Utils::RenderBoundingBox(
     bboxTransformMatrix.setScale(scales, MSpace::kTransform);
     return RenderWireCubes(
             { GfMatrix4f(bboxTransformMatrix.asMatrix().matrix) },
-            color, 
-            GfMatrix4d(worldViewMat.matrix), 
+            color,
+            GfMatrix4d(worldViewMat.matrix),
             GfMatrix4d(projectionMat.matrix));
 }
 
@@ -875,7 +875,7 @@ void main()
     // Populate the shader variables.
     GfMatrix4f vpMatrix(worldViewMat * projectionMat);
     GLuint vpMatrixLoc = glGetUniformLocation(renderBoundsProgramId, "vpMatrix");
-    glUniformMatrix4fv(vpMatrixLoc, 1, 
+    glUniformMatrix4fv(vpMatrixLoc, 1,
             GL_TRUE, // transpose
             vpMatrix.data());
 
@@ -891,12 +891,12 @@ void main()
     // since we're copying these directly from GfMatrix4f, we need to
     // transpose() them in the shader.
     const GLuint cubeXformLoc = glGetAttribLocation(renderBoundsProgramId, "cubeXformT");
-    glBufferData(GL_ARRAY_BUFFER, 
+    glBufferData(GL_ARRAY_BUFFER,
             sizeof(GfMatrix4f) * numCubes, cubeXforms.data(), GL_DYNAMIC_DRAW);
     for (size_t r = 0; r < 4; r++) {
         GLuint loc = cubeXformLoc + r;
         glEnableVertexAttribArray(loc);
-        glVertexAttribPointer(loc, 4, GL_FLOAT, GL_FALSE, sizeof(GfMatrix4f), 
+        glVertexAttribPointer(loc, 4, GL_FLOAT, GL_FALSE, sizeof(GfMatrix4f),
                 (char*)(sizeof(float)*4*r));
         glVertexAttribDivisor(loc, 1);
     }

lib/mayaUsd/render/px_vp20/utils.h

@@ -25,6 +25,7 @@
 #include <maya/MBoundingBox.h>
 #include <maya/MDrawContext.h>
 #include <maya/MHWGeometryUtilities.h>
+#include <maya/MFrameContext.h>
 #include <maya/MMatrix.h>
 #include <maya/MSelectionContext.h>
 
@@ -64,6 +65,14 @@ class px_vp20Utils
                 const MHWRender::MDrawContext& context,
                 M3dView& view);
 
+        /// Returns true if the given Maya display style indicates that a
+        /// bounding box should be rendered.
+        static bool ShouldRenderBoundingBox(unsigned int displayStyle) {
+            const bool boundingBoxStyle =
+                displayStyle & MHWRender::MFrameContext::DisplayStyle::kBoundingBox;
+            return boundingBoxStyle;
+        }
+
         /// Renders the given bounding box in the given \p color via OpenGL.
         MAYAUSD_CORE_PUBLIC
         static bool RenderBoundingBox(
@@ -74,7 +83,7 @@ class px_vp20Utils
 
         /// Helper to draw multiple wireframe boxes, where \p cubeXforms is a
         /// list of transforms to apply to the unit cube centered around the
-        /// origin.  Those transforms will all be concatenated with the 
+        /// origin.  Those transforms will all be concatenated with the
         /// \p worldViewMat and \p projectionMat.
         MAYAUSD_CORE_PUBLIC
         static bool RenderWireCubes(

lib/mayaUsd/render/px_vp20/utils_legacy.h

@@ -18,9 +18,24 @@
 
 /// \file px_vp20/utils_legacy.h
 
+// XXX: With Maya versions up through 2019 on Linux, M3dView.h ends up
+// indirectly including an X11 header that #define's "Bool" as int:
+//   - <maya/M3dView.h> includes <maya/MNativeWindowHdl.h>
+//   - <maya/MNativeWindowHdl.h> includes <X11/Intrinsic.h>
+//   - <X11/Intrinsic.h> includes <X11/Xlib.h>
+//   - <X11/Xlib.h> does: "#define Bool int"
+// This can cause compilation issues if <pxr/usd/sdf/types.h> is included
+// afterwards, so to fix this, we ensure that it gets included first.
+//
+// The X11 include appears to have been removed in Maya 2020+, so this should
+// no longer be an issue with later versions.
+#include <pxr/usd/sdf/types.h>
+
 #include <pxr/pxr.h>
 #include <pxr/base/gf/matrix4d.h>
 
+#include <maya/M3dView.h>
+#include <maya/MFrameContext.h>
 #include <maya/MSelectInfo.h>
 
 #include <mayaUsd/base/api.h>
@@ -40,6 +55,47 @@ class px_LegacyViewportUtils
                 GfMatrix4d& viewMatrix,
                 GfMatrix4d& projectionMatrix);
 
+        /// Helper function that converts M3dView::DisplayStyle from the legacy
+        /// viewport into MHWRender::MFrameContext::DisplayStyle for Viewport
+        /// 2.0.
+        ///
+        /// In the legacy viewport, the M3dView can be in exactly one
+        /// displayStyle whereas Viewport 2.0's displayStyle is a bitmask of
+        /// potentially multiple styles. To translate from the legacy viewport
+        /// to Viewport 2.0, we simply bitwise-OR the single legacy viewport
+        /// displayStyle into an empty mask.
+        static unsigned int GetMFrameContextDisplayStyle(
+                M3dView::DisplayStyle legacyDisplayStyle) {
+            unsigned int displayStyle = 0u;
+
+            switch (legacyDisplayStyle) {
+                case M3dView::kBoundingBox:
+                    displayStyle |= MHWRender::MFrameContext::DisplayStyle::kBoundingBox;
+                    break;
+                case M3dView::kFlatShaded:
+                    displayStyle |= MHWRender::MFrameContext::DisplayStyle::kFlatShaded;
+                    break;
+                case M3dView::kGouraudShaded:
+                    displayStyle |= MHWRender::MFrameContext::DisplayStyle::kGouraudShaded;
+                    break;
+                case M3dView::kWireFrame:
+                    displayStyle |= MHWRender::MFrameContext::DisplayStyle::kWireFrame;
+                    break;
+                case M3dView::kPoints:
+                    // Not supported.
+                    break;
+            }
+
+            return displayStyle;
+        }
+
+        /// Returns true if the given Maya display style indicates that a
+        /// bounding box should be rendered.
+        static bool ShouldRenderBoundingBox(
+                M3dView::DisplayStyle legacyDisplayStyle) {
+            return (legacyDisplayStyle == M3dView::kBoundingBox);
+        }
+
     private:
         px_LegacyViewportUtils() = delete;
         ~px_LegacyViewportUtils() = delete;