- Implement dirtying instanced rprims on display layer changes

- For instanced meshes, fetch display layer status of every instance and handle its visibility attribute
Autodesk · Oct 27, 2022 · 736720e · 736720e
1 parent 9005e26
commit 736720e
Show file tree

Hide file tree

Showing 5 changed files with 168 additions and 71 deletions.
diff --git a/lib/mayaUsd/render/vp2RenderDelegate/mayaPrimCommon.cpp b/lib/mayaUsd/render/vp2RenderDelegate/mayaPrimCommon.cpp
@@ -687,13 +687,74 @@ void MayaUsdRPrim::_UpdatePrimvarSourcesGeneric(
     }
 }
 
-void MayaUsdRPrim::_SyncDisplayLayerModes(const HdRprim&
 #ifdef MAYA_HAS_DISPLAY_LAYER_API
-                                              refThis
+void MayaUsdRPrim::_PopulateDisplayLayerModes(
+    const MString&     pathString,
+    DisplayLayerModes& displayLayerModes)
+{
+    // Fetch display layers for the item and all its ancestors
+    MObjectArray ancestorDisplayLayers;
+    {
+        MFnDisplayLayerManager displayLayerManager(
+            MFnDisplayLayerManager::currentDisplayLayerManager());
+
+        // Function getAncestorLayersInclusive is not multithreadable because of the
+        // use of Ufe::Path inside, so we use a mutex here
+        std::lock_guard<std::mutex> mutexGuard(sUfePathsMutex);
+        ancestorDisplayLayers = displayLayerManager.getAncestorLayersInclusive(pathString);
+    }
+
+    // Now populate displayLayerModes
+    displayLayerModes = DisplayLayerModes();
+    for (unsigned int i = 0; i < ancestorDisplayLayers.length(); i++) {
+        MFnDependencyNode displayLayerNodeFn(ancestorDisplayLayers[i]);
+        MPlug             layerEnabled = displayLayerNodeFn.findPlug("enabled");
+        if (!layerEnabled.asBool()) {
+            continue;
+        }
+
+        MPlug layerVisible = displayLayerNodeFn.findPlug("visibility");
+        MPlug layerHidesOnPlayback = displayLayerNodeFn.findPlug("hideOnPlayback");
+        MPlug layerDisplayType = displayLayerNodeFn.findPlug("displayType");
+        MPlug levelOfDetail = displayLayerNodeFn.findPlug("levelOfDetail");
+        MPlug shading = displayLayerNodeFn.findPlug("shading");
+        MPlug texturing = displayLayerNodeFn.findPlug("texturing");
+        MPlug colorIndex = displayLayerNodeFn.findPlug("color");
+        MPlug useRGBColors = displayLayerNodeFn.findPlug("overrideRGBColors");
+        MPlug colorRGB = displayLayerNodeFn.findPlug("overrideColorRGB");
+        MPlug colorA = displayLayerNodeFn.findPlug("overrideColorA");
+
+        displayLayerModes._visibility &= layerVisible.asBool();
+        displayLayerModes._hideOnPlayback |= layerHidesOnPlayback.asBool();
+        displayLayerModes._texturing = texturing.asBool();
+        if (levelOfDetail.asShort() != 0) {
+            displayLayerModes._reprOverride = kBBox;
+        } else if (shading.asShort() == 0 && displayLayerModes._reprOverride != kBBox) {
+            displayLayerModes._reprOverride = kWire;
+        }
+        if (displayLayerModes._displayType == kNormal) {
+            displayLayerModes._displayType = (DisplayType)layerDisplayType.asShort();
+        }
+
+        if (useRGBColors.asBool()) {
+            const float3& rgbColor = colorRGB.asMDataHandle().asFloat3();
+            displayLayerModes._wireframeColorIndex = -1;
+            displayLayerModes._wireframeColorRGBA
+                = MColor(rgbColor[0], rgbColor[1], rgbColor[2], colorA.asFloat());
+        } else {
+            displayLayerModes._wireframeColorIndex = colorIndex.asInt();
+        }
+    }
+}
+#else
+void MayaUsdRPrim::_PopulateDisplayLayerModes(const MString&, DisplayLayerModes& displayLayerModes)
+{
+    displayLayerModes = DisplayLayerModes();
+}
 #endif
-)
+
+void MayaUsdRPrim::_SyncDisplayLayerModes(const HdRprim& refThis)
 {
-#ifdef MAYA_HAS_DISPLAY_LAYER_API
     auto* const          param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
     ProxyRenderDelegate& drawScene = param->GetDrawScene();
 
@@ -703,66 +764,16 @@ void MayaUsdRPrim::_SyncDisplayLayerModes(const HdRprim&
     }
 
     _displayLayerModesFrame = drawScene.GetFrameCounter();
-    _displayLayerModes = DisplayLayerModes();
 
-    // Display layer features are currently implemented only for non-instanced geometry
+    // If the prim is not instanced, populate _displayLayerModes.
+    // Otherwise display layer modes will be handled later, in the instancing loop.
     if (refThis.GetInstancerId().IsEmpty()) {
-        MFnDisplayLayerManager displayLayerManager(
-            MFnDisplayLayerManager::currentDisplayLayerManager());
-        MStatus status;
         MString pathString = drawScene.GetProxyShapeDagPath().fullPathName()
             + Ufe::PathString::pathSegmentSeparator().c_str() + _PrimSegmentString[0];
-
-        MObjectArray ancestorDisplayLayers;
-        {
-            // Function getAncestorLayersInclusive is not multithreadable because of the
-            // use of Ufe::Path inside, so we use a mutex here
-            std::lock_guard<std::mutex> mutexGuard(sUfePathsMutex);
-            ancestorDisplayLayers
-                = displayLayerManager.getAncestorLayersInclusive(pathString, &status);
-        }
-
-        for (unsigned int i = 0; i < ancestorDisplayLayers.length(); i++) {
-            MFnDependencyNode displayLayerNodeFn(ancestorDisplayLayers[i]);
-            MPlug             layerEnabled = displayLayerNodeFn.findPlug("enabled");
-            if (!layerEnabled.asBool()) {
-                continue;
-            }
-
-            MPlug layerVisible = displayLayerNodeFn.findPlug("visibility");
-            MPlug layerHidesOnPlayback = displayLayerNodeFn.findPlug("hideOnPlayback");
-            MPlug layerDisplayType = displayLayerNodeFn.findPlug("displayType");
-            MPlug levelOfDetail = displayLayerNodeFn.findPlug("levelOfDetail");
-            MPlug shading = displayLayerNodeFn.findPlug("shading");
-            MPlug texturing = displayLayerNodeFn.findPlug("texturing");
-            MPlug colorIndex = displayLayerNodeFn.findPlug("color");
-            MPlug useRGBColors = displayLayerNodeFn.findPlug("overrideRGBColors");
-            MPlug colorRGB = displayLayerNodeFn.findPlug("overrideColorRGB");
-            MPlug colorA = displayLayerNodeFn.findPlug("overrideColorA");
-
-            _displayLayerModes._visibility &= layerVisible.asBool();
-            _displayLayerModes._hideOnPlayback |= layerHidesOnPlayback.asBool();
-            _displayLayerModes._texturing = texturing.asBool();
-            if (levelOfDetail.asShort() != 0) {
-                _displayLayerModes._reprOverride = kBBox;
-            } else if (shading.asShort() == 0 && _displayLayerModes._reprOverride != kBBox) {
-                _displayLayerModes._reprOverride = kWire;
-            }
-            if (_displayLayerModes._displayType == kNormal) {
-                _displayLayerModes._displayType = (DisplayType)layerDisplayType.asShort();
-            }
-
-            if (useRGBColors.asBool()) {
-                const float3& rgbColor = colorRGB.asMDataHandle().asFloat3();
-                _displayLayerModes._wireframeColorIndex = -1;
-                _displayLayerModes._wireframeColorRGBA
-                    = MColor(rgbColor[0], rgbColor[1], rgbColor[2], colorA.asFloat());
-            } else {
-                _displayLayerModes._wireframeColorIndex = colorIndex.asInt();
-            }
-        }
+        _PopulateDisplayLayerModes(pathString, _displayLayerModes);
+    } else {
+        _displayLayerModes = DisplayLayerModes();
     }
-#endif
 }
 
 void MayaUsdRPrim::_SyncSharedData(
@@ -808,6 +819,21 @@ void MayaUsdRPrim::_SyncSharedData(
         }
     }
 
+    // If instancer is dirty, update instancing map
+    if (HdChangeTracker::IsInstancerDirty(*dirtyBits, id)) {
+        bool instanced = !refThis.GetInstancerId().IsEmpty();
+
+        // UpdateInstancingMapEntry is not multithread-safe, so enqueue the call
+        _delegate->GetVP2ResourceRegistry().EnqueueCommit([this, id, instanced]() {
+            auto* const param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
+            auto&       drawScene = param->GetDrawScene();
+
+            SdfPath newPathInPrototype = instanced ? drawScene.GetPathInPrototype(id) : SdfPath();
+            drawScene.UpdateInstancingMapEntry(_pathInPrototype, newPathInPrototype, id);
+            _pathInPrototype = newPathInPrototype;
+        });
+    }
+
 #if PXR_VERSION > 2111
     // Hydra now manages and caches render tags under the hood and is clearing
     // the dirty bit prior to calling sync. Unconditionally set the render tag

diff --git a/lib/mayaUsd/render/vp2RenderDelegate/mayaPrimCommon.h b/lib/mayaUsd/render/vp2RenderDelegate/mayaPrimCommon.h
@@ -213,6 +213,9 @@ class MayaUsdRPrim
         MColor _wireframeColorRGBA;
     };
 
+    static void
+    _PopulateDisplayLayerModes(const MString& pathString, DisplayLayerModes& displayLayerModes);
+
     void _CommitMVertexBuffer(MHWRender::MVertexBuffer* const, void*) const;
 
     void _UpdateTransform(
@@ -332,6 +335,9 @@ class MayaUsdRPrim
 
     //! The string representation of the runtime only path to this object
     MStringArray _PrimSegmentString;
+
+    //! For instanced prim, holds the corresponding path in USD prototype
+    SdfPath _pathInPrototype;
 };
 
 PXR_NAMESPACE_CLOSE_SCOPE

diff --git a/lib/mayaUsd/render/vp2RenderDelegate/mesh.cpp b/lib/mayaUsd/render/vp2RenderDelegate/mesh.cpp
@@ -45,6 +45,7 @@
 #include <maya/MMatrix.h>
 #include <maya/MProfiler.h>
 #include <maya/MSelectionMask.h>
+#include <ufe/pathString.h>
 
 #include <numeric>
 #include <type_traits>
@@ -1860,12 +1861,24 @@ void HdVP2Mesh::_UpdateDrawItem(
             }
 #endif
 
+            DisplayLayerModes displayLayerModes;
+            const MString     ufePathPrefix = drawScene.GetProxyShapeDagPath().fullPathName()
+                + Ufe::PathString::pathSegmentSeparator().c_str();
+
             stateToCommit._instanceTransforms = std::make_shared<MMatrixArray>();
             stateToCommit._instanceColors = std::make_shared<MFloatArray>();
             for (unsigned int usdInstanceId = 0; usdInstanceId < instanceCount; usdInstanceId++) {
                 unsigned char info = instanceInfo[usdInstanceId];
                 if (info == invalid)
                     continue;
+
+                // Check display layer modes of this instance
+                const MString instancePath = ufePathPrefix
+                    + drawScene.GetScenePrimPath(id, usdInstanceId).GetString().c_str();
+                _PopulateDisplayLayerModes(instancePath, displayLayerModes);
+                if (!displayLayerModes._visibility)
+                    continue;
+
 #ifndef MAYA_UPDATE_UFE_IDENTIFIER_SUPPORT
                 stateToCommit._ufeIdentifiers.append(
                     drawScene.GetScenePrimPath(GetId(), usdInstanceId).GetString().c_str());

diff --git a/lib/mayaUsd/render/vp2RenderDelegate/proxyRenderDelegate.cpp b/lib/mayaUsd/render/vp2RenderDelegate/proxyRenderDelegate.cpp
@@ -825,6 +825,38 @@ void ProxyRenderDelegate::_UpdateSceneDelegate()
     }
 }
 
+SdfPath ProxyRenderDelegate::GetPathInPrototype(const SdfPath& id)
+{
+    auto usdInstancePath = GetScenePrimPath(id, 0);
+    auto usdInstancePrim = _proxyShapeData->UsdStage()->GetPrimAtPath(usdInstancePath);
+    return usdInstancePrim.GetPrimInPrototype().GetPath();
+}
+
+void ProxyRenderDelegate::UpdateInstancingMapEntry(
+    const SdfPath& oldPathInPrototype,
+    const SdfPath& newPathInPrototype,
+    const SdfPath& rprimId)
+{
+    if (oldPathInPrototype != newPathInPrototype) {
+        // remove the old entry from the map
+        if (!oldPathInPrototype.IsEmpty()) {
+            auto range = _instancingMap.equal_range(oldPathInPrototype);
+            auto it = std::find(
+                range.first,
+                range.second,
+                std::pair<const SdfPath, SdfPath>(oldPathInPrototype, rprimId));
+            if (it != range.second) {
+                _instancingMap.erase(it);
+            }
+        }
+
+        // add new entry to the map
+        if (!newPathInPrototype.IsEmpty()) {
+            _instancingMap.insert(std::make_pair(newPathInPrototype, rprimId));
+        }
+    }
+}
+
 #ifdef MAYA_HAS_DISPLAY_LAYER_API
 void ProxyRenderDelegate::_DirtyUsdSubtree(const UsdPrim& prim)
 {
@@ -836,21 +868,28 @@ void ProxyRenderDelegate::_DirtyUsdSubtree(const UsdPrim& prim)
         | HdChangeTracker::DirtyDisplayStyle | MayaUsdRPrim::DirtySelectionHighlight
         | HdChangeTracker::DirtyMaterialId;
 
-    if (prim.IsA<UsdGeomGprim>() && prim.IsActive()) {
-        auto indexPath = _sceneDelegate->ConvertCachePathToIndexPath(prim.GetPath());
-        if (_renderIndex->HasRprim(indexPath)) {
-            changeTracker.MarkRprimDirty(indexPath, dirtyBits);
+    auto markRprimDirty = [this, &changeTracker, dirtyBits](const UsdPrim& prim) {
+        if (prim.IsA<UsdGeomGprim>()) {
+            if (prim.IsInstanceProxy()) {
+                auto range = _instancingMap.equal_range(prim.GetPrimInPrototype().GetPath());
+                for (auto it = range.first; it != range.second; ++it) {
+                    if (_renderIndex->HasRprim(it->second)) {
+                        changeTracker.MarkRprimDirty(it->second, dirtyBits);
+                    }
+                }
+            } else {
+                auto indexPath = _sceneDelegate->ConvertCachePathToIndexPath(prim.GetPath());
+                if (_renderIndex->HasRprim(indexPath)) {
+                    changeTracker.MarkRprimDirty(indexPath, dirtyBits);
+                }
+            }
         }
-    }
+    };
 
-    UsdPrimSubtreeRange range = prim.GetDescendants();
+    markRprimDirty(prim);
+    auto range = prim.GetFilteredDescendants(UsdTraverseInstanceProxies());
     for (auto iter = range.begin(); iter != range.end(); ++iter) {
-        if (iter->IsA<UsdGeomGprim>()) {
-            auto indexPath = _sceneDelegate->ConvertCachePathToIndexPath(iter->GetPath());
-            if (_renderIndex->HasRprim(indexPath)) {
-                changeTracker.MarkRprimDirty(indexPath, dirtyBits);
-            }
-        }
+        markRprimDirty(iter->GetPrim());
     }
 }
 

diff --git a/lib/mayaUsd/render/vp2RenderDelegate/proxyRenderDelegate.h b/lib/mayaUsd/render/vp2RenderDelegate/proxyRenderDelegate.h
@@ -234,6 +234,16 @@ class ProxyRenderDelegate
     MAYAUSD_CORE_PUBLIC
     MDagPath GetProxyShapeDagPath() const;
 
+    // Takes in a path to instanced rprim and returns a path to the correspoding UsdPrim
+    MAYAUSD_CORE_PUBLIC
+    SdfPath GetPathInPrototype(const SdfPath& id);
+
+    MAYAUSD_CORE_PUBLIC
+    void UpdateInstancingMapEntry(
+        const SdfPath& oldPathInPrototype,
+        const SdfPath& newPathInPrototype,
+        const SdfPath& rprimId);
+
 #ifdef MAYA_NEW_POINT_SNAPPING_SUPPORT
     MAYAUSD_CORE_PUBLIC
     bool SnapToSelectedObjects() const;
@@ -339,6 +349,9 @@ class ProxyRenderDelegate
     std::map<TfToken, uint64_t>         _combinedDisplayStyles;
     bool                                _needTexturedMaterials = false;
 
+    // maps from a path in USD prototype to the corresponding rprim paths
+    std::multimap<SdfPath, SdfPath> _instancingMap;
+
     bool _isPopulated {
         false
     }; //!< If false, scene delegate wasn't populated yet within render index