MAYA-127944 - As a user, I'd like to add prototypes to display layers

lib/mayaUsd/render/vp2RenderDelegate/mayaPrimCommon.cpp

@@ -46,6 +46,8 @@ const MString MayaUsdRPrim::kSolidColorStr("solidColor");
 constexpr auto sDrawModeAllButBBox = (MHWRender::MGeometry::DrawMode)(
     MHWRender::MGeometry::kAll & ~MHWRender::MGeometry::kBoundingBox);
 
+static const InstancePrototypePath sVoidInstancePrototypePath { SdfPath(), kNativeInstancing };
+
 #ifdef MAYA_NEW_POINT_SNAPPING_SUPPORT
 
 namespace {
@@ -99,6 +101,7 @@ void MayaUsdCustomData::RemoveInstancePrimPaths(const SdfPath& prim)
 
 MayaUsdRPrim::MayaUsdRPrim(HdVP2RenderDelegate* delegate, const SdfPath& id)
     : _delegate(delegate)
+    , _hydraId(id)
     , _rprimId(id.GetText())
 {
     // Store a string version of the Cache Path to be used to tag MRenderItems. The CachePath is
@@ -109,6 +112,16 @@ MayaUsdRPrim::MayaUsdRPrim(HdVP2RenderDelegate* delegate, const SdfPath& id)
         drawScene.GetScenePrimPath(id, UsdImagingDelegate::ALL_INSTANCES).GetString().c_str());
 }
 
+MayaUsdRPrim::~MayaUsdRPrim()
+{
+    if (!_pathInPrototype.first.IsEmpty()) {
+        // Clear my entry from the instancing map
+        auto* const          param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
+        ProxyRenderDelegate& drawScene = param->GetDrawScene();
+        drawScene.UpdateInstancingMapEntry(_pathInPrototype, sVoidInstancePrototypePath, _hydraId);
+    }
+}
+
 void MayaUsdRPrim::_CommitMVertexBuffer(MHWRender::MVertexBuffer* const buffer, void* bufferData)
     const
 {
@@ -337,11 +350,15 @@ HdReprSharedPtr MayaUsdRPrim::_InitReprCommon(
 
     // display layers handling
     if (!drawScene.GetUsdImagingDelegate()->GetInstancerId(id).IsEmpty()) {
-        _displayLayerModes = DisplayLayerModes();
+        // Sync display layer modes for instanced prims.
+        // This also sets the value of '_useInstancedDisplayLayerModes' that identifies whether
+        // display layer modes will be handled on per-primitive or per-instance basis
+        _SyncDisplayLayerModes(id, true);
 
         // Instanced primitives with instances in display layers use 'forced' representations to
         // draw those specific instances, so the 'forced' representations should be inited alongside
-        if (reprToken != HdVP2ReprTokens->forcedBbox && reprToken != HdVP2ReprTokens->forcedWire
+        if (_useInstancedDisplayLayerModes && reprToken != HdVP2ReprTokens->forcedBbox
+            && reprToken != HdVP2ReprTokens->forcedWire
             && reprToken != HdVP2ReprTokens->forcedUntextured) {
             refThis.InitRepr(
                 drawScene.GetUsdImagingDelegate(), HdVP2ReprTokens->forcedBbox, dirtyBits);
@@ -352,8 +369,7 @@ HdReprSharedPtr MayaUsdRPrim::_InitReprCommon(
         }
     } else {
         // Sync display layer modes for non-instanced prims.
-        // For instanced prims, this will be done inside Sync method on a per-instance basis
-        _SyncDisplayLayerModes(id);
+        _SyncDisplayLayerModes(id, false);
     }
 
     _UpdateReprOverrides(reprs);
@@ -750,7 +766,7 @@ void MayaUsdRPrim::_PopulateDisplayLayerModes(
 }
 #endif
 
-void MayaUsdRPrim::_SyncDisplayLayerModes(SdfPath const& id)
+void MayaUsdRPrim::_SyncDisplayLayerModes(SdfPath const& id, bool instancedPrim)
 {
     auto* const          param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
     ProxyRenderDelegate& drawScene = param->GetDrawScene();
@@ -761,7 +777,21 @@ void MayaUsdRPrim::_SyncDisplayLayerModes(SdfPath const& id)
     }
 
     _displayLayerModesFrame = drawScene.GetFrameCounter();
-    auto usdPath = drawScene.GetScenePrimPath(id, UsdImagingDelegate::ALL_INSTANCES);
+
+    // Obtain scene prim path
+    HdInstancerContext instancerContext;
+    int                instanceIndex = instancedPrim ? 0 : UsdImagingDelegate::ALL_INSTANCES;
+    auto               usdPath = drawScene.GetScenePrimPath(id, instanceIndex, &instancerContext);
+
+    // Native instances use per-instance _displayLayerModes
+    if (instancedPrim && instancerContext.empty()) {
+        _useInstancedDisplayLayerModes = true;
+        _displayLayerModes = DisplayLayerModes();
+        return;
+    }
+
+    // Otherwise, populate display layer modes
+    _useInstancedDisplayLayerModes = false;
     _PopulateDisplayLayerModes(usdPath, _displayLayerModes, drawScene);
 }
 
@@ -779,7 +809,7 @@ void MayaUsdRPrim::_SyncDisplayLayerModesInstanced(SdfPath const& id, unsigned i
 
     _forcedReprFlags = 0;
     _requiredModFlagsBitset.reset();
-    if (drawScene.SupportPerInstanceDisplayLayers(id)) {
+    if (_useInstancedDisplayLayerModes) {
         _displayLayerModesInstanced.resize(instanceCount);
         for (unsigned int usdInstanceId = 0; usdInstanceId < instanceCount; usdInstanceId++) {
             auto  usdPath = drawScene.GetScenePrimPath(id, usdInstanceId);
@@ -860,7 +890,8 @@ void MayaUsdRPrim::_SyncSharedData(
             auto* const param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
             auto&       drawScene = param->GetDrawScene();
 
-            SdfPath newPathInPrototype = instanced ? drawScene.GetPathInPrototype(id) : SdfPath();
+            InstancePrototypePath newPathInPrototype
+                = instanced ? drawScene.GetPathInPrototype(id) : sVoidInstancePrototypePath;
             drawScene.UpdateInstancingMapEntry(_pathInPrototype, newPathInPrototype, id);
             _pathInPrototype = newPathInPrototype;
         });
@@ -929,25 +960,31 @@ bool MayaUsdRPrim::_SyncCommon(
     return true;
 }
 
-MColor MayaUsdRPrim::_GetHighlightColor(const TfToken& className)
+MColor
+MayaUsdRPrim::_GetHighlightColor(const TfToken& className, HdVP2SelectionStatus selectionStatus)
 {
     auto* const          param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
     ProxyRenderDelegate& drawScene = param->GetDrawScene();
 
     if (_displayLayerModes._displayType == MayaUsdRPrim::kTemplate) {
-        return drawScene.GetTemplateColor(_selectionStatus != kUnselected);
+        return drawScene.GetTemplateColor(selectionStatus != kUnselected);
     } else if (
         _displayLayerModes._displayType == MayaUsdRPrim::kReference
-        && _selectionStatus == kUnselected) {
+        && selectionStatus == kUnselected) {
         return drawScene.GetReferenceColor();
     } else {
         return (
-            _selectionStatus != kUnselected ? drawScene.GetSelectionHighlightColor(
-                _selectionStatus == kFullyLead ? TfToken() : className)
-                                            : _GetWireframeColor());
+            selectionStatus != kUnselected ? drawScene.GetSelectionHighlightColor(
+                selectionStatus == kFullyLead ? TfToken() : className)
+                                           : _GetWireframeColor());
     }
 }
 
+MColor MayaUsdRPrim::_GetHighlightColor(const TfToken& className)
+{
+    return _GetHighlightColor(className, _selectionStatus);
+}
+
 MColor MayaUsdRPrim::_GetWireframeColor()
 {
     if (_displayLayerModes._wireframeColorIndex > 0) {

lib/mayaUsd/render/vp2RenderDelegate/mayaPrimCommon.h

@@ -163,7 +163,7 @@ class MayaUsdRPrim
     static const MString kSolidColorStr;
 
     MayaUsdRPrim(HdVP2RenderDelegate* delegate, const SdfPath& id);
-    virtual ~MayaUsdRPrim() = default;
+    virtual ~MayaUsdRPrim();
 
 protected:
     using ReprVector = std::vector<std::pair<TfToken, HdReprSharedPtr>>;
@@ -298,7 +298,7 @@ class MayaUsdRPrim
         ReprVector const&  reprs,
         TfToken const&     renderTag);
 
-    void _SyncDisplayLayerModes(SdfPath const& id);
+    void _SyncDisplayLayerModes(SdfPath const& id, bool instancedPrim);
     void _SyncDisplayLayerModesInstanced(SdfPath const& id, unsigned int instanceCount);
 
     bool _FilterInstanceByDisplayLayer(
@@ -327,6 +327,7 @@ class MayaUsdRPrim
 
     SdfPath _GetUpdatedMaterialId(HdRprim* rprim, HdSceneDelegate* delegate);
     MColor  _GetHighlightColor(const TfToken& className);
+    MColor  _GetHighlightColor(const TfToken& className, HdVP2SelectionStatus selectionStatus);
     MColor  _GetWireframeColor();
 
     void _PropagateDirtyBitsCommon(HdDirtyBits& bits, const ReprVector& reprs) const;
@@ -374,13 +375,17 @@ class MayaUsdRPrim
     //! VP2 render delegate for which this prim was created
     HdVP2RenderDelegate* _delegate { nullptr };
 
+    //! Rprim id in Hydra
+    const SdfPath _hydraId;
+
     //! Rprim id cached as a maya string for easier debugging and profiling
     const MString _rprimId;
 
     //! Selection status of the Rprim
     HdVP2SelectionStatus _selectionStatus { kUnselected };
 
     //! Modes requested by display layer along with the frame they are updated on
+    bool                           _useInstancedDisplayLayerModes { false };
     DisplayLayerModes              _displayLayerModes;
     std::vector<DisplayLayerModes> _displayLayerModesInstanced;
     uint64_t                       _displayLayerModesFrame { 0 };
@@ -403,7 +408,7 @@ class MayaUsdRPrim
     MStringArray _PrimSegmentString;
 
     //! For instanced prim, holds the corresponding path in USD prototype
-    SdfPath _pathInPrototype;
+    InstancePrototypePath _pathInPrototype { SdfPath(), kNativeInstancing };
 };
 
 PXR_NAMESPACE_CLOSE_SCOPE

lib/mayaUsd/render/vp2RenderDelegate/mesh.cpp

@@ -1913,9 +1913,9 @@ void HdVP2Mesh::_UpdateDrawItem(
 
             // Set up the source color buffers.
             const MColor wireframeColors[]
-                = { drawScene.GetWireframeColor(),
-                    drawScene.GetSelectionHighlightColor(HdPrimTypeTokens->mesh),
-                    drawScene.GetSelectionHighlightColor(),
+                = { _GetHighlightColor(HdPrimTypeTokens->mesh, kUnselected),
+                    _GetHighlightColor(HdPrimTypeTokens->mesh, kFullyActive),
+                    _GetHighlightColor(HdPrimTypeTokens->mesh, kFullyLead),
                     drawScene.GetTemplateColor(false),
                     drawScene.GetTemplateColor(true),
                     drawScene.GetReferenceColor() };

lib/mayaUsd/render/vp2RenderDelegate/proxyRenderDelegate.cpp

@@ -891,33 +891,42 @@ void ProxyRenderDelegate::_UpdateSceneDelegate()
     }
 }
 
-SdfPath ProxyRenderDelegate::GetPathInPrototype(const SdfPath& id)
+InstancePrototypePath ProxyRenderDelegate::GetPathInPrototype(const SdfPath& id)
 {
-    auto usdInstancePath = GetScenePrimPath(id, 0);
+    HdInstancerContext instancerContext;
+    auto               usdInstancePath = GetScenePrimPath(id, 0, &instancerContext);
+
+    // In case of point instancer, we already have the path in prototype, return it.
+    if (!instancerContext.empty()) {
+        return InstancePrototypePath(usdInstancePath, kPointInstancing);
+    }
+
+    // In case of a native instance, obtain the path in prototype and return it.
     auto usdInstancePrim = _proxyShapeData->UsdStage()->GetPrimAtPath(usdInstancePath);
-    return usdInstancePrim.GetPrimInPrototype().GetPath();
+    auto usdPrototypePath = usdInstancePrim.GetPrimInPrototype().GetPath();
+    return InstancePrototypePath(usdPrototypePath, kNativeInstancing);
 }
 
 void ProxyRenderDelegate::UpdateInstancingMapEntry(
-    const SdfPath& oldPathInPrototype,
-    const SdfPath& newPathInPrototype,
-    const SdfPath& rprimId)
+    const InstancePrototypePath& oldPathInPrototype,
+    const InstancePrototypePath& newPathInPrototype,
+    const SdfPath&               rprimId)
 {
     if (oldPathInPrototype != newPathInPrototype) {
         // remove the old entry from the map
-        if (!oldPathInPrototype.IsEmpty()) {
+        if (!oldPathInPrototype.first.IsEmpty()) {
             auto range = _instancingMap.equal_range(oldPathInPrototype);
             auto it = std::find(
                 range.first,
                 range.second,
-                std::pair<const SdfPath, SdfPath>(oldPathInPrototype, rprimId));
+                std::pair<const InstancePrototypePath, SdfPath>(oldPathInPrototype, rprimId));
             if (it != range.second) {
                 _instancingMap.erase(it);
             }
         }
 
         // add new entry to the map
-        if (!newPathInPrototype.IsEmpty()) {
+        if (!newPathInPrototype.first.IsEmpty()) {
             _instancingMap.insert(std::make_pair(newPathInPrototype, rprimId));
         }
     }
@@ -937,14 +946,26 @@ void ProxyRenderDelegate::_DirtyUsdSubtree(const UsdPrim& prim)
             | MayaUsdRPrim::DirtySelectionHighlight | HdChangeTracker::DirtyMaterialId;
 
         if (prim.IsA<UsdGeomGprim>()) {
-            if (prim.IsInstanceProxy()) {
-                auto range = _instancingMap.equal_range(prim.GetPrimInPrototype().GetPath());
+            auto range = _instancingMap.equal_range(
+                InstancePrototypePath(prim.GetPath(), kPointInstancing));
+            if (range.first != range.second) {
+                // Point instancing prim
+                for (auto it = range.first; it != range.second; ++it) {
+                    if (_renderIndex->HasRprim(it->second)) {
+                        changeTracker.MarkRprimDirty(it->second, dirtyBits);
+                    }
+                }
+            } else if (prim.IsInstanceProxy()) {
+                // Native instancing prim
+                range = _instancingMap.equal_range(
+                    InstancePrototypePath(prim.GetPrimInPrototype().GetPath(), kNativeInstancing));
                 for (auto it = range.first; it != range.second; ++it) {
                     if (_renderIndex->HasRprim(it->second)) {
                         changeTracker.MarkRprimDirty(it->second, dirtyBits);
                     }
                 }
             } else {
+                // Non-instanced prim
                 auto indexPath = _sceneDelegate->ConvertCachePathToIndexPath(prim.GetPath());
                 if (_renderIndex->HasRprim(indexPath)) {
                     changeTracker.MarkRprimDirty(indexPath, dirtyBits);
@@ -1323,15 +1344,6 @@ SdfPath ProxyRenderDelegate::GetScenePrimPath(const SdfPath& rprimId, int instan
     return usdPath;
 }
 
-bool ProxyRenderDelegate::SupportPerInstanceDisplayLayers(const SdfPath& rprimId) const
-{
-    // For now, per-instance display layers are supported only for native instancing
-    HdInstancerContext instancerContext;
-    GetScenePrimPath(rprimId, 0, &instancerContext);
-    bool nativeInstancing = instancerContext.empty();
-    return nativeInstancing;
-}
-
 //! \brief  Selection for both instanced and non-instanced cases.
 bool ProxyRenderDelegate::getInstancedSelectionPath(
     const MHWRender::MRenderItem&   renderItem,

lib/mayaUsd/render/vp2RenderDelegate/proxyRenderDelegate.h

@@ -105,6 +105,14 @@ enum class UsdPointInstancesPickMode
     Prototypes
 };
 
+enum InstancingType
+{
+    kNativeInstancing,
+    kPointInstancing
+};
+
+using InstancePrototypePath = std::pair<SdfPath, InstancingType>;
+
 /*! \brief  USD Proxy rendering routine via VP2 MPxSubSceneOverride
 
     This drawing routine leverages HdVP2RenderDelegate for synchronization
@@ -173,9 +181,6 @@ class ProxyRenderDelegate
     SdfPath GetScenePrimPath(const SdfPath& rprimId, int instanceIndex) const;
 #endif
 
-    MAYAUSD_CORE_PUBLIC
-    bool SupportPerInstanceDisplayLayers(const SdfPath& rprimId) const;
-
     MAYAUSD_CORE_PUBLIC
     void SelectionChanged();
 
@@ -254,13 +259,13 @@ class ProxyRenderDelegate
 
     // Takes in a path to instanced rprim and returns a path to the correspoding UsdPrim
     MAYAUSD_CORE_PUBLIC
-    SdfPath GetPathInPrototype(const SdfPath& id);
+    InstancePrototypePath GetPathInPrototype(const SdfPath& id);
 
     MAYAUSD_CORE_PUBLIC
     void UpdateInstancingMapEntry(
-        const SdfPath& oldPathInPrototype,
-        const SdfPath& newPathInPrototype,
-        const SdfPath& rprimId);
+        const InstancePrototypePath& oldPathInPrototype,
+        const InstancePrototypePath& newPathInPrototype,
+        const SdfPath&               rprimId);
 
 #ifdef MAYA_NEW_POINT_SNAPPING_SUPPORT
     MAYAUSD_CORE_PUBLIC
@@ -368,7 +373,7 @@ class ProxyRenderDelegate
     bool                                _needTexturedMaterials = false;
 
     // maps from a path in USD prototype to the corresponding rprim paths
-    std::multimap<SdfPath, SdfPath> _instancingMap;
+    std::multimap<InstancePrototypePath, SdfPath> _instancingMap;
 
     bool _isPopulated {
         false