Added intersection test command for ProxyDrawOverride

plugin/al/lib/AL_USDMaya/AL/usdmaya/PluginRegister.h

@@ -272,6 +272,7 @@ template <typename AFnPlugin> MStatus registerPlugin(AFnPlugin& plugin)
     AL_REGISTER_COMMAND(plugin, AL::usdmaya::cmds::EventQuery);
     AL_REGISTER_COMMAND(plugin, AL::usdmaya::cmds::EventLookup);
     AL_REGISTER_COMMAND(plugin, AL::usdmaya::cmds::TranslatePrim);
+    AL_REGISTER_COMMAND(plugin, AL::usdmaya::cmds::ProxyShapeTestIntersection);
     AL_REGISTER_COMMAND(plugin, AL::usdmaya::cmds::LayerManager);
     AL_REGISTER_COMMAND(plugin, AL::usdmaya::cmds::SyncFileIOGui);
     AL_REGISTER_COMMAND(plugin, AL::usdmaya::fileio::ImportCommand);
@@ -472,6 +473,7 @@ template <typename AFnPlugin> MStatus unregisterPlugin(AFnPlugin& plugin)
     AL_UNREGISTER_COMMAND(plugin, AL::usdmaya::fileio::ImportCommand);
     AL_UNREGISTER_COMMAND(plugin, AL::usdmaya::fileio::ExportCommand);
     AL_UNREGISTER_COMMAND(plugin, AL::usdmaya::cmds::TranslatePrim);
+    AL_UNREGISTER_COMMAND(plugin, AL::usdmaya::cmds::ProxyShapeTestIntersection);
     AL_UNREGISTER_COMMAND(plugin, AL::usdmaya::cmds::LayerManager);
     AL_UNREGISTER_COMMAND(plugin, AL::usdmaya::cmds::CreateUsdPrim);
     AL_UNREGISTER_TRANSLATOR(plugin, AL::usdmaya::fileio::ImportTranslator);

plugin/al/lib/AL_USDMaya/AL/usdmaya/cmds/ProxyShapeCommands.cpp

@@ -19,12 +19,20 @@
 #include "AL/maya/utils/MenuBuilder.h"
 #include "AL/maya/utils/Utils.h"
 #include "AL/usdmaya/cmds/ProxyShapePostLoadProcess.h"
+#include "AL/usdmaya/nodes/Engine.h"
 #include "AL/usdmaya/nodes/LayerManager.h"
+#include "AL/usdmaya/nodes/ProxyShape.h"
 
+#include <pxr/imaging/hdx/pickTask.h>
+#include <pxr/usd/sdf/path.h>
+
+#include <maya/M3dView.h>
 #include <maya/MArgDatabase.h>
 #include <maya/MArgList.h>
 #include <maya/MDagPathArray.h>
 #include <maya/MFnDagNode.h>
+#include <maya/MItDependencyNodes.h>
+#include <maya/MMatrix.h>
 #include <maya/MSyntax.h>
 
 namespace {
@@ -1341,6 +1349,113 @@ MStatus TranslatePrim::redoIt()
     return MStatus::kSuccess;
 }
 
+//----------------------------------------------------------------------------------------------------------------------
+
+AL_MAYA_DEFINE_COMMAND(ProxyShapeTestIntersection, AL_usdmaya);
+
+//----------------------------------------------------------------------------------------------------------------------
+
+MSyntax ProxyShapeTestIntersection::createSyntax()
+{
+    MSyntax syntax = setUpCommonSyntax();
+    syntax.addFlag("-sx", "-screenX", MSyntax::kDouble);
+    syntax.addFlag("-sy", "-screenY", MSyntax::kDouble);
+    return syntax;
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+MStatus ProxyShapeTestIntersection::doIt(const MArgList& args)
+{
+    TF_DEBUG(ALUSDMAYA_COMMANDS).Msg("ProxyShapeTestIntersection::doIt\n");
+    try {
+        MArgDatabase db = makeDatabase(args);
+        AL_MAYA_COMMAND_HELP(db, g_helpText);
+        MDagPath proxyDagPath;
+        m_proxy = getShapeNode(db);
+        if (!m_proxy || !m_proxy->usdStage()) {
+            throw MS::kFailure;
+        }
+
+        if (!db.isFlagSet("-sx") || !db.isFlagSet("-sy")) {
+            MGlobal::displayError("-sx and -sy are required");
+            return MStatus::kFailure;
+        }
+
+        db.getFlagArgument("-sx", 0, m_sx);
+        db.getFlagArgument("-sy", 0, m_sy);
+    } catch (const MStatus& status) {
+        return status;
+    }
+
+    return redoIt();
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+bool ProxyShapeTestIntersection::isUndoable() const { return false; }
+
+//----------------------------------------------------------------------------------------------------------------------
+MStatus ProxyShapeTestIntersection::redoIt()
+{
+    MStatus status;
+    auto    currentView = M3dView::active3dView(&status);
+
+    MMatrix worldViewMatrix, projectionMatrix;
+    status = currentView.modelViewMatrix(worldViewMatrix);
+    status = currentView.projectionMatrix(projectionMatrix);
+
+    // Compute a pick matrix that, when it is post-multiplied with the projection
+    // matrix, will cause the picking region to fill the entire viewport for
+    // OpenGL selection.
+    {
+        unsigned int view_x, view_y, view_w, view_h;
+        currentView.viewport(view_x, view_y, view_w, view_h);
+
+        MMatrix pickMatrix;
+        pickMatrix[0][0] = view_w;
+        pickMatrix[1][1] = view_h;
+        pickMatrix[3][0] = view_w - 2.0 * (m_sx - view_x);
+        pickMatrix[3][1] = view_h - 2.0 * (m_sy - view_y);
+
+        projectionMatrix *= pickMatrix;
+    }
+
+    auto engine = m_proxy->engine();
+    if (!engine)
+        return MStatus::kFailure;
+
+    auto          stage = m_proxy->getUsdStage();
+    UsdPrim       root = stage->GetPseudoRoot();
+    SdfPathVector rootPath;
+    rootPath.push_back(root.GetPath());
+    UsdImagingGLRenderParams params;
+    unsigned int             resolution = 10;
+    nodes::Engine::HitBatch  hitBatch;
+
+    bool hit = engine->TestIntersectionBatch(
+        GfMatrix4d(worldViewMatrix.matrix),
+        GfMatrix4d(projectionMatrix.matrix),
+        GfMatrix4d(1.f),
+        rootPath,
+        params,
+        HdxPickTokens->resolveNearestToCamera,
+        resolution,
+        &hitBatch);
+
+    if (hit) {
+        clearResult();
+        for (const auto& it : hitBatch) {
+            const double* p = it.second.GetArray();
+            appendToResult(p[0]);
+            appendToResult(p[1]);
+            appendToResult(p[2]);
+        }
+    } else {
+        MGlobal::displayInfo("[AL_usdmaya_ProxyShapeTestIntersection]: No hit points found\n");
+    }
+
+    return status;
+}
+
 //----------------------------------------------------------------------------------------------------------------------
 void constructProxyShapeCommandGuis()
 {
@@ -1770,6 +1885,15 @@ TranslatePrim Overview:
   their corresponding prim type is brought into the scene.
 )";
 //----------------------------------------------------------------------------------------------------------------------
+const char* const ProxyShapeTestIntersection::g_helpText = R"(
+ProxyShapeTestIntersection Overview:
+
+  Used to retrieve the world space location of the point of the closest USD prim hit by ray cast from a selected
+  position.
+
+    AL_usdmaya_ProxyShapeTestIntersection -sx 30 -sy 80;
+)";
+//----------------------------------------------------------------------------------------------------------------------
 } // namespace cmds
 } // namespace usdmaya
 } // namespace AL

plugin/al/lib/AL_USDMaya/AL/usdmaya/cmds/ProxyShapeCommands.h

@@ -296,6 +296,25 @@ class TranslatePrim : public ProxyShapeCommandBase
     MStatus redoIt() override;
 };
 
+//----------------------------------------------------------------------------------------------------------------------
+/// \brief  ProxyShapeTestIntersection
+/// \ingroup commands
+//----------------------------------------------------------------------------------------------------------------------
+class ProxyShapeTestIntersection : public ProxyShapeCommandBase
+{
+    nodes::ProxyShape* m_proxy;
+    double             m_sx;
+    double             m_sy;
+
+public:
+    AL_MAYA_DECLARE_COMMAND();
+
+private:
+    bool    isUndoable() const override;
+    MStatus doIt(const MArgList& args) override;
+    MStatus redoIt() override;
+};
+
 /// \brief  This function will generate all of the MEL script menus, and the option boxes, for all
 /// of the proxy shape
 ///         commands.

plugin/al/lib/AL_USDMaya/AL/usdmaya/nodes/ProxyDrawOverride.cpp

@@ -653,32 +653,40 @@ bool ProxyDrawOverride::userSelect(
 
     auto selected = false;
 
-    auto addSelection = [&hitBatch, &selectionList, &worldSpaceHitPts, proxyShape, &selected](
-                            const MString& command) {
-        selected = true;
-        MStringArray nodes;
-        MGlobal::executeCommand(command, nodes, false, true);
-
-        for (const auto& it : hitBatch) {
-            // Retarget hit path based on pick mode policy. The retargeted prim must
-            // align with the path used in the 'AL_usdmaya_ProxyShapeSelect' command.
-            const SdfPath hitPath = it.first;
-            const UsdPrim retargetedHitPrim
-                = retargetSelectPrim(proxyShape->getUsdStage()->GetPrimAtPath(hitPath));
-            const MObject obj = proxyShape->findRequiredPath(retargetedHitPrim.GetPath());
-
-            if (obj != MObject::kNullObj) {
-                MFnDagNode dagNode(obj);
-                MDagPath   dg;
-                dagNode.getPath(dg);
-                const double* p = it.second.GetArray();
-
-                selectionList.add(dg);
-                worldSpaceHitPts.append(MPoint(p[0], p[1], p[2]));
-            }
-        }
+    auto transformPath = [proxyShape](const SdfPath& path) {
+        // if we encounter an instance proxy, select it's parent prim instead!
+        // Modifying an InstanceProxy will cause an exception to be thrown.
+        auto prim = proxyShape->getUsdStage()->GetPrimAtPath(path);
+        return prim.IsInstanceProxy() ? path.GetParentPath() : path;
     };
 
+    auto addSelection
+        = [&hitBatch, &selectionList, &worldSpaceHitPts, proxyShape, &selected, transformPath](
+              const MString& command) {
+              selected = true;
+              MStringArray nodes;
+              MGlobal::executeCommand(command, nodes, false, true);
+
+              for (const auto& it : hitBatch) {
+                  // Retarget hit path based on pick mode policy. The retargeted prim must
+                  // align with the path used in the 'AL_usdmaya_ProxyShapeSelect' command.
+                  const SdfPath hitPath = transformPath(it.first);
+                  const UsdPrim retargetedHitPrim
+                      = retargetSelectPrim(proxyShape->getUsdStage()->GetPrimAtPath(hitPath));
+                  const MObject obj = proxyShape->findRequiredPath(retargetedHitPrim.GetPath());
+
+                  if (obj != MObject::kNullObj) {
+                      MFnDagNode dagNode(obj);
+                      MDagPath   dg;
+                      dagNode.getPath(dg);
+                      const double* p = it.second.GetArray();
+
+                      selectionList.add(dg);
+                      worldSpaceHitPts.append(MPoint(p[0], p[1], p[2]));
+                  }
+              }
+          };
+
     // Maya determines the selection list adjustment mode by Ctrl/Shift modifiers.
     int modifiers = 0;
     MGlobal::executeCommand("getModifiers", modifiers);
@@ -732,7 +740,7 @@ bool ProxyDrawOverride::userSelect(
         if (!hitBatch.empty()) {
             paths.reserve(hitBatch.size());
             for (const auto& it : hitBatch) {
-                paths.push_back(it.first);
+                paths.push_back(transformPath(it.first));
             }
         }