Merge pull request #2170 from alicevision/mug/scenePreviewMask

[blender] apply masks to scene preview

meshroom/nodes/blender/ScenePreview.py

@@ -1,4 +1,4 @@
-__version__ = "1.0"
+__version__ = "2.0"
 
 from meshroom.core import desc
 import os.path
@@ -66,6 +66,21 @@ class ScenePreview(desc.CommandLineNode):
             uid=[0],
             enabled=lambda node: node.useBackground.value,
         ),
+        desc.BoolParam(
+            name="useMasks",
+            label="Apply Masks",
+            description="Apply mask to the rendered geometry.",
+            value=True,
+            uid=[0],
+        ),
+        desc.File(
+            name="masks",
+            label="Masks",
+            description="Folder containing the masks.",
+            value="",
+            uid=[0],
+            enabled=lambda node: node.useMasks.value,
+        ),
         desc.GroupAttribute(
             name="pointCloudParams",
             label="Point Cloud Settings",

meshroom/nodes/blender/scripts/preview.py

@@ -36,12 +36,20 @@ def createParser():
     )
     parser.add_argument(
         "--undistortedImages", metavar='FILE', required=False,
-        help="Save the generated file to the specified path",
+        help="Path to folder containing undistorted images to use as background.",
     )
     parser.add_argument(
         "--model", metavar='FILE', required=True,
         help="Point Cloud or Mesh used in the rendering.",
     )
+    parser.add_argument(
+        "--useMasks", type=lambda x: (str(x).lower() == 'true'), required=True,
+        help="Apply mask to the rendered geometry or not.",
+    )
+    parser.add_argument(
+        "--masks", metavar='FILE', required=False,
+        help="Path to folder containing masks to apply on rendered geometry.",
+    )
     parser.add_argument(
         "--particleSize", type=float, required=False,
         help="Scale of particles used to show the point cloud",
@@ -132,21 +140,31 @@ def initScene():
     bpy.context.scene.cycles.use_denoising = False
 
 
-def initCompositing():
+def initCompositing(useBackground, useMasks):
     '''Initialize Blender compositing graph for adding background image to render.'''
     bpy.context.scene.render.film_transparent = True
     bpy.context.scene.use_nodes = True
-    bpy.context.scene.node_tree.nodes.new(type="CompositorNodeAlphaOver")
-    bpy.context.scene.node_tree.nodes.new(type="CompositorNodeImage")
-    bpy.context.scene.node_tree.links.new(
-        bpy.context.scene.node_tree.nodes['Alpha Over'].outputs['Image'],
-        bpy.context.scene.node_tree.nodes['Composite'].inputs['Image'])
-    bpy.context.scene.node_tree.links.new(
-        bpy.context.scene.node_tree.nodes['Image'].outputs['Image'],
-        bpy.context.scene.node_tree.nodes['Alpha Over'].inputs[1])
-    bpy.context.scene.node_tree.links.new(
-        bpy.context.scene.node_tree.nodes['Render Layers'].outputs['Image'],
-        bpy.context.scene.node_tree.nodes['Alpha Over'].inputs[2])
+    nodeAlphaOver = bpy.context.scene.node_tree.nodes.new(type="CompositorNodeAlphaOver")
+    nodeSetAlpha = bpy.context.scene.node_tree.nodes.new(type="CompositorNodeSetAlpha")
+    nodeBackground = bpy.context.scene.node_tree.nodes.new(type="CompositorNodeImage")
+    nodeMask = bpy.context.scene.node_tree.nodes.new(type="CompositorNodeImage")
+    nodeRender = bpy.context.scene.node_tree.nodes['Render Layers']
+    nodeComposite = bpy.context.scene.node_tree.nodes['Composite']
+    if useBackground and useMasks:
+        bpy.context.scene.node_tree.links.new(nodeBackground.outputs['Image'], nodeAlphaOver.inputs[1])
+        bpy.context.scene.node_tree.links.new(nodeRender.outputs['Image'], nodeSetAlpha.inputs['Image'])
+        bpy.context.scene.node_tree.links.new(nodeMask.outputs['Image'], nodeSetAlpha.inputs['Alpha'])
+        bpy.context.scene.node_tree.links.new(nodeSetAlpha.outputs['Image'], nodeAlphaOver.inputs[2])
+        bpy.context.scene.node_tree.links.new(nodeAlphaOver.outputs['Image'], nodeComposite.inputs['Image'])
+    elif useBackground:
+        bpy.context.scene.node_tree.links.new(nodeBackground.outputs['Image'], nodeAlphaOver.inputs[1])
+        bpy.context.scene.node_tree.links.new(nodeRender.outputs['Image'], nodeAlphaOver.inputs[2])
+        bpy.context.scene.node_tree.links.new(nodeAlphaOver.outputs['Image'], nodeComposite.inputs['Image'])
+    elif useMasks:
+        bpy.context.scene.node_tree.links.new(nodeRender.outputs['Image'], nodeSetAlpha.inputs['Image'])
+        bpy.context.scene.node_tree.links.new(nodeMask.outputs['Image'], nodeSetAlpha.inputs['Alpha'])
+        bpy.context.scene.node_tree.links.new(nodeSetAlpha.outputs['Image'], nodeComposite.inputs['Image'])
+    return nodeBackground, nodeMask
 
 
 def setupRender(view, intrinsic, pose, outputDir):
@@ -157,7 +175,7 @@ def setupRender(view, intrinsic, pose, outputDir):
     bpy.context.scene.render.filepath = os.path.abspath(outputDir + '/' + baseImgName + '_preview.jpg')
 
 
-def setupBackground(view, folderUndistorted):
+def setupBackground(view, folderUndistorted, nodeBackground):
     '''Retrieve undistorted image corresponding to view and use it as background.'''
     matches = glob.glob(folderUndistorted + '/*' + view['viewId'] + "*") # try with viewId
     if len(matches) == 0:
@@ -168,10 +186,25 @@ def setupBackground(view, folderUndistorted):
         return None
     undistortedImgPath = matches[0]
     img = bpy.data.images.load(filepath=undistortedImgPath)
-    bpy.context.scene.node_tree.nodes["Image"].image = img
+    nodeBackground.image = img
     return img
 
 
+def setupMask(view, folderMasks, nodeMask):
+    '''Retrieve mask corresponding to view and use it in compositing graph.'''
+    matches = glob.glob(folderMasks + '/*' + view['viewId'] + "*") # try with viewId
+    if len(matches) == 0:
+        baseImgName = os.path.splitext(os.path.basename(view['path']))[0]
+        matches = glob.glob(folderMasks + '/*' + baseImgName + "*") # try with image name
+    if len(matches) == 0:
+        # no background image found
+        return None
+    maskPath = matches[0]
+    mask = bpy.data.images.load(filepath=maskPath)
+    nodeMask.image = mask
+    return mask
+
+
 def loadModel(filename):
     '''Load model in Alembic of OBJ format. Make sure orientation matches camera orientation.'''
     if filename.lower().endswith('.obj'):
@@ -300,8 +333,7 @@ def main():
     initScene()
 
     print("Init compositing")
-    if args.useBackground:
-        initCompositing()
+    nodeBackground, nodeMask = initCompositing(args.useBackground, args.useMasks)
 
     print("Parse cameras SfM file")
     views, intrinsics, poses = parseSfMCameraFile(args.cameras)
@@ -344,19 +376,31 @@ def main():
             continue
 
         print("Rendering view " + view['viewId'])
+
         img = None
         if args.useBackground:
-            img = setupBackground(view, args.undistortedImages)
+            img = setupBackground(view, args.undistortedImages, nodeBackground)
             if not img:
                 # background setup failed
                 # do not render this frame
                 continue
+
+        mask = None
+        if args.useMasks:
+            mask = setupMask(view, args.masks, nodeMask)
+            if not mask:
+                # mask setup failed
+                # do not render this frame
+                continue
+
         setupRender(view, intrinsic, pose, args.output)
         bpy.ops.render.render(write_still=True)
 
+        # clear memory
         if img:
-            # clear memory 
             bpy.data.images.remove(img)
+        if mask:
+            bpy.data.images.remove(mask)
 
     print("Done")
     return 0

meshroom/pipelines/cameraTracking.mg

@@ -5,31 +5,31 @@
         "fileVersion": "1.1",
         "template": true,
         "nodesVersions": {
-            "ExportDistortion": "1.0",
-            "FeatureExtraction": "1.3",
-            "DepthMap": "4.0",
-            "Texturing": "6.0",
             "ImageSegmentation": "1.0",
             "ApplyCalibration": "1.0",
-            "ScenePreview": "1.0",
             "MeshDecimate": "1.0",
-            "KeyframeSelection": "5.0",
             "PrepareDenseScene": "3.1",
+            "MeshFiltering": "3.0",
             "SfMTransfer": "2.1",
             "Publish": "1.3",
-            "MeshFiltering": "3.0",
             "DepthMapFilter": "3.0",
             "Meshing": "7.0",
             "ImageMatchingMultiSfM": "1.0",
+            "ScenePreview": "2.0",
             "FeatureMatching": "2.0",
             "CameraInit": "9.0",
             "ImageMatching": "2.0",
             "CheckerboardDetection": "1.0",
             "ConvertSfMFormat": "2.0",
             "SfMTriangulation": "1.0",
+            "KeyframeSelection": "5.0",
             "StructureFromMotion": "3.1",
             "ExportAnimatedCamera": "2.0",
-            "DistortionCalibration": "3.0"
+            "DistortionCalibration": "3.0",
+            "ExportDistortion": "1.0",
+            "FeatureExtraction": "1.3",
+            "DepthMap": "4.0",
+            "Texturing": "6.0"
         }
     },
     "graph": {
@@ -363,7 +363,8 @@
             "inputs": {
                 "cameras": "{ConvertSfMFormat_1.output}",
                 "model": "{MeshDecimate_1.output}",
-                "undistortedImages": "{ExportAnimatedCamera_1.outputUndistorted}"
+                "undistortedImages": "{ExportAnimatedCamera_1.outputUndistorted}",
+                "masks": "{ImageSegmentation_1.output}"
             },
             "internalInputs": {
                 "color": "#4c594c"

meshroom/pipelines/photogrammetryAndCameraTracking.mg

@@ -5,29 +5,29 @@
         "fileVersion": "1.1",
         "template": true,
         "nodesVersions": {
-            "ExportDistortion": "1.0",
-            "FeatureExtraction": "1.3",
-            "DepthMap": "4.0",
-            "Texturing": "6.0",
             "ImageSegmentation": "1.0",
             "ApplyCalibration": "1.0",
-            "ScenePreview": "1.0",
             "MeshDecimate": "1.0",
-            "KeyframeSelection": "5.0",
             "PrepareDenseScene": "3.1",
             "Publish": "1.3",
             "MeshFiltering": "3.0",
             "DepthMapFilter": "3.0",
             "Meshing": "7.0",
             "ImageMatchingMultiSfM": "1.0",
+            "ScenePreview": "2.0",
             "FeatureMatching": "2.0",
             "CameraInit": "9.0",
             "ImageMatching": "2.0",
             "CheckerboardDetection": "1.0",
             "ConvertSfMFormat": "2.0",
+            "KeyframeSelection": "5.0",
             "StructureFromMotion": "3.1",
             "ExportAnimatedCamera": "2.0",
-            "DistortionCalibration": "3.0"
+            "DistortionCalibration": "3.0",
+            "ExportDistortion": "1.0",
+            "FeatureExtraction": "1.3",
+            "DepthMap": "4.0",
+            "Texturing": "6.0"
         }
     },
     "graph": {
@@ -240,7 +240,8 @@
             "inputs": {
                 "cameras": "{ConvertSfMFormat_1.output}",
                 "model": "{MeshDecimate_1.output}",
-                "undistortedImages": "{ExportAnimatedCamera_1.outputUndistorted}"
+                "undistortedImages": "{ExportAnimatedCamera_1.outputUndistorted}",
+                "masks": "{ImageSegmentation_1.output}"
             },
             "internalInputs": {
                 "color": "#4c594c"