Thermal Camera: Visualize objects with a heat signature

ogre2/src/Ogre2ThermalCamera.cc

@@ -22,9 +22,14 @@
 #endif
 
 #include <math.h>
+
 #include <limits>
+#include <string>
+#include <unordered_map>
+#include <variant>
 
 #include <ignition/common/Console.hh>
+#include <ignition/common/Filesystem.hh>
 #include <ignition/math/Helpers.hh>
 
 #include "ignition/rendering/RenderTypes.hh"
@@ -50,7 +55,9 @@ class Ogre2ThermalCameraMaterialSwitcher : public Ogre::RenderTargetListener
 {
   /// \brief constructor
   /// \param[in] _scene the scene manager responsible for rendering
-  public: explicit Ogre2ThermalCameraMaterialSwitcher(Ogre2ScenePtr _scene);
+  /// \param[in] _name the name of the thermal camera
+  public: explicit Ogre2ThermalCameraMaterialSwitcher(Ogre2ScenePtr _scene,
+              const std::string & _name);
 
   /// \brief destructor
   public: ~Ogre2ThermalCameraMaterialSwitcher() = default;
@@ -73,6 +80,21 @@ class Ogre2ThermalCameraMaterialSwitcher : public Ogre::RenderTargetListener
   /// \brief Pointer to the heat source material
   private: Ogre::MaterialPtr heatSourceMaterial;
 
+  /// \brief Pointer to the "base" heat signature material.
+  /// All renderable items with a heat signature texture use their own
+  /// copy of this base material, with the item's specific heat
+  /// signature texture applied to it
+  private: Ogre::MaterialPtr baseHeatSigMaterial;
+
+  /// \brief A map of all items that have a heat signature material.
+  /// The key is the item's ID, and the value is the heat signature
+  /// for that item.
+  private: std::unordered_map<Ogre::IdType, Ogre::MaterialPtr>
+            heatSignatureMaterials;
+
+  /// \brief The name of the thermal camera sensor
+  private: const std::string name;
+
   /// \brief Custom parameter index of temperature data in an ogre subitem.
   /// This has to match the custom index specifed in ThermalHeatSource material
   /// script in media/materials/scripts/thermal_camera.material
@@ -135,7 +157,8 @@ using namespace rendering;
 
 //////////////////////////////////////////////////
 Ogre2ThermalCameraMaterialSwitcher::Ogre2ThermalCameraMaterialSwitcher(
-    Ogre2ScenePtr _scene)
+    Ogre2ScenePtr _scene, const std::string & _name) :
+  name(_name)
 {
   this->scene = _scene;
   // plain opaque material
@@ -145,6 +168,9 @@ Ogre2ThermalCameraMaterialSwitcher::Ogre2ThermalCameraMaterialSwitcher(
 
   this->heatSourceMaterial = res.staticCast<Ogre::Material>();
   this->heatSourceMaterial->load();
+
+  this->baseHeatSigMaterial = Ogre::MaterialManager::getSingleton().
+    getByName("ThermalHeatSignature");
 }
 
 //////////////////////////////////////////////////
@@ -162,7 +188,7 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
     Ogre::MovableObject *object = itor.peekNext();
     Ogre::Item *item = static_cast<Ogre::Item *>(object);
 
-    std::string tempKey = "temperature";
+    const std::string tempKey = "temperature";
     // get visual
     Ogre::Any userAny = item->getUserObjectBindings().getUserAny();
     if (!userAny.isEmpty() && userAny.getType() == typeid(unsigned int))
@@ -181,7 +207,7 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
 
       // get temperature
       Variant tempAny = ogreVisual->UserData(tempKey);
-      if (tempAny.index() != 0)
+      if (tempAny.index() != 0 && !std::holds_alternative<std::string>(tempAny))
       {
         float temp = -1.0;
         try
@@ -231,6 +257,102 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
           }
         }
       }
+      // get heat signature and the corresponding min/max temperature values
+      else if (auto heatSignature = std::get_if<std::string>(&tempAny))
+      {
+        // if this is the first time rendering the heat signature,
+        // we need to make sure that the texture is loaded and applied to
+        // the heat signature material (if the texture exists) before loading
+        // the heat signature material
+        if (this->heatSignatureMaterials.find(item->getId()) ==
+            this->heatSignatureMaterials.end())
+        {
+          bool foundTexture = false;
+          std::string baseName;
+
+          // TODO(adlarkin) re-use the texture searching code from
+          // Ogre2Material::SetTextureMapImpl somehow?
+          // This may require an API breaking change in Ogre2Material.cc
+          const auto& texture = *heatSignature;
+          if (!common::isFile(texture))
+          {
+            ignerr << "texture given (" << texture << ") is not a file\n";
+          }
+          else
+          {
+            baseName = common::basename(texture);
+            auto idx = texture.rfind(baseName);
+            if (idx != std::string::npos)
+            {
+              std::string dirPath = texture.substr(0, idx);
+              if (!dirPath.empty())
+              {
+                foundTexture = true;
+                if (!Ogre::ResourceGroupManager::getSingleton().
+                      resourceLocationExists(dirPath))
+                {
+                  Ogre::ResourceGroupManager::getSingleton().addResourceLocation(
+                      dirPath, "FileSystem", "General");
+                }
+              }
+            }
+          }
+
+          // create a material for this item, now that the texture has been
+          // searched for. We must clone the base heat signature material since
+          // different items may use different textures. We also append the
+          // item's ID to the end of the new material name to ensure new
+          // material uniqueness in case two items use the same heat signature
+          // texture, but have different temperature ranges
+          auto heatSignatureMaterial = this->baseHeatSigMaterial->clone(
+              this->name + "_" + baseName + "_" +
+              Ogre::StringConverter::toString(item->getId()));
+          if (foundTexture)
+          {
+            auto textureUnitStatePtr = heatSignatureMaterial->
+              getTechnique(0)->getPass(0)->getTextureUnitState(0);
+            Ogre::String textureName = baseName;
+            textureUnitStatePtr->setTextureName(textureName);
+          }
+          else
+          {
+            ignerr << "Error loading texture: " << texture << "\n"
+              << "(material will be loaded without a texture applied to it)\n";
+          }
+          // set temperature range for the heat signature
+          auto minTempVariant = ogreVisual->UserData("minTemp");
+          auto maxTempVariant = ogreVisual->UserData("maxTemp");
+          auto minTemperature = std::get_if<float>(&minTempVariant);
+          auto maxTemperature = std::get_if<float>(&maxTempVariant);
+          if (minTemperature && maxTemperature)
+          {
+            // TODO(adlarkin) check for bad input
+            // (negative temps, values above the camera's max kelvin, etc)
+            Ogre::GpuProgramParametersSharedPtr params =
+              heatSignatureMaterial->getTechnique(0)->getPass(0)->
+              getFragmentProgramParameters();
+            params->setNamedConstant("minTemp",
+                static_cast<float>(*minTemperature));
+            params->setNamedConstant("maxTemp",
+                static_cast<float>(*maxTemperature));
+          }
+          heatSignatureMaterial->load();
+          this->heatSignatureMaterials[item->getId()] = heatSignatureMaterial;
+        }
+
+        // set visibility flag so thermal camera can see it
+        item->addVisibilityFlags(0x10000000);
+
+        for (unsigned int i = 0; i < item->getNumSubItems(); ++i)
+        {
+          Ogre::SubItem *subItem = item->getSubItem(i);
+
+          Ogre::HlmsDatablock *datablock = subItem->getDatablock();
+          this->datablockMap[subItem] = datablock;
+
+          subItem->setMaterial(this->heatSignatureMaterials[item->getId()]);
+        }
+      }
     }
     itor.moveNext();
   }
@@ -633,7 +755,7 @@ void Ogre2ThermalCamera::CreateThermalTexture()
     if (c.textures[0]->getSrcFormat() == Ogre::PF_R8G8B8)
     {
       this->dataPtr->thermalMaterialSwitcher.reset(
-          new Ogre2ThermalCameraMaterialSwitcher(this->scene));
+          new Ogre2ThermalCameraMaterialSwitcher(this->scene, this->Name()));
       c.target->addListener(this->dataPtr->thermalMaterialSwitcher.get());
       break;
     }

ogre2/src/media/materials/programs/heat_signature_fs.glsl

@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2021 Open Source Robotics Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#version 330
+
+// The input texture, which is set up by the Ogre Compositor infrastructure.
+uniform sampler2D RT;
+
+// input params from vertex shader
+in block
+{
+  vec2 uv0;
+} inPs;
+
+// final output color
+out vec4 fragColor;
+
+// The minimum and maximum temprature values (in Kelvin) that the
+// heat signature texture should be normalized to
+// (users can override these defaults)
+uniform float minTemp = 0.0;
+uniform float maxTemp = 100.0;
+
+// map a temperature from the [0, 655.35] range to the [minTemp, maxTemp]
+// range (655.35 is the largest Kelvin value allowed)
+float mapNormalized(float num)
+{
+  float mappedKelvin = ((maxTemp - minTemp) * num) + minTemp;
+  return mappedKelvin / 655.35;
+}
+
+void main()
+{
+  float heat = texture(RT, inPs.uv0.xy).x;
+  fragColor = vec4(mapNormalized(heat), 0, 0, 1.0);
+}

ogre2/src/media/materials/scripts/thermal.material

@@ -98,3 +98,49 @@ material ThermalHeatSource
     }
   }
 }
+
+// copied from gaussian_noise.material
+// TODO(adlarkin) try to reference the GaussianNoiseVS definition in
+// gaussian_noise.material so that we don't have a duplicate definition here?
+vertex_program GaussianNoiseVS glsl
+{
+  source gaussian_noise_vs.glsl
+  default_params
+  {
+    param_named_auto worldViewProj worldviewproj_matrix
+  }
+}
+
+fragment_program HeatSignatureFS glsl
+{
+  source heat_signature_fs.glsl
+  default_params
+  {
+    param_named RT int 0
+  }
+}
+
+material ThermalHeatSignature
+{
+  technique
+  {
+    pass
+    {
+      fog_override true
+
+      // using gaussian noise VS because this is a simple VS that accomplishes
+      // what we need for a heat signature VS
+      vertex_program_ref GaussianNoiseVS { }
+
+      fragment_program_ref HeatSignatureFS { }
+
+      texture_unit RT
+      {
+        tex_coord_set 0
+
+        // the texture for this texture unit is set programmatically
+        // since the texture file location and name are specified by the user
+      }
+    }
+  }
+}