Skip to content

Commit

Permalink
Thermal Camera: Visualize objects with a heat signature (#189)
Browse files Browse the repository at this point in the history 
* started incorporating heat signature to thermal objects

Signed-off-by: Ashton Larkin <[email protected]>

* promising results with hardcoded texture

Signed-off-by: Ashton Larkin <[email protected]>

* heat signature for one object is working

Signed-off-by: Ashton Larkin <[email protected]>

* enable multiple objects with different heat signatures

Signed-off-by: Ashton Larkin <[email protected]>

* allow for users to specify a custom temperature range

Signed-off-by: Ashton Larkin <[email protected]>

* add check back in to fix missing background

Signed-off-by: Ashton Larkin <[email protected]>

* Add temperature variations to thermal camera readings based on color (#211)

* vary non-heat source temp based on rgb balues

Signed-off-by: Ian Chen <[email protected]>

* add more comments, var to set rgb to temp

Signed-off-by: Ian Chen <[email protected]>

* add more doc

Signed-off-by: Ian Chen <[email protected]>

* remove comment

Signed-off-by: Ian Chen <[email protected]>

* feedback changes

Signed-off-by: Ian Chen <[email protected]>

* added reference link to RGB/grayscale equation

Signed-off-by: Ashton Larkin <[email protected]>

Co-authored-by: Ashton Larkin <[email protected]>

* make sure user input temperature range is within [0,655.35] kelvin

Signed-off-by: Ashton Larkin <[email protected]>

* added heat signature thermal camera test

Signed-off-by: Ashton Larkin <[email protected]>

* added missing test texture and fixed codecheck

Signed-off-by: Ashton Larkin <[email protected]>

* revert addition of Ogre2Material::PbsToUnlitDatablock helper function

Signed-off-by: Ashton Larkin <[email protected]>

* address review feedback

Signed-off-by: Ashton Larkin <[email protected]>

* address windows warnings

Signed-off-by: Ashton Larkin <[email protected]>

Co-authored-by: Ian Chen <[email protected]>
  • Loading branch information
@adlarkin @iche033
adlarkin and iche033 authored Feb 2, 2021
1 parent cac86c8 commit d9d5374
Show file tree
Hide file tree
Showing 7 changed files with 349 additions and 73 deletions.
4 changes: 2 additions & 2 deletions ogre2/include/ignition/rendering/ogre2/Ogre2RenderEngine.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@ namespace ignition
// Documentation Inherited.
public: virtual std::string Name() const override;

/// \brief Add path to resourcea in ogre2's resource manager
/// \param[in] _uri Reousrce path in the form of an uri
/// \brief Add path to resource in ogre2's resource manager
/// \param[in] _uri Resource path in the form of an uri
public: void AddResourcePath(const std::string &_uri) override;

/// \brief Get the ogre2 root object
Expand Down
214 changes: 163 additions & 51 deletions ogre2/src/Ogre2ThermalCamera.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,30 @@
#endif

#include <math.h>

#include <algorithm>
#include <limits>
#include <string>
#include <unordered_map>
#include <variant>

#ifdef _MSC_VER
#pragma warning(push, 0)
#endif
#include <Hlms/Pbs/OgreHlmsPbsDatablock.h>
#include <Hlms/Unlit/OgreHlmsUnlitDatablock.h>
#ifdef _MSC_VER
#pragma warning(pop)
#endif

#include <ignition/common/Console.hh>
#include <ignition/common/Filesystem.hh>
#include <ignition/math/Helpers.hh>

#include "ignition/rendering/RenderTypes.hh"
#include "ignition/rendering/ogre2/Ogre2Conversions.hh"
#include "ignition/rendering/ogre2/Ogre2Includes.hh"
#include "ignition/rendering/ogre2/Ogre2Material.hh"
#include "ignition/rendering/ogre2/Ogre2RenderEngine.hh"
#include "ignition/rendering/ogre2/Ogre2RenderTarget.hh"
#include "ignition/rendering/ogre2/Ogre2RenderTypes.hh"
Expand All @@ -52,7 +68,9 @@ class Ogre2ThermalCameraMaterialSwitcher : public Ogre::RenderTargetListener
{
/// \brief constructor
/// \param[in] _scene the scene manager responsible for rendering
public: explicit Ogre2ThermalCameraMaterialSwitcher(Ogre2ScenePtr _scene);
/// \param[in] _name the name of the thermal camera
public: explicit Ogre2ThermalCameraMaterialSwitcher(Ogre2ScenePtr _scene,
const std::string & _name);

/// \brief destructor
public: ~Ogre2ThermalCameraMaterialSwitcher() = default;
Expand All @@ -75,13 +93,32 @@ class Ogre2ThermalCameraMaterialSwitcher : public Ogre::RenderTargetListener
/// \brief Pointer to the heat source material
private: Ogre::MaterialPtr heatSourceMaterial;

/// \brief Pointer to the "base" heat signature material.
/// All renderable items with a heat signature texture use their own
/// copy of this base material, with the item's specific heat
/// signature texture applied to it
private: Ogre::MaterialPtr baseHeatSigMaterial;

/// \brief A map of all items that have a heat signature material.
/// The key is the item's ID, and the value is the heat signature
/// for that item.
private: std::unordered_map<Ogre::IdType, Ogre::MaterialPtr>
heatSignatureMaterials;

/// \brief The name of the thermal camera sensor
private: const std::string name;

/// \brief The thermal camera
private: const Ogre::Camera* ogreCamera{nullptr};

/// \brief Custom parameter index of temperature data in an ogre subitem.
/// This has to match the custom index specifed in ThermalHeatSource material
/// script in media/materials/scripts/thermal_camera.material
private: const unsigned int customParamIdx = 10u;

/// \brief A map of ogre sub item pointer to their original hlms material
private: std::map<Ogre::SubItem *, Ogre::HlmsDatablock *> datablockMap;
private: std::unordered_map<Ogre::SubItem *, Ogre::HlmsDatablock *>
datablockMap;
};
}
}
Expand Down Expand Up @@ -130,14 +167,19 @@ class ignition::rendering::Ogre2ThermalCameraPrivate
/// \brief Pointer to material switcher
public: std::unique_ptr<Ogre2ThermalCameraMaterialSwitcher>
thermalMaterialSwitcher = nullptr;

/// \brief Add variation to temperature values based on object rgb values
/// This only affects objects that are not heat sources
/// TODO(anyone) add API for setting this value?
public: bool rgbToTemp = true;
};

using namespace ignition;
using namespace rendering;

//////////////////////////////////////////////////
Ogre2ThermalCameraMaterialSwitcher::Ogre2ThermalCameraMaterialSwitcher(
Ogre2ScenePtr _scene)
Ogre2ScenePtr _scene, const std::string & _name) : name(_name)
{
this->scene = _scene;
// plain opaque material
Expand All @@ -147,6 +189,11 @@ Ogre2ThermalCameraMaterialSwitcher::Ogre2ThermalCameraMaterialSwitcher(

this->heatSourceMaterial = res.staticCast<Ogre::Material>();
this->heatSourceMaterial->load();

this->baseHeatSigMaterial = Ogre::MaterialManager::getSingleton().
getByName("ThermalHeatSignature");

this->ogreCamera = this->scene->OgreSceneManager()->findCamera(this->name);
}

//////////////////////////////////////////////////
Expand All @@ -164,7 +211,7 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
Ogre::MovableObject *object = itor.peekNext();
Ogre::Item *item = static_cast<Ogre::Item *>(object);

std::string tempKey = "temperature";
const std::string tempKey = "temperature";
// get visual
Ogre::Any userAny = item->getUserObjectBindings().getUserAny();
if (!userAny.isEmpty() && userAny.getType() == typeid(unsigned int))
Expand All @@ -183,7 +230,7 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(

// get temperature
Variant tempAny = ogreVisual->UserData(tempKey);
if (tempAny.index() != 0)
if (tempAny.index() != 0 && !std::holds_alternative<std::string>(tempAny))
{
float temp = -1.0;
try
Expand Down Expand Up @@ -213,8 +260,6 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
// only accept positive temperature (in kelvin)
if (temp >= 0.0)
{
// set visibility flag so thermal camera can see it
item->addVisibilityFlags(0x10000000);
for (unsigned int i = 0; i < item->getNumSubItems(); ++i)
{
Ogre::SubItem *subItem = item->getSubItem(i);
Expand All @@ -223,8 +268,12 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
// normalize temperature value
float color = temp * 100.0 /
static_cast<float>(std::numeric_limits<uint16_t>::max());

// set g, b, a to 0. This will be used by shaders to determine
// if particular fragment is a heat source or not
// see media/materials/programs/thermal_camera_fs.glsl
subItem->setCustomParameter(this->customParamIdx,
Ogre::Vector4(color, color, color, 1.0));
Ogre::Vector4(color, 0, 0, 0.0));
}
Ogre::HlmsDatablock *datablock = subItem->getDatablock();
this->datablockMap[subItem] = datablock;
Expand All @@ -233,6 +282,96 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
}
}
}
// get heat signature and the corresponding min/max temperature values
else if (auto heatSignature = std::get_if<std::string>(&tempAny))
{
// if this is the first time rendering the heat signature,
// we need to make sure that the texture is loaded and applied to
// the heat signature material before loading the material
if (this->heatSignatureMaterials.find(item->getId()) ==
this->heatSignatureMaterials.end())
{
// make sure the texture is in ogre's resource path
const auto &texture = *heatSignature;
auto engine = Ogre2RenderEngine::Instance();
engine->AddResourcePath(texture);

// create a material for this item, now that the texture has been
// searched for. We must clone the base heat signature material since
// different items may use different textures. We also append the
// item's ID to the end of the new material name to ensure new
// material uniqueness in case two items use the same heat signature
// texture, but have different temperature ranges
std::string baseName = common::basename(texture);
auto heatSignatureMaterial = this->baseHeatSigMaterial->clone(
this->name + "_" + baseName + "_" +
Ogre::StringConverter::toString(item->getId()));
auto textureUnitStatePtr = heatSignatureMaterial->
getTechnique(0)->getPass(0)->getTextureUnitState(0);
Ogre::String textureName = baseName;
textureUnitStatePtr->setTextureName(textureName);

// set temperature range for the heat signature
auto minTempVariant = ogreVisual->UserData("minTemp");
auto maxTempVariant = ogreVisual->UserData("maxTemp");
auto minTemperature = std::get_if<float>(&minTempVariant);
auto maxTemperature = std::get_if<float>(&maxTempVariant);
if (minTemperature && maxTemperature)
{
// make sure the temperature range is between [0, 655.35] kelvin
Ogre::GpuProgramParametersSharedPtr params =
heatSignatureMaterial->getTechnique(0)->getPass(0)->
getFragmentProgramParameters();
params->setNamedConstant("minTemp",
std::max(static_cast<float>(*minTemperature), 0.0f));
params->setNamedConstant("maxTemp",
std::min(static_cast<float>(*maxTemperature), 655.35f));
}
heatSignatureMaterial->load();
this->heatSignatureMaterials[item->getId()] = heatSignatureMaterial;
}

for (unsigned int i = 0; i < item->getNumSubItems(); ++i)
{
Ogre::SubItem *subItem = item->getSubItem(i);

Ogre::HlmsDatablock *datablock = subItem->getDatablock();
this->datablockMap[subItem] = datablock;

subItem->setMaterial(this->heatSignatureMaterials[item->getId()]);
}
}
// background objects
else
{
Ogre::Aabb aabb = item->getWorldAabbUpdated();
Ogre::AxisAlignedBox box = Ogre::AxisAlignedBox(aabb.getMinimum(),
aabb.getMaximum());

// we will be converting rgb values to tempearture values in shaders
// but we want to make sure the object rgb values are not affected by
// lighting, so disable lighting
// Also check if objects are within camera view
if (ogreVisual->GeometryCount() > 0u &&
this->ogreCamera->isVisible(box))
{
auto geom = ogreVisual->GeometryByIndex(0);
if (geom)
{
MaterialPtr mat = geom->Material();
Ogre2MaterialPtr ogreMat =
std::dynamic_pointer_cast<Ogre2Material>(mat);
Ogre::HlmsUnlitDatablock *unlit = ogreMat->UnlitDatablock();
for (unsigned int i = 0; i < item->getNumSubItems(); ++i)
{
Ogre::SubItem *subItem = item->getSubItem(i);
Ogre::HlmsDatablock *datablock = subItem->getDatablock();
this->datablockMap[subItem] = datablock;
subItem->setDatablock(unlit);
}
}
}
}
}
itor.moveNext();
}
Expand All @@ -242,7 +381,7 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
void Ogre2ThermalCameraMaterialSwitcher::postRenderTargetUpdate(
const Ogre::RenderTargetEvent & /*_evt*/)
{
// restore item to use hlms material
// restore item to use pbs hlms material
for (auto it : this->datablockMap)
{
Ogre::SubItem *subItem = it.first;
Expand Down Expand Up @@ -441,6 +580,8 @@ void Ogre2ThermalCamera::CreateThermalTexture()
static_cast<float>(this->ambientRange));
psParams->setNamedConstant("heatSourceTempRange",
static_cast<float>(this->heatSourceTempRange));
psParams->setNamedConstant("rgbToTemp",
static_cast<int>(this->dataPtr->rgbToTemp));

// Create thermal camera compositor
auto engine = Ogre2RenderEngine::Instance();
Expand All @@ -455,17 +596,7 @@ void Ogre2ThermalCamera::CreateThermalTexture()
// {
// in 0 rt_input
// texture depthTexture target_width target_height PF_D32_FLOAT
// texture colorTexture target_width target_height PF_R8G8B8
// target depthTexture
// {
// pass clear
// {
// colour_value 0.0 0.0 0.0 1.0
// }
// pass render_scene
// {
// }
// }
// texture colorTexture target_width target_height PF_R8G8B8A8
// target colorTexture
// {
// pass clear
Expand Down Expand Up @@ -517,7 +648,9 @@ void Ogre2ThermalCamera::CreateThermalTexture()
thermalTexDef->uav = false;
thermalTexDef->automipmaps = false;
thermalTexDef->hwGammaWrite = Ogre::TextureDefinitionBase::BoolFalse;
thermalTexDef->depthBufferId = Ogre::DepthBuffer::POOL_NON_SHAREABLE;
// set to default pool so that when the colorTexture pass is rendered, its
// depth data get populated to depthTexture
thermalTexDef->depthBufferId = Ogre::DepthBuffer::POOL_DEFAULT;
thermalTexDef->depthBufferFormat = Ogre::PF_UNKNOWN;
thermalTexDef->fsaaExplicitResolve = false;

Expand All @@ -530,7 +663,7 @@ void Ogre2ThermalCamera::CreateThermalTexture()
colorTexDef->numMipmaps = 0;
colorTexDef->widthFactor = 1;
colorTexDef->heightFactor = 1;
colorTexDef->formatList = {Ogre::PF_R8G8B8};
colorTexDef->formatList = {Ogre::PF_R8G8B8A8};
colorTexDef->fsaa = 0;
colorTexDef->uav = false;
colorTexDef->automipmaps = false;
Expand All @@ -540,24 +673,7 @@ void Ogre2ThermalCamera::CreateThermalTexture()
colorTexDef->preferDepthTexture = true;
colorTexDef->fsaaExplicitResolve = false;

nodeDef->setNumTargetPass(3);
Ogre::CompositorTargetDef *depthTargetDef =
nodeDef->addTargetPass("depthTexture");
depthTargetDef->setNumPasses(2);
{
// clear pass
Ogre::CompositorPassClearDef *passClear =
static_cast<Ogre::CompositorPassClearDef *>(
depthTargetDef->addPass(Ogre::PASS_CLEAR));
passClear->mColourValue = Ogre::ColourValue(this->FarClipPlane(), 0, 0);
// scene pass
Ogre::CompositorPassSceneDef *passScene =
static_cast<Ogre::CompositorPassSceneDef *>(
depthTargetDef->addPass(Ogre::PASS_SCENE));
passScene->mVisibilityMask = IGN_VISIBILITY_ALL
& ~(IGN_VISIBILITY_GUI | IGN_VISIBILITY_SELECTABLE);
}

nodeDef->setNumTargetPass(2);
Ogre::CompositorTargetDef *colorTargetDef =
nodeDef->addTargetPass("colorTexture");
colorTargetDef->setNumPasses(2);
Expand All @@ -568,14 +684,10 @@ void Ogre2ThermalCamera::CreateThermalTexture()
colorTargetDef->addPass(Ogre::PASS_CLEAR));
passClear->mColourValue = Ogre::ColourValue(0, 0, 0);
// scene pass
Ogre::CompositorPassSceneDef *passScene =
static_cast<Ogre::CompositorPassSceneDef *>(
colorTargetDef->addPass(Ogre::PASS_SCENE));
// set thermal camera custom visibility mask when rendering heat sources
passScene->mVisibilityMask = 0x10000000;
colorTargetDef->addPass(Ogre::PASS_SCENE);
}

// rt_input target - converts depth to thermal
// rt_input target - converts to thermal
Ogre::CompositorTargetDef *inputTargetDef =
nodeDef->addTargetPass("rt_input");
inputTargetDef->setNumPasses(2);
Expand Down Expand Up @@ -632,10 +744,10 @@ void Ogre2ThermalCamera::CreateThermalTexture()
auto channels = node->getLocalTextures();
for (auto c : channels)
{
if (c.textures[0]->getSrcFormat() == Ogre::PF_R8G8B8)
if (c.textures[0]->getSrcFormat() == Ogre::PF_R8G8B8A8)
{
this->dataPtr->thermalMaterialSwitcher.reset(
new Ogre2ThermalCameraMaterialSwitcher(this->scene));
new Ogre2ThermalCameraMaterialSwitcher(this->scene, this->Name()));
c.target->addListener(this->dataPtr->thermalMaterialSwitcher.get());
break;
}
Expand Down Expand Up @@ -699,14 +811,14 @@ void Ogre2ThermalCamera::PostRender()
this->dataPtr->thermalImage, width, height, 1, "L16");

// Uncomment to debug thermal output
// igndbg << "wxh: " << width << " x " << height << std::endl;
// std::cout << "wxh: " << width << " x " << height << std::endl;
// for (unsigned int i = 0; i < height; ++i)
// {
// for (unsigned int j = 0; j < width; ++j)
// {
// igndbg << "[" << this->dataPtr->thermalImage[i*width + j] << "]";
// std::cout << "[" << this->dataPtr->thermalImage[i*width + j] << "]";
// }
// igndbg << std::endl;
// std::cout << std::endl;
// }
}

Expand Down
Loading

0 comments on commit d9d5374

Please sign in to comment.