contrib/mikeroberts3000/jupyter/02_rendering_hypersim_meshes_with_pytorch3d.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8908b85a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import torch\n",
    "import pytorch3d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b9c79863",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data structures and functions for rendering\n",
    "from pytorch3d.structures import Meshes\n",
    "from pytorch3d.renderer import (\n",
    "    look_at_view_transform,\n",
    "    FoVPerspectiveCameras,\n",
    "    RasterizationSettings, \n",
    "    MeshRenderer, \n",
    "    MeshRasterizer\n",
    ")\n",
    "\n",
    "from pytorch3d.renderer.mesh.shader import (\n",
    "    ShaderBase,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1c492744",
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.linalg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d7b2f150",
   "metadata": {},
   "outputs": [],
   "source": [
    "# taken from https://github.com/facebookresearch/pytorch3d/blob/main/pytorch3d/renderer/mesh/shader.py\n",
    "\n",
    "class HardDepthShader(ShaderBase):\n",
    "    \"\"\"\n",
    "    Renders the Z distances of the closest face for each pixel. If no face is\n",
    "    found it returns the zfar value of the camera.\n",
    "    Output from this shader is [N, H, W, 1] since it's only depth.\n",
    "    To use the default values, simply initialize the shader with the desired\n",
    "    device e.g.\n",
    "    .. code-block::\n",
    "        shader = HardDepthShader(device=torch.device(\"cuda:0\"))\n",
    "    \"\"\"\n",
    "\n",
    "    def forward(self, fragments, meshes, **kwargs) -> torch.Tensor:\n",
    "        cameras = self.cameras\n",
    "\n",
    "        zfar = kwargs.get(\"zfar\", getattr(cameras, \"zfar\", 100.0))\n",
    "        mask = fragments.pix_to_face[..., 0:1] < 0\n",
    "\n",
    "        zbuf = fragments.zbuf[..., 0:1].clone()\n",
    "        zbuf[mask] = zfar\n",
    "        return zbuf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c713826c",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3c422b89",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata_camera_parameters_csv_file = os.path.join(\"..\", \"metadata_camera_parameters.csv\")\n",
    "\n",
    "scene_name  = \"ai_037_002\"\n",
    "camera_name = \"cam_00\"\n",
    "frame_id    = 0\n",
    "\n",
    "camera_dir = os.path.join(scene_name, \"_detail\", camera_name)\n",
    "img_dir    = os.path.join(scene_name, \"images\")\n",
    "\n",
    "# we need access to the Hypersim meshes in order to execute parts of this notebook\n",
    "mesh_dir = \"/Volumes/lacie/hypersim/evermotion_dataset/scenes/\" + scene_name + \"/_detail/mesh\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "08283695",
   "metadata": {},
   "outputs": [],
   "source": [
    "mesh_vertices_hdf5_file = os.path.join(mesh_dir, \"mesh_vertices.hdf5\")\n",
    "mesh_faces_vi_hdf5_file = os.path.join(mesh_dir, \"mesh_faces_vi.hdf5\")\n",
    "\n",
    "with h5py.File(mesh_vertices_hdf5_file, \"r\") as f: mesh_vertices   = f[\"dataset\"][:]\n",
    "with h5py.File(mesh_faces_vi_hdf5_file, \"r\") as f: mesh_faces_vi   = f[\"dataset\"][:]\n",
    "\n",
    "mesh_positions_world = mesh_vertices\n",
    "\n",
    "camera_positions_hdf5_file    = os.path.join(camera_dir, \"camera_keyframe_positions.hdf5\")\n",
    "camera_orientations_hdf5_file = os.path.join(camera_dir, \"camera_keyframe_orientations.hdf5\")\n",
    "\n",
    "with h5py.File(camera_positions_hdf5_file,    \"r\") as f: camera_positions    = f[\"dataset\"][:]\n",
    "with h5py.File(camera_orientations_hdf5_file, \"r\") as f: camera_orientations = f[\"dataset\"][:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "05827285",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAItCAYAAAAJ/GG4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmYUlEQVR4nO3de6yu2V0X8O/ae5/LXDttB6bTmQJFC14IJk1tqxgt1kC5xGoCBDSxYBM0QTRCIiXGYNQ/MBoRg2IaqRaDXESUhqDYQBFRKbRikIvKWFJmpu2007kxnXPZ+32Xf5y3cqyzfmfmmbP3es7Zn09yMnu/630u+917v7P291m/59d67wEAmGlv9gkAAJiQAADTmZAAANOZkAAA05mQAADTHcw+AQA4zb74C2/rH39scyLHev8vXfqJ3vubTuRgz5MJCQBM9PHHNvn5n/iMEznW/r2/fveJHGgBExIAmKgn2WY7+zSms4YEAJhOQgIAU/VsuoREQgIATGdCAgBM55INAEx0ZVGrRrcSEgBgOgkJAEym7FdCAgCsgIQEACbq6dl0a0gkJADAdBISAJhMlY2EBABYAQkJAEzUk2wkJBISAGA+CQkATGYNiYQEAFgBCQkATNQT9yGJhAQAWAEJCQBMppONhAQAWAETEgBgOpdsAGCinu7GaJGQAAArICEBgJl6shGQSEgAgPkkJAAwUY+y30RCAgCsgIQEAKZq2aTNPonpJCQAwHQSEgCYqCfZqrKRkAAA80lIAGAya0gkJADATmvtHa21j7bWfvmqx/5Oa+1/tNZ+qbX2r1trd1019q2ttQdaa/+ztfbFVz3+pt1jD7TW3vZcjm1CAgAT9VxJSE7i33Pwz5K86VMee3eSz+u9f36S/5XkW5OktfZ7knx1kt+72+Yftdb2W2v7Sf5hki9J8nuSfM3uuSUTEgAgSdJ7/5kkj33KY/++9360+/Tnkty/+/jNSX6g936p9/4bSR5I8trdvwd67x/ovV9O8gO755asIQGAybb9xNaQ3N1ae99Vn7+99/7257H9n03yg7uP78uVCconPbR7LEke/JTHX3etHZuQAMDp8Wjv/TVLNmyt/dUkR0m+7/qe0hUmJABAqbX2tUm+PMkbe++fvGvKw0lecdXT7t89luLxIRMSAJjok4ta16q19qYkfyXJH+m9P3PV0LuS/IvW2t9L8vIkr0ry80lakle11l6ZKxORr07yp651HBMSACBJ0lr7/iRvyJW1Jg8l+bZcqao5l+TdrbUk+bne+5/vvf9Ka+2HkvxqrlzK+Ybe+2a3n7+Q5CeS7Cd5R+/9V6557N9OXgCAk/a7P/9c/94fu/dEjvXaz/zg+5euITluyn4BgOlcsgGAyU6w7He1JCQAwHQSEgCYaO1VNidFQgIATCchAYCpWjZdPuAVAACmk5AAwEQ9yVY+4BUAAOaTkADAZKpsJCQAwApISABgot5V2SQSEgBgBUxIAIDpXLIBgMm2FrVKSACA+SQkADDRleZ68gGvAAAwnYQEAKZS9ptISACAFZCQAMBEmutd4RUAAKaTkADAZJvuPiQSEgBgOgkJAEzU09yHJBISAGAFJCQAMNnWfUgkJADAfBISAJhIL5srvAIAwHQmJADAdC7ZAMBEPc2N0SIhAQBWQEICAJNprichAQBWQEICABP1nmzcGE1CAgDMJyEBgKlatlFlIyEBAKaTkADARD3WkCQSEgBgBSQkADCZ5noSEgBgBSQkADBRT8tWLxsJCQAwn4QEACazhkRCAgCsgAkJADCdSzYAMFFPsnVjNAkJADCfhAQApmrZaK4nIQEA5pOQAMBE1pBc4RUAAKaTkADAZNaQSEgAgBWQkADARL03a0giIQEAVkBCAgCTbSQkEhIAYD4JCQBM1JNsVdnUE5I3fNG39yU77fvjF7YfjMe2xdje0fhU2mZ8Lu1oO97nptjn4Xi7ti22q8aKc0l1Ltvj2K4YK7ZrC7crj9eLH7NN8c2ttgNY6N9+6LvMDiaQkADAVM0aklhDAgCsgIQEACa60svGVSIJCQAwnQkJADBdecnm6Pb94Vhv43hp/1JRUVEpEqvDW8dzp72j8XZ7R0VVT1G5s3emOF5RgZOi8KOqsmlVtUxRoVJWvWyK+WZZSTM+l37SlTv71ddQvNjVeVZjABNs5ANeAQBgPotaAWCinmZRayQkAMAKSEgAYLKtfMArAADMJyEBgIl6TzbWkNQTkm3VJK+qxjwzLhfev3z9Sy6Pzo/Ps22XNewrx84Uxysb/RXnUpX9Ho1f7LKZX1kuXJT2VuXJx9Dor590o7+ymV/19RXfXABeEAkJAEymysYaEgBgBSQkADDRlfuQyAe8AgDAdBISAJhsUzVzOyUkJADAdHXZ70ExYyuGqkthm6KL7v7lohxzoRMvCa46Dx8WnYfL8t1l59KKbr9l5+GDYruqlLjqvlscb1Wdh/eK7Q7G5exLj6fzMNCjyiaRkAAAK2BCAgBMZ1ErAEyl7DeRkAAAKyAhAYDJtsp+JSQAwHzXKPstBsuy36JLcLHd5ux48DhKgjfnlp3LXtW1tywJLr6+w6pTblH6Wm13HJ2Ht1UpcbHdwfUv7a1Kl0+883BVLlx1Fx7tszqPJfsDVqv3ZKPsV0ICAMxnDQkATKbKRkICAKyAhAQAJuppbh0fCQkAsAISEgCYzH1IJnT7LdftFPs8LMpwq33uXyqOV9icGx9vc3683d7hsrHN0lLiM8VYUfZbdh6uSmZPuvNwVWa8v6zzcC+/vvGLdpKdhxeXEes6DNygJCQAMFFPrCGJNSQAwApISABgMvchkZAAACtgQgIATGdCAgAz9Ss3RjuJf9fSWntHa+2jrbVfvuqxl7TW3t1a+/Xdf1+8e7y11v5Ba+2B1tovtdZefdU2b9k9/9dba295Li/D8m6/VWnv4pLgYsOFpcRVR9/9S8s6CG/PFsc7X5TvXh5vd9sj47LLS3cW+yzKd+ty4WXdfqt9bo+GQ3W5cPE1tKIada8oca06D6coCd47Gv8wlZ2Hi68vZVnzs4+tputwovMwnC7/LMl3Jfneqx57W5Kf7L1/e2vtbbvPvyXJlyR51e7f65J8d5LXtdZekuTbkrwmV4qI3t9ae1fv/fHqwBISAJio58qN0U7i3zXPpfefSfLYpzz85iTv3H38ziR/4qrHv7df8XNJ7mqt3Zvki5O8u/f+2G4S8u4kb7rWsVXZAMDpcXdr7X1Xff723vvbr7HNPb33D+8+/kiSe3Yf35fkwaue99DusdHjJRMSAJjsBG+M9mjv/TVLN+6999basvUO1+CSDQBQeWR3KSa7/3509/jDSV5x1fPu3z02erxkQgIAE33y1vFrqLIZeFeST1bKvCXJj171+J/ZVdu8PsmTu0s7P5Hki1prL95V5HzR7rGSSzYAQJKktfb9Sd6QK2tNHsqVaplvT/JDrbW3Jvlgkq/aPf3Hk3xpkgeSPJPk65Kk9/5Ya+1vJvmF3fP+Ru/9UxfK/n/KCUkvRpeW7y7uBLxwu7okeHye+0WJbmV7pjpeseEj46HD26uy3/F25VhRoru3tPNwuc+iXLjo5Lx3WGy3WXaeZdfeg6K7cFlmXIwV+xx1Hh6VA1/rWNe763CStP0TLhcuOi7DzWotzfV6718zGHrjszy3J/mGwX7ekeQdz+fYLtkAANO5ZAMAE/W8oPUdNw0JCQAwnYQEACZ7LndRvdlJSACA6SQkADBTX0+VzUx1t9+iI+zSUts1lf1WY1VH30orSl/7/njs8c9dFlZVxzv75Pj7tzk/3q4qta27Cxf7XFpmfOaEOw8XY63oBHy9Ow/vVaW9Bwu7Di8tJT6Gbr9VCXX2q1p9nYfhZuWSDQAwnUs2ADDRJ28df9pJSACA6SQkADCZhERCAgCsgIQEACZy6/grrlH2Ox476bLfxSXBRalt31tW3lqpOiQfh80tRZnqE+PtLt1RdDquSnuXdhde3JX4hDsPVyXBZZnxws7Dg7LfXpQYl12HB/tL6tLe7a1FR+KiDHf/yYvFuSwsJV5aLqzzMNzQJCQAMFmXkFhDAgDMJyEBgMk015OQAAArICEBgIm65npJJCQAwAosLvvdFuW01aWwsrS3LNGtxsZle0v3WZWi3iie+uxqdPya7V8eb7V3aWlJcFFmXB2v6jxcldqWpcTV2MLy3aI6tPoaRt15j6PrcA6KH/j98de9Lba7fOcdw7F+MN7nrR98ajhWdSzuS0t7j6PzcFFiXZYg6zzMs1BlIyEBAFbAGhIAmMqdWhMJCQCwAiYkAMB0LtkAwGQWtUpIAIAVKBOSTVH2exwddpeX/Vbnsux4+cTNPVvd3DauU93cNt5u71LR+bUqCS5KezeXq/LdpeXCy8ZaURJcdkGuOg+fGW83LPstOxlX3YqLLsFFt99elf2eKX7BFv6aPPOZdy7bsHDrbxSlxGvqPFx2F67KmnUevln1uDFaIiEBAFbAGhIAmKnXwdlpISEBAKaTkADAZNulC7JuIhISAGA6CQkATNTjPiTJtbr9nh2PleW0i0t0i5NZWi5clv1Wq4hOb3jUbitqX+8Yv2aHF8Y/Tq0oF967WJQSLy0XLkuCl3UJ3hRj+4tLiQfnUXQdrrrhLu1kfDN45pXLSolX1Xl4T+dhTi8JCQBMpblecppjAABgNSQkADCZ+5BISACAFZCQAMBkqmwkJADACtRlv+eKkrdiMre0w25Zhrt0u6JcOGXZL8/mvk9/Yjj2xIXzw7GLF8Y15IcXxz+Gm4vjb3xVLrxXdB4uuwSXnYfH222ucylx2a24KN+tuwQv3K4ofW03QQNanYcHdB7mhLlkAwAT9e6STeKSDQCwAhISAJjMjdEkJADACkhIAGAyN0aTkAAAK1AmJJuzVTnteGhxh91jGNs7KErlyrLfotXxKfb77/7gcOyJw1uHYx++MC6tfPSZ24ZjT184Nxy7+PDtw7HNLePv7eaW4VD2DoufpaIkeL8qF75UHG9Qilt3K672t3S7paXESzsP39x/Di7tPFy57QNPDsdOvPPwQXEfhqUlyKe887AqGwkJALAC1pAAwEQ9TUISCQkAsAISEgCY7OZeVfXcSEgAgOkkJAAwk142Sa4xIelVt9+lHXYPis6h+8tKdPeL7faLst+Dg3HHyk3GJayn2Wff8rHh2F23PzMc+2s//xXDsT/5Bb8wHHvwwouHY+97+FXDsaW2Z6qxopT41qL8teg8PCrvLcuIq/Ldhdv14p3g6PyyN8o7fnP8u6fz8PP3ic9+0aLtTrzzcFUuXN39a2np8tJ9sjoSEgCYzSISa0gAgPlMSACA6VyyAYDJLGqVkAAAKyAhAYDJqmKh06Iu+61Ke88UJX1nx/V3S8t3qxLdg2K7s8V2Zw/G7UgfH46cbr/33MPDsbv2Low3LNLIP3f3fxyO/fdL9w7H3pfrX/a7VJW2bs4X5cLnB49Xpa9V1+EFJcZJjmWF/299xrIA9vaHlnYXrsqFx8erOjzfDKrOw614zfYvjd9Xzz3y9HDsWDoPV/8vOobjMYeEBAAm6rGGJLGGBABYAQkJAMzUU1/3PSUkJADAdBISAJhMlY2EBABYgTIhOffIegKUqpFnNXZp4fGqmdp97xl3tX34C5d1CX7Ze8c1mR953dlF+6y0w+IrfGJ8vD/7Y19/3c/lTf/mm6/7Pm902+JXb1t0zD4qug7vX1zWCbgqJW7jyvnFnr6/upZ+DJ2HzxSvS9FduCozvtk7Dx+9aFCvnmTvcnHbh4vjH5jj6Dxcdpev9jmDhERCAgDMt54IBABOpeY+JJGQAAArICEBgNmsIZGQAADzmZAAANOd6ks29/2HcVHw3mFRgnZmPI/79PeP6ycPLozL4bb74wVN9/zCeJ+9mFI+9IWn+tvLp1jSdThJNrcs6zxcj42P146hGnNp5+HKnR8sSomLcui6XLgYW1mV6kg/KF7r8+P3pF5+7UUn4KNlJcF9b0UvaNdcL5GQAAAr4E9oAJjNolYJCQAwn4QEAKazhkRCAgBMJyEBgNmsITndE5L9ovPk5pbxS7N3eVwuVpX2bs7tF9uNz+WoKAlOUSp2/3vG+/zNL7sJwrEjEedJWNp5OFXn4aKDcFUSXJYSFx2Lj8NTn7nsd+iOB6ty4eLrW9hduNqu0g/G57Ipynf3ii66bb94zbZVZ+WifPegOF5RSlx2CWaKUz0hAYBVkJBYQwIAzCchAYCZesrL76eFhAQAmE5CAgCTdWtIJCQAwHynOiHpe1WJXVFKVpSnVR0bq66UR+fHJcH7l8bbbc+O55TVJcneihLC28blwntVmWexz3Pnrn9N5oWH7rju++RkbM4VnYfPjberyjj3q5LghWXGx9J5+BXj39kznxhvd/DM+Gs/U4wtLSXeFq91vd3466veA/eK4/Wjhfs8HNdDt8OVRRIrO50ZJCQAwHQmJADAdKf6kg0ArIKyXwkJADCfhAQAJitqAk4NCQkAMN2pTki2Rffdqrxwe348j/vwHxzXLL78Zy+Oj1eU9lZdN6vS5b6i6ebr7/vgdd/nex76vOu+T9atj39lc3RL8SdmMVZ1EN6vOg9XpcQLq9yrZQSbs8VgMVSdy964wj8H47erbIv3pKpUuno9U7wH7he3Wrgp9Cj7jYQEAFiBU52QAMB8TZVNJCQAwApISABgNmtIJCQAwBWttb/cWvuV1tovt9a+v7V2vrX2ytbae1trD7TWfrC1dnb33HO7zx/YjX/WCzm2CQkAzNZP6F+htXZfkr+Y5DW9989Lsp/kq5P87STf0Xv/nUkeT/LW3SZvTfL47vHv2D1vsVN9yebhPzwu0X3FT10Yjn3oD54fjt33M+PtUpQSH91+Zji2d7no9luU3z38R4oayYy7YB6HV985Lvt9/S3/ezh2pqghfE+U/fLCbc8WJf5nx9u1W4tS4qIkuCp97fvLSom3Vafjquy3GDu4uPAaworWZu594tJ48Ohk3wNvIAdJbmmtHSa5NcmHk/zRJH9qN/7OJH89yXcnefPu4yT54STf1VprvfdFPzwSEgCY7eQSkrtba++76t/X/99T6P3hJH83yW/mykTkySTvT/JE7/2Td615KMl9u4/vS/Lgbtuj3fNfuvQlONUJCQCcMo/23l/zbAOttRfnSurxyiRPJPmXSd50UidmQgIAM/Ws5T4kfyzJb/TeP5YkrbUfSfIFSe5qrR3sUpD7kzy8e/7DSV6R5KHW2kGSFyX5+NKDu2QDACRXLtW8vrV2a2utJXljkl9N8p4kX7F7zluS/Oju43ftPs9u/KeWrh9JTEgAgCS99/fmyuLU/5rkv+fKHOHtSb4lyTe11h7IlTUi37Pb5HuSvHT3+DcledsLOb5LNgAwWVvJjdF679+W5Ns+5eEPJHntszz3YpKvvF7HNiEZ2O6Pw6N7/9O4lGx7MN5uL+MS1kd+/7jstzreI6+tSnuvvyqMa8dwCfTzz45LrE/aWt4wjsM6Ll/fWKpu2puiu/DmlvF2e4fj7c4+Pj7gtngnPyjuRNAWVr7uHY3Ps+r2u9T+0+Na6Xap+CKKLsFtcwwnygtiQgIAs93Ef/A8V9aQAADTmZAAANOZkAAA01lDAgCT3cyL5p8rE5KBD3/BuPHePT9frPgufqiODsYVMS9773if1bncDCuh7qm6jaXobnbS1vJSr6gi5jjeRE9zxc92XGy3KlU10P6lcfXK3tF4rBVNRI9D3zvFP2grZUICALOd5pn4jjUkAMB0EhIAmKlnPZeEJ5KQAADTSUgAYDYJiYQEAJhPQrLAI69dUSnqQs2Kbq6H4/irzo/mKhxHk7zyeFXnzlPAfUgkJADACkhIAGA2CYmEBACYz4QEAJjOJRsAmM0lGwkJADCfhOSU6kWNmarLgRv8L5iqjHPp91z1+LO7/TfHf+s9/RknXE97DNqmeP+oxhZ29O1nxp3S2+Gm2K74X1w1dsJaV/abSEgAgBVYzxQRAE4rcaOEBACYT0ICALNZQyIhAQDmk5AAwGSqbFY2Iam+Idb7XF/H0e3Xt+iEVG9c1TfhGN7wyjfR4lyqH7/9S+PB7dllX0R1vL2jYrtxtWm5zzWV9h7dMh7r++Mvon9svN32zDhcr34myo6+RUnw3sXim1SVsx+Ot6vGmGNVExIAOJUkJNaQAADzSUgAYCZ3ak0iIQEAVkBCAgCzSUgkJADAfKtKSJT2nhzdfp+/qgR0/+L4Vducf/5/+lzv/SVJX/jnx3Gcy0nvsx8U+9wu3K6wd/n6ly4fh83Z2WcAv21VExIAOJXWM0+dxiUbAGA6CQkATKbsV0ICAKyACQkAMJ0JCQAwnTUkXDc3wyXQqlxz6TXeqnvt5tyCkuCF+6u2O46S++q1XLrd0u9B2yz72qvX7OwTVTvj4lyKMuNWdK5VonuTuxneQF8gCQkAMJ2EBABm0lwviYQEAFgBCQkAzCYhkZAAAPNJSABgNgmJCQnPTzvFrYCr8tDl5ajPf5tt0YF2yf5eiOM43poW9y0v7b3+Y2uyOVd1Mx63xd6/NP6B6ftFOfttZ4ZjZx4/Go5lf3wRoO+ro14bExIAmKhlXRPxWawhAQCmMyEBAKZzyQYAZnPJRkICAMwnIQGAmdw6PsmECcn5j4/HtsXZ9HElWXqV8xRVe0e3FNvxrO7+t+eGY3tF9d33/8iXDse+7+DLhmP/+e/94+d0XtdLL0pqK63oUFuWCxddaJdsc+9/GdeNbsaVk6WPvbr42qrfy/2i8/DF8T5v/fCy17I6l+q9pXIc5btlaW8xtn95PHYsnYCLX4X9S+PBgwvLasHbZrzPvaOF9dBH43NpxRhzSEgAYDYJiTUkAMB8EhIAmE1CIiEBAOaTkADAZKpsJCQAwAqceEJSdQdtVQlh1VmzGKsmnQfPjMe2x1BG91n/6tHhWD87/lb0qsVuWfJclE8ejDc8Kjpr9v1xbe/h7eNvYFWuuXc0/i79gW/+88OxF9+5rDy0KgXve+PBp37HwtLD8gf0+Zf9vuy94/PYFr9D1dddlVxWG+4V5c6L/+IrtqtKy3v17am+BcVrVr5fFcc7/1jRkXnhj1FVtl1137304mXHW5XqfwA3i1PwJV6LhAQAmM4aEgCYqUdCEgkJALACEhIAmEyVjYQEAFgBExIAYLoTv2Rz4dPHY7d9aFlmVVVOHt7x/Msqk+TlP3txOHb2Q08t2melXR7XM/ZzRb1fVUK4V5QeFt0z958Zn8vmlvGPzJmnxttdflHVynk8tLew7HLZd70urazKQ3M4HtoW+9wrOriOfOR1478jqtj3094/HvzwHxq/YvvjX4XjUZX4F9/zvcOiA+3S7rsLbzewVNm5fGFX86XKavXi1/nwjvEvyvlHxu8Re4envPuuSzYSEgBgPotaAWAyi1olJADACkhIAGA2CYmEBACYT0ICADO5dXySlU1IPvHyce3aLY8s22fZ0bf66osfjsOX3TEcO/OR37r2ST2bzbj2cO9SUVNaWdghc29blE8uO5Ocf7AY3F8W1D39OXct2q7qnnx4a1H+emlZbeXS7Uaqbr9HRdfXvaKj72f++Ljkslps99RnjGuan/yc8XaVM59Y1il3cflusV1Vsn2juPWR6/9/ulZ0Xa66d29uHb+DtN8afyNa8f5YOihq9S8vfF/l2KxqQgIAp03LsdxK5oZjDQkAMJ2EBABms4ZEQgIAzCchAYDJ3KlVQgIArMANk5BcuGf2GUy0tORtYdnvsShKbZd+fec/emk41veK0t47x7Wcl29f1pW4dJ2/DfuXxjssxy4s7KZavJa3PTLe55Ovqtojjx3eVpUuj7fbFoertjvzdFGmena83f6CTs3XUnUur8bKfRZ/dlYl1pVWvLdUncSX/i60omS9dFjVJ6/s7/EVvV3PsrLvCABwGpmQAADT3TCXbADgpuWSjYQEAJhPQgIAM3Vlv4mEBABYgTIhObhwUqdxbeceG08fL71kXA9XdfKsyuGWOpZOwJWqtLcqtT3pkuCl51nY/8S47nJz+7nieOOhquP0uceey1ldH/f+p6JN9YrsF6d57rFbr/vxqtLX8i/MYuzolvHYp73/6fHxiq7Yi3/3Fv4uLC0JfuJ33T7e5974PLdnxgc8qLo1Ly3fPQ28NBISAGA+a0gAYDJrSCQkAMAKSEgAYDYJiYQEAJhPQgIAk1lDco0JyflHb4xX6FjOsyqjKw7XF5btHYs1lfYeR2fN4rWuyiCfftn4x/5G+Jl//HOLOtUVuRFey2vZu1y0CS662lbdcMvfkxP+nX3p+8Yds5/+nLuGY3uXi5Lgs+Pf9arsd++w+KXdLux4zg1FQgIAM/VYQxJrSACAFZCQAMBsEhIJCQAwnwkJAJAkaa3d1Vr74dba/2it/Vpr7Q+01l7SWnt3a+3Xd/998e65rbX2D1prD7TWfqm19uoXcmwTEgCYqOVK2e9J/HsOvjPJv+u9/64kvy/JryV5W5Kf7L2/KslP7j5Pki9J8qrdv69P8t0v5HWoy36fOL2lVhdfemY4dv7Rw0X7vPzyO4djZz/01KJ9HktX0aVlicdR2rvQhXvG3X5v9J/rG/38T4XjKO2tugtX9pbdiuC23xh3J79w/7hLcFnaW5RKV/r+fjE23m7/wrisWSnx/6+19qIkfzjJ1yZJ7/1yksuttTcnecPuae9M8tNJviXJm5N8b++9J/m5Xbpyb+/9w0uOv57/gwDAadVP6F9yd2vtfVf9+/qrzuKVST6W5J+21n6xtfZPWmu3JbnnqknGR5Lcs/v4viQPXrX9Q7vHFlFlAwCnx6O999cMxg6SvDrJN/be39ta+8789uWZJEnvvbd2PPeVlZAAwGSt9xP5dw0PJXmo9/7e3ec/nCsTlEdaa/cmye6/H92NP5zkFVdtf//usUVMSACA9N4/kuTB1trn7h56Y5JfTfKuJG/ZPfaWJD+6+/hdSf7Mrtrm9UmeXLp+JHHJBgDmWtet478xyfe11s4m+UCSr8uV8OKHWmtvTfLBJF+1e+6PJ/nSJA8keWb33MVMSACAJEnv/b8lebY1Jm98luf2JN9wvY5dTkjOPlV0ujzNFjb0rToBX3zFi4Zj5x98stjpwtLeytKS4BN26e5x19uzTx6d4JmcrJv5a+PGsHepKJmt3pIOiw7JRbnwUpuXFOXJh+v6/9vxLBO9sVhDAgBM55INAMwmIZGQAADzSUgAYDJrSCQkAMAKSEgAYDYJST0hOfitZV1t1+Q53Cp39TYvvnX2KazSmadv/J/PJfae1qWU47e9Zdzx/Dh+93pxt4GFNzDgBuOSDQAwnUs2ADBTt6g1kZAAACsgIQGA2SQkEhIAYD4JCQBM1GINSXKNCcn+Jy6d1Hlcm0pH4EZxg3TMruw9fXn2KTwnbVv8z+Fw3Bm7bfxPZW0kJAAw2w0yUT1O1pAAANNJSABgMmtIJCQAwApISABgph73IYmEBABYgTIhac+sqOwXgOdvs2wzHXZPVlOFLCEBAOazhgQAZrOGREICAMxnQgIATOeSDQBM5sZoEhIAYAXqst8Lyn4B4Fj1aK4XCQkAsALWkADAZNaQSEgAgBWQkADAbBISCQkAMJ+EBAAmarGGJLnGhKRfUvYLABw/CQkAzNS7+5DEGhIAYAUkJAAwmTUkEhIAYAUkJAAwm4REQgIAzFeX/V64eFLnAQCcYi7ZAMBkFrW6ZAMArICEBABm6km2IhIJCQAwnYQEAGYTkEhIAID56rLfy4cndR4AcGqpspGQAAArYA0JAMzWRSQSEgBgOgkJAExmDYmEBABYAQkJAMzU4z4kuVbZ7+HlkzoPAOAUk5AAwEQtSVNlYw0JADCfCQkAMJ1LNgAw23b2CcwnIQEAppOQAMBkFrVeY0Ly7u2/bCd1IgDA6SUhAYCZ3BgtiTUkAMAKSEgAYKqeWEMiIQEA5pOQAMBkTUAiIQEA5pOQAMBs1pBISACA+SQkADBTT5peNhISAGA+CQkAzGYNiYQEAJhPQgIAswlIJCQAwHwmJADAdC7ZAMBkzaJWCQkAMJ+EBABmk5BISACA+SQkADBTT+LW8RISAGA+CQkATNTSVdlEQgIArICEBABmk5BISACA+SQkADCbhERCAgDMJyEBgJnchySJhAQAWAEJCQBM5j4kEhIAYAVMSACA6VyyAYDZXLKRkAAA80lIAGCqLiGJhAQAWAEJCQDM1CMhiYQEALhKa22/tfaLrbUf233+ytbae1trD7TWfrC1dnb3+Lnd5w/sxj/rhRzXhAQAZtue0L/n5i8l+bWrPv/bSb6j9/47kzye5K27x9+a5PHd49+xe95iJiQAQJKktXZ/ki9L8k92n7ckfzTJD++e8s4kf2L38Zt3n2c3/sbd8xexhgQAJjvBW8ff3Vp731Wfv733/varPv/7Sf5Kkjt2n780yRO996Pd5w8luW/38X1JHkyS3vtRa+3J3fMfXXJiJiQAcHo82nt/zbMNtNa+PMlHe+/vb6294UTPKiYkADDfOqpsviDJH2+tfWmS80nuTPKdSe5qrR3sUpL7kzy8e/7DSV6R5KHW2kGSFyX5+NKDW0MCAKT3/q299/t775+V5KuT/FTv/U8neU+Sr9g97S1JfnT38bt2n2c3/lO9L59ZSUgAYKaeZLuKhGTkW5L8QGvtbyX5xSTfs3v8e5L889baA0key5VJzGImJADA/6P3/tNJfnr38QeSvPZZnnMxyVder2OakADAVHrZJNaQAAArYEICAEznkg0AzOaSjYQEAJhPQgIAs0lIJCQAwHwSEgCYaf03RjsREhIAYDoJCQBM1ZO+nX0S00lIAIDpJCQAMJsqGwkJADCfhAQAZlJlk0RCAgCsgIQEAGazhkRCAgDMJyEBgNkkJBISAGA+ExIAYDqXbABgqu6STSQkAMAKSEgAYKaeZKu5noQEAJhOQgIAs1lDIiEBAOaTkADAbBISCQkAMJ+EBACm6slWQiIhAQCmk5AAwEw96d19SCQkAMB0EhIAmM0aEgkJADCfhAQAZnMfEgkJADCfCQkAMJ1LNgAwU+/JVtmvhAQAmE5CAgCzWdQqIQEA5pOQAMBk3RoSCQkAMJ+EBACm6taQREICAKyAhAQAZurRXC8SEgBgBSQkADBbV2UjIQEAppOQAMBEPUm3hkRCAgDMJyEBgJl6t4YkEhIAYAVMSACA6VyyAYDJLGqVkAAAKyAhAYDZLGqVkAAA87Wu5TEATNNa+3dJ7j6hwz3ae3/TCR3reTEhAQCmc8kGAJjOhAQAmM6EBACYzoQEAJjOhAQAmO7/AKHXpE3ptx03AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "camera_position_world = camera_positions[frame_id]\n",
    "R_world_from_cam      = camera_orientations[frame_id]\n",
    "\n",
    "t_world_from_cam = np.matrix(camera_position_world).T\n",
    "R_cam_from_world = np.matrix(R_world_from_cam).T\n",
    "t_cam_from_world = -R_cam_from_world*t_world_from_cam\n",
    "\n",
    "M_cam_from_world = np.matrix(np.block([[R_cam_from_world,       t_cam_from_world],\n",
    "                                       [np.matrix(np.zeros(3)), 1.0]]))\n",
    "\n",
    "#\n",
    "# load per-scene Hypersim camera intrinsics data\n",
    "#\n",
    "\n",
    "df_camera_parameters = pd.read_csv(metadata_camera_parameters_csv_file, index_col=\"scene_name\")\n",
    "df_ = df_camera_parameters.loc[scene_name]\n",
    "\n",
    "# divide image width and height by k to make rendering go faster\n",
    "k = 16\n",
    "width_pixels  = int(df_[\"settings_output_img_width\"] / k)\n",
    "height_pixels = int(df_[\"settings_output_img_height\"] / k)\n",
    "\n",
    "if df_[\"use_camera_physical\"]:\n",
    "    fov_x = df_[\"camera_physical_fov\"]\n",
    "else:\n",
    "    fov_x = df_[\"settings_camera_fov\"]\n",
    "\n",
    "fov_y = 2.0 * np.arctan(height_pixels * np.tan(fov_x/2.0) / width_pixels)\n",
    "\n",
    "M_cam_from_uv = np.matrix([[ df_[\"M_cam_from_uv_00\"], df_[\"M_cam_from_uv_01\"], df_[\"M_cam_from_uv_02\"] ],\n",
    "                           [ df_[\"M_cam_from_uv_10\"], df_[\"M_cam_from_uv_11\"], df_[\"M_cam_from_uv_12\"] ],\n",
    "                           [ df_[\"M_cam_from_uv_20\"], df_[\"M_cam_from_uv_21\"], df_[\"M_cam_from_uv_22\"] ]])\n",
    "\n",
    "M_cam_from_uv_canonical = np.matrix([[np.tan(fov_x/2.0), 0.0,               0.0],\n",
    "                                     [0.0,               np.tan(fov_y/2.0), 0.0],\n",
    "                                     [0.0,               0.0,               -1.0]])\n",
    "\n",
    "# PyTorch3D seems to have problems with the non-standard perspective projection matrices\n",
    "# found in Hypersim, so we construct a matrix to transform a camera-space point from its\n",
    "# original position to a warped position, such that the warped position can be projected\n",
    "# with a standard perspective projection matrix. This matrix completely accounts for the\n",
    "# non-standard Hypersim camera parameters.\n",
    "M_warp_cam_pts_ = M_cam_from_uv_canonical*M_cam_from_uv.I\n",
    "M_warp_cam_pts  = scipy.linalg.block_diag(M_warp_cam_pts_, 1)\n",
    "\n",
    "# construct a matrix to account for the different camera-space conventions in Hypersim and PyTorch3D\n",
    "#   Hypersim:  +x points to the right, +y points up, +z points away from the viewing direction\n",
    "#   PyTorch3D: +x points to the left,  +y points up, +z points towards the viewing direction\n",
    "M_p3dcam_from_cam = np.matrix(np.identity(4))\n",
    "M_p3dcam_from_cam[0,0] = -1\n",
    "M_p3dcam_from_cam[2,2] = -1\n",
    "\n",
    "#\n",
    "# We transform our vertices into camera-space manually, because this enables us to account\n",
    "# for Hypersim's non-standard camera parameters purely in camera-space. We can then use the\n",
    "# default perspective projection matrix available in PyTorch3D.\n",
    "#\n",
    "\n",
    "num_points = mesh_positions_world.shape[0]\n",
    "\n",
    "P_world  = np.matrix( np.c_[ mesh_positions_world, np.ones(num_points) ] ).T\n",
    "P_p3dcam = M_p3dcam_from_cam*M_warp_cam_pts*M_cam_from_world*P_world\n",
    "\n",
    "# construct PyTorch3D mesh\n",
    "verts = torch.tensor(P_p3dcam.T[:,0:3], device=device, dtype=torch.float32)\n",
    "faces = torch.tensor(mesh_faces_vi, device=device)\n",
    "mesh  = Meshes(verts=[verts], faces=[faces])\n",
    "\n",
    "# aspect_ratio refers to the pixel aspect ratio, so we set it to 1.0 even though our images are not square\n",
    "cameras = FoVPerspectiveCameras(device=device, fov=fov_y, degrees=False, aspect_ratio=1.0, znear=1.0, zfar=400.0)\n",
    "\n",
    "raster_settings = RasterizationSettings(\n",
    "    image_size=[height_pixels, width_pixels],\n",
    "    blur_radius=0.0, \n",
    "    faces_per_pixel=1, \n",
    ")\n",
    "\n",
    "renderer = MeshRenderer(\n",
    "    rasterizer=MeshRasterizer(\n",
    "        cameras=cameras, \n",
    "        raster_settings=raster_settings\n",
    "    ),\n",
    "    shader=HardDepthShader(\n",
    "        device=device, \n",
    "        cameras=cameras\n",
    "    )\n",
    ")\n",
    "\n",
    "# render\n",
    "images = renderer(mesh)\n",
    "\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.imshow(images[0, ..., 0].cpu().numpy())\n",
    "plt.colorbar()\n",
    "plt.axis(\"off\");"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}