feat: video to texture example

src/examples/gettingstarted/13_videoTexture/example.json

@@ -0,0 +1,8 @@
+{
+  "slug": "getting_started_canvas_texture",
+  "en": {
+    "name": "Getting Started - Canvas Texture",
+    "description": "Redrawing a Canvas Texture every frame and using that as the source of the texture.",
+    "keywords": ["basics", "canvas", "framebuffer"]
+  }
+}

src/examples/gettingstarted/13_videoTexture/fragment.glsl

@@ -0,0 +1,18 @@
+precision highp float;
+
+varying vec3 v_viewPosition;
+varying vec3 v_viewNormal;
+varying vec2 v_uv;
+
+uniform sampler2D map;
+uniform vec3 viewLightPosition;
+
+void main() {
+
+  vec3 albedo = texture2D(map, v_uv).xyz;
+  vec3 directionToLight = normalize( viewLightPosition - v_viewPosition );
+  float lambertianIntensity = dot( directionToLight, v_viewNormal );
+
+  gl_FragColor = vec4( albedo * lambertianIntensity, 1. );
+
+}

src/examples/gettingstarted/13_videoTexture/index.ts

@@ -0,0 +1,76 @@
+import { boxGeometry } from "../../../lib/geometry/primitives/boxGeometry";
+import { ShaderMaterial } from "../../../lib/materials/ShaderMaterial";
+import { Euler } from "../../../lib/math/Euler";
+import { Matrix4 } from "../../../lib/math/Matrix4";
+import {
+  makeMatrix4OrthographicSimple,
+  makeMatrix4RotationFromEuler,
+  makeMatrix4Translation,
+} from "../../../lib/math/Matrix4.Functions";
+import { Vector2 } from "../../../lib/math/Vector2";
+import { Vector3 } from "../../../lib/math/Vector3";
+import { makeBufferGeometryFromGeometry } from "../../../lib/renderers/webgl/buffers/BufferGeometry";
+import { ClearState } from "../../../lib/renderers/webgl/ClearState";
+import { DepthTestFunc, DepthTestState } from "../../../lib/renderers/webgl/DepthTestState";
+import { AttachmentBits } from "../../../lib/renderers/webgl/framebuffers/AttachmentBits";
+import { makeProgramFromShaderMaterial } from "../../../lib/renderers/webgl/programs/Program";
+import { RenderingContext } from "../../../lib/renderers/webgl/RenderingContext";
+import { makeTexImage2DFromTexture } from "../../../lib/renderers/webgl/textures/TexImage2D";
+import { makeTextureFromVideoElement } from "../../../lib/textures/Texture";
+import fragmentSourceCode from "./fragment.glsl";
+import vertexSourceCode from "./vertex.glsl";
+
+async function init(): Promise<null> {
+  const context = new RenderingContext(document.getElementById("framebuffer") as HTMLCanvasElement);
+  const canvasFramebuffer = context.canvasFramebuffer;
+  window.addEventListener("resize", () => canvasFramebuffer.resize());
+
+  const geometry = boxGeometry(0.75, 0.75, 0.75);
+  const material = new ShaderMaterial(vertexSourceCode, fragmentSourceCode);
+  const program = makeProgramFromShaderMaterial(context, material);
+
+  const video = document.createElement("video");
+  const source = document.createElement("source");
+  source.type = "video/mp4";
+  source.src = "/assets/textures/videos/sintel.mp4";
+  video.appendChild(source);
+
+  await video.play();
+  video.currentTime = 3; // jump ahead to content
+
+  const texture = makeTextureFromVideoElement(video);
+  const uvTestTexture = makeTexImage2DFromTexture(context, texture);
+  const uniforms = {
+    localToWorld: new Matrix4(),
+    worldToView: makeMatrix4Translation(new Vector3(0, 0, -1)),
+    viewToScreen: makeMatrix4OrthographicSimple(1.5, new Vector2(), 0.1, 2.0, 1.0, canvasFramebuffer.aspectRatio),
+    viewLightPosition: new Vector3(0, 0, 0),
+    map: uvTestTexture,
+  };
+  const bufferGeometry = makeBufferGeometryFromGeometry(context, geometry);
+  const depthTestState = new DepthTestState(true, DepthTestFunc.Less);
+  const whiteClearState = new ClearState(new Vector3(1, 1, 1), 1.0);
+
+  function animate(): void {
+    const now = Date.now();
+    uniforms.localToWorld = makeMatrix4RotationFromEuler(
+      new Euler(now * 0.0001, now * 0.0013, now * 0.00077),
+      uniforms.localToWorld,
+    );
+    if (video.readyState >= video.HAVE_CURRENT_DATA) {
+      uvTestTexture.loadImages([video]);
+    }
+    uniforms.map = uvTestTexture;
+
+    canvasFramebuffer.clear(AttachmentBits.All, whiteClearState);
+    canvasFramebuffer.renderBufferGeometry(program, uniforms, bufferGeometry, depthTestState);
+
+    requestAnimationFrame(animate);
+  }
+
+  animate();
+
+  return null;
+}
+
+init();

src/examples/gettingstarted/13_videoTexture/vertex.glsl

@@ -0,0 +1,20 @@
+attribute vec3 position;
+attribute vec3 normal;
+attribute vec2 uv;
+
+uniform mat4 localToWorld;
+uniform mat4 worldToView;
+uniform mat4 viewToScreen;
+
+varying vec3 v_viewPosition;
+varying vec3 v_viewNormal;
+varying vec2 v_uv;
+
+void main() {
+
+  v_viewNormal = normalize( ( worldToView * localToWorld * vec4( normal, 0. ) ).xyz );
+  v_viewPosition = ( worldToView * localToWorld * vec4( position, 1. ) ).xyz;
+  v_uv = uv;
+  gl_Position = viewToScreen * vec4( v_viewPosition, 1. );
+
+}

src/lib/renderers/webgl/textures/TexImage2D.ts

@@ -18,6 +18,7 @@ import { DataType } from "./DataType";
 import { PixelFormat } from "./PixelFormat";
 import { TexParameters } from "./TexParameters";
 import { TextureTarget } from "./TextureTarget";
+import { TextureWrap } from "./TextureWrap";
 
 export class TexImage2D implements IDisposable {
   disposed = false;
@@ -133,6 +134,13 @@ export function makeTexImage2DFromTexture(
   level = 0,
   internalFormat: PixelFormat = PixelFormat.RGBA,
 ): TexImage2D {
+  const params = new TexParameters();
+  params.anisotropyLevels = texture.anisotropicLevels;
+  params.generateMipmaps = texture.generateMipmaps;
+  params.magFilter = texture.magFilter;
+  params.minFilter = texture.minFilter;
+  params.wrapS = texture.wrapS;
+  params.wrapT = texture.wrapT;
   return new TexImage2D(
     context,
     [texture.image],
@@ -142,6 +150,7 @@ export function makeTexImage2DFromTexture(
     texture.dataType,
     internalFormat,
     TextureTarget.Texture2D,
+    params,
   );
 }
 
@@ -151,6 +160,13 @@ export function makeTexImage2DFromCubeTexture(
   level = 0,
   internalFormat: PixelFormat = PixelFormat.RGBA,
 ): TexImage2D {
+  const params = new TexParameters();
+  params.anisotropyLevels = texture.anisotropicLevels;
+  params.generateMipmaps = texture.generateMipmaps;
+  params.magFilter = texture.magFilter;
+  params.minFilter = texture.minFilter;
+  params.wrapS = TextureWrap.ClampToEdge;
+  params.wrapT = TextureWrap.ClampToEdge;
   return new TexImage2D(
     context,
     texture.images,
@@ -160,6 +176,7 @@ export function makeTexImage2DFromCubeTexture(
     texture.dataType,
     internalFormat,
     TextureTarget.TextureCubeMap,
+    params,
   );
 }
 

src/lib/textures/Texture.ts

@@ -39,3 +39,18 @@ export class Texture extends VirtualTexture {
     this.size = new Vector2(image.width, image.height);
   }
 }
+
+export function makeTextureFromVideoElement(video: HTMLVideoElement): Texture {
+  return new Texture(
+    video,
+    TextureWrap.ClampToEdge,
+    TextureWrap.ClampToEdge,
+    0,
+    TextureFilter.Linear,
+    TextureFilter.Linear,
+    PixelFormat.RGB,
+    DataType.UnsignedByte,
+    false,
+    0,
+  );
+}