Add heightmap effect

components/HeightMapImage.tsx

@@ -0,0 +1,52 @@
+import { DoubleSide, RepeatWrapping, sRGBEncoding } from "three";
+import {
+  useTexture,
+} from "@react-three/drei";
+
+import { vertexShader, fragmentShader } from "../shaders";
+
+interface HeightMapImageProps {
+    url: string;
+    position: [number, number, number];
+    rotation: [number, number, number];
+    scale: [number, number, number];
+  }
+
+export default function HeightMapImage(props: HeightMapImageProps) {
+  const url = props.url;
+
+  // Load the heightmap image
+  const heightMap = useTexture(url);
+  heightMap.wrapS = RepeatWrapping;
+  heightMap.wrapT = RepeatWrapping;
+
+  // Load the texture map
+  const textureMap = useTexture(url);
+  textureMap.wrapS = RepeatWrapping;
+  textureMap.wrapT = RepeatWrapping;
+
+  return (
+    <mesh
+      position={props.position}
+      rotation={props.rotation}
+      scale={props.scale}
+    >
+      {/* TODO hayk reduce */}
+      <planeGeometry args={[1, 1, 256, 256]} />
+      <shaderMaterial
+        uniforms={{
+          // Feed the heightmap
+          bumpTexture: { value: heightMap },
+          // Feed the scaling constant for the heightmap
+          bumpScale: { value: -0.1 },
+          // Feed the texture map
+          terrainTexture: { value: textureMap },
+        }}
+        // Feed the shaders as strings
+        vertexShader={vertexShader}
+        fragmentShader={fragmentShader}
+        side={DoubleSide}
+      />
+    </mesh>
+  );
+}

components/SpectrogramViewer.tsx

@@ -1,23 +1,25 @@
-import ImagePlane from "./ImagePlane";
-
 import { useRef } from "react";
 import { useFrame, useThree } from "@react-three/fiber";
+import { QuadraticBezierLine } from "@react-three/drei";
 
 import { InferenceResult } from "../types";
-import { QuadraticBezierLine } from "@react-three/drei";
+import HeightMapImage from "./HeightMapImage";
+import ImagePlane from "./ImagePlane";
 
 interface SpectrogramViewerProps {
   paused: boolean;
   inferenceResults: InferenceResult[];
+  use_height_map?: boolean;
 }
 
 /**
  * Spectrogram drawing code.
  */
-export default function SpectrogramViewer(props: SpectrogramViewerProps) {
-  const paused = props.paused;
-  const inferenceResults = props.inferenceResults;
-
+export default function SpectrogramViewer({
+  paused,
+  inferenceResults,
+  use_height_map = true,
+}: SpectrogramViewerProps) {
   const camera = useThree((state) => state.camera);
 
   const playheadRef = useRef(null);
@@ -37,7 +39,20 @@ export default function SpectrogramViewer(props: SpectrogramViewerProps) {
     <group>
       {inferenceResults.map((value: InferenceResult, index: number) => {
         const height = 5 * (-1 - value.counter) - 2;
+
+        if (use_height_map) {
+          return (
+            <HeightMapImage
+              url={value.image}
+              position={[0, height, 0]}
+              rotation={[0, 0, -Math.PI / 2]}
+              scale={[5, 5, 5]}
+              key={index}
+            />
+          );
+        } else {
           return <ImagePlane url={value.image} height={height} key={index} />;
+        }
       })}
 
       {/* TODO make into playhead component */}

components/ThreeCanvas.tsx

@@ -13,7 +13,7 @@ interface CanvasProps {
  */
 export default function ThreeCanvas(props: CanvasProps) {
   return (
-    <Canvas camera={{ position: [0, 0, 7], rotation: [0.2, 0, 0] }}>
+    <Canvas camera={{ position: [0, 0, 7], rotation: [0.4, 0, 0] }}>
       <ambientLight intensity={2} />
       <pointLight position={[40, 40, 40]} />
       <SpectrogramViewer

shaders.js

@@ -0,0 +1,60 @@
+export const vertexShader = `
+// Uniforms are data that are shared between shaders
+// The contain data that are uniform across the entire frame.
+// The heightmap and scaling constant for each point are uniforms in this respect.
+
+// A uniform to contain the heightmap image
+uniform sampler2D bumpTexture;
+// A uniform to contain the scaling constant
+uniform float bumpScale;
+
+// Varyings are variables whose values are decided in the vertext shader
+// But whose values are then needed in the fragment shader
+
+// A variable to store the height of the point
+varying float vAmount;
+// The UV mapping coordinates of a vertex
+varying vec2 vUV;
+
+void main()
+{
+    // The "coordinates" in UV mapping representation
+    vUV = uv;
+
+    // The heightmap data at those coordinates
+    vec4 bumpData = texture2D(bumpTexture, uv);
+
+    // height map is grayscale, so it doesn't matter if you use r, g, or b.
+    vAmount = bumpData.r;
+
+    // move the position along the normal
+    vec3 newPosition = position + normal * bumpScale * vAmount;
+
+    // Compute the position of the vertex using a standard formula
+    gl_Position = projectionMatrix * modelViewMatrix * vec4(newPosition, 1.0);
+}
+`;
+
+export const fragmentShader = `
+// A uniform for the terrain texture image
+uniform sampler2D terrainTexture;
+
+// Get the varyings from the vertex shader
+varying vec2 vUV;
+// vAmount isn't really used, but could be if necessary
+varying float vAmount;
+
+uniform lowp float shadows;
+uniform lowp float highlights;
+
+const mediump vec3 luminanceWeighting = vec3(0.3, 0.3, 0.3);
+
+void main()
+{
+    // Get the color of the fragment from the texture map
+    // at that coordinate in the UV mapping
+    vec4 source = texture2D(terrainTexture, vUV);
+    gl_FragColor = vec4(source.r - 0.05, source.g - 0.05, source.b + 0.1, 1.0);
+}
+
+`;