Get infinite scrolling and basic audio working

components/ImagePlane.tsx

@@ -1,11 +1,10 @@
 import * as THREE from "three";
-import React, { useRef, Suspense } from "react";
+import React, { Suspense } from "react";
-import { useLoader, useFrame } from "@react-three/fiber";
+import { useLoader } from "@react-three/fiber";
 
 interface ImagePlaneProps {
   url: string;
   height: number;
-  paused: boolean;
 }
 
 /**
@@ -14,20 +13,10 @@ interface ImagePlaneProps {
 export default function ImagePlane(props: ImagePlaneProps) {
   const texture = useLoader(THREE.TextureLoader, props.url);
 
-  const mesh = useRef<THREE.Mesh>(null);
-
-  useFrame(() => {
-    if (props.paused) {
-      return;
-    }
-
-    mesh.current.position.y += 1 / 30.0;
-  });
-
   return (
     <Suspense fallback={null}>
-      <mesh ref={mesh} rotation-z={-Math.PI / 2} position-y={props.height}>
+      <mesh rotation-z={-Math.PI / 2} position-y={props.height}>
-        <planeBufferGeometry attach="geometry" args={[30, 30]} />
+        <planeGeometry attach="geometry" args={[5, 5]} />
         <meshBasicMaterial attach="material" map={texture} />
       </mesh>
     </Suspense>

components/RotatingBox.tsx

@@ -27,7 +27,7 @@ export default function RotatingBox(props: JSX.IntrinsicElements["mesh"]) {
       smoothness={4}
       {...props}
       ref={mesh}
-      scale={active ? [3, 3, 3] : [2, 2, 2]}
+      scale={active ? [.3, .3, .3] : [.2, .2, .2]}
       onClick={() => setActive(!active)}
       onPointerOver={() => setHover(true)}
       onPointerOut={() => setHover(false)}

components/SpectrogramViewer.tsx

@@ -0,0 +1,56 @@
+import ImagePlane from "./ImagePlane";
+
+import { useRef } from "react";
+import { useFrame, useThree } from "@react-three/fiber";
+
+import { InferenceResult } from "../types";
+import { QuadraticBezierLine } from "@react-three/drei";
+
+interface SpectrogramViewerProps {
+  paused: boolean;
+  inferenceResults: InferenceResult[];
+}
+
+/**
+ * Spectrogram drawing code.
+ */
+export default function SpectrogramViewer(props: SpectrogramViewerProps) {
+  const paused = props.paused;
+  const inferenceResults = props.inferenceResults;
+
+  const camera = useThree((state) => state.camera);
+
+  const playheadRef = useRef(null);
+
+  // Move the camera and playhead every frame.
+  useFrame(() => {
+    if (paused) {
+      return;
+    }
+
+    const framerate = 120.0;
+    camera.position.y -= 1 / framerate;
+    playheadRef.current.position.y -= 1 / framerate;
+  });
+
+  return (
+    <group>
+      {inferenceResults.map((value: InferenceResult, index: number) => {
+        const height = 5 * (-1 - value.counter) - 2;
+        return <ImagePlane url={value.image} height={height} key={index} />;
+      })}
+
+      {/* TODO make into playhead component */}
+      <group ref={playheadRef}>
+        <QuadraticBezierLine
+          start={[-3, 0, 1]} // Starting point, can be an array or a vec3
+          end={[3, 0, 1]} // Ending point, can be an array or a vec3
+          mid={[0, -0.8, 0.4]} // Optional control point, can be an array or a vec3
+          color="#aa3333" // Default
+          lineWidth={5} // In pixels (default)
+          dashed={false} // Default
+        />
+      </group>
+    </group>
+  );
+}

components/ThreeCanvas.tsx

@@ -1,34 +1,25 @@
 import { Canvas } from "@react-three/fiber";
 
-import RotatingBox from "./RotatingBox";
+import { InferenceResult } from "../types";
-import ImagePlane from "./ImagePlane";
+import SpectrogramViewer from "./SpectrogramViewer";
 
 interface CanvasProps {
   paused: boolean;
+  inferenceResults: InferenceResult[];
 }
 
 /**
  * React three fiber canvas with spectrogram drawing.
  */
 export default function ThreeCanvas(props: CanvasProps) {
-  // change the image URL
-  const spectrogram_image = "spectrogram.jpeg";
-
-  const height = -30.0;
-
   return (
-    <Canvas camera={{ position: [0, 0, 35], rotation: [0.2, 0, 0] }}>
+    <Canvas camera={{ position: [0, 0, 7], rotation: [0.2, 0, 0] }}>
       <ambientLight intensity={2} />
       <pointLight position={[40, 40, 40]} />
-      <ImagePlane
+      <SpectrogramViewer
-        url={spectrogram_image}
-        height={height}
         paused={props.paused}
+        inferenceResults={props.inferenceResults}
       />
-      <RotatingBox position={[-12, 0, 1]} />
-      <RotatingBox position={[-4, 0, 1]} />
-      <RotatingBox position={[4, 0, 1]} />
-      <RotatingBox position={[12, 0, 1]} />
     </Canvas>
   );
 }

package-lock.json

@@ -17,7 +17,8 @@
         "react": "18.2.0",
         "react-dom": "18.2.0",
         "react-icons": "^4.6.0",
-        "three": "^0.146.0"
+        "three": "^0.146.0",
+        "tone": "^14.7.77"
       },
       "devDependencies": {
         "@types/node": "^18.11.9",
@@ -938,6 +939,18 @@
       "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz",
       "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag=="
     },
+    "node_modules/automation-events": {
+      "version": "4.0.24",
+      "resolved": "https://registry.npmjs.org/automation-events/-/automation-events-4.0.24.tgz",
+      "integrity": "sha512-v/c//VCE1TGU3R56cyTFnebXFR3NOOx5Kiuaz7TXwLGpYMp9VgKCwelODRW3ZZ5TP2OFymfrWsvNVuPSi6vVPQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.20.1",
+        "tslib": "^2.4.1"
+      },
+      "engines": {
+        "node": ">=12.20.1"
+      }
+    },
     "node_modules/autoprefixer": {
       "version": "10.4.13",
       "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz",
@@ -3642,6 +3655,16 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/standardized-audio-context": {
+      "version": "25.3.35",
+      "resolved": "https://registry.npmjs.org/standardized-audio-context/-/standardized-audio-context-25.3.35.tgz",
+      "integrity": "sha512-61KBkor9fOsMEl3NZllnis8KDmTJzonuuW8h6JBmxXXcUDucypGCJp+hlbyKG+olav8wgA+VKlqFmiJEbh4CfA==",
+      "dependencies": {
+        "@babel/runtime": "^7.20.1",
+        "automation-events": "^4.0.24",
+        "tslib": "^2.4.1"
+      }
+    },
     "node_modules/stats.js": {
       "version": "0.17.0",
       "resolved": "https://registry.npmjs.org/stats.js/-/stats.js-0.17.0.tgz",
@@ -3906,6 +3929,15 @@
         "node": ">=8.0"
       }
     },
+    "node_modules/tone": {
+      "version": "14.7.77",
+      "resolved": "https://registry.npmjs.org/tone/-/tone-14.7.77.tgz",
+      "integrity": "sha512-tCfK73IkLHyzoKUvGq47gyDyxiKLFvKiVCOobynGgBB9Dl0NkxTM2p+eRJXyCYrjJwy9Y0XCMqD3uOYsYt2Fdg==",
+      "dependencies": {
+        "standardized-audio-context": "^25.1.8",
+        "tslib": "^2.0.1"
+      }
+    },
     "node_modules/troika-three-text": {
       "version": "0.46.4",
       "resolved": "https://registry.npmjs.org/troika-three-text/-/troika-three-text-0.46.4.tgz",
@@ -4770,6 +4802,15 @@
       "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz",
       "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag=="
     },
+    "automation-events": {
+      "version": "4.0.24",
+      "resolved": "https://registry.npmjs.org/automation-events/-/automation-events-4.0.24.tgz",
+      "integrity": "sha512-v/c//VCE1TGU3R56cyTFnebXFR3NOOx5Kiuaz7TXwLGpYMp9VgKCwelODRW3ZZ5TP2OFymfrWsvNVuPSi6vVPQ==",
+      "requires": {
+        "@babel/runtime": "^7.20.1",
+        "tslib": "^2.4.1"
+      }
+    },
     "autoprefixer": {
       "version": "10.4.13",
       "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz",
@@ -6645,6 +6686,16 @@
       "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz",
       "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw=="
     },
+    "standardized-audio-context": {
+      "version": "25.3.35",
+      "resolved": "https://registry.npmjs.org/standardized-audio-context/-/standardized-audio-context-25.3.35.tgz",
+      "integrity": "sha512-61KBkor9fOsMEl3NZllnis8KDmTJzonuuW8h6JBmxXXcUDucypGCJp+hlbyKG+olav8wgA+VKlqFmiJEbh4CfA==",
+      "requires": {
+        "@babel/runtime": "^7.20.1",
+        "automation-events": "^4.0.24",
+        "tslib": "^2.4.1"
+      }
+    },
     "stats.js": {
       "version": "0.17.0",
       "resolved": "https://registry.npmjs.org/stats.js/-/stats.js-0.17.0.tgz",
@@ -6836,6 +6887,15 @@
         "is-number": "^7.0.0"
       }
     },
+    "tone": {
+      "version": "14.7.77",
+      "resolved": "https://registry.npmjs.org/tone/-/tone-14.7.77.tgz",
+      "integrity": "sha512-tCfK73IkLHyzoKUvGq47gyDyxiKLFvKiVCOobynGgBB9Dl0NkxTM2p+eRJXyCYrjJwy9Y0XCMqD3uOYsYt2Fdg==",
+      "requires": {
+        "standardized-audio-context": "^25.1.8",
+        "tslib": "^2.0.1"
+      }
+    },
     "troika-three-text": {
       "version": "0.46.4",
       "resolved": "https://registry.npmjs.org/troika-three-text/-/troika-three-text-0.46.4.tgz",

package.json

@@ -9,16 +9,17 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@headlessui/react": "^1.5.0",
     "@react-three/drei": "^9.41.2",
     "@react-three/fiber": "^8.9.1",
     "eslint": "8.27.0",
     "eslint-config-next": "13.0.4",
-    "@headlessui/react": "^1.5.0",
     "next": "13.0.4",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-icons": "^4.6.0",
-    "three": "^0.146.0"
+    "three": "^0.146.0",
+    "tone": "^14.7.77"
   },
   "devDependencies": {
     "@types/node": "^18.11.9",

pages/index.tsx

@@ -1,5 +1,5 @@
 import Head from "next/head";
-import { useState } from "react";
+import { useEffect, useState } from "react";
 
 import ThreeCanvas from "../components/ThreeCanvas";
 import PromptPanel from "../components/PromptPanel";
@@ -8,17 +8,81 @@ import Pause from "../components/Pause";
 
 import { InferenceResult, PromptInput } from "../types";
 
+import * as Tone from "tone";
+
 const defaultPromptInputs = [
   { prompt: "A jazz pianist playing a classical concerto", seed: 10 },
   { prompt: "Taylor Swift singing with a tropical beat", seed: 10 },
 ];
 
+const defaultInferenceResults = [
+  {
+    input: {
+      alpha: 0.0,
+      start: defaultPromptInputs[0],
+      end: defaultPromptInputs[1],
+    },
+    image: "rap_sample.jpg",
+    audio: "rap_sample.mp3",
+    counter: 0,
+  },
+];
+
+const timeout = 5000;
+const maxLength = 10;
+
 export default function Home() {
   const [paused, setPaused] = useState(false);
 
   const [promptInputs, setPromptInputs] =
     useState<PromptInput[]>(defaultPromptInputs);
 
+  const [inferenceResults, setInferenceResults] = useState<InferenceResult[]>(
+    defaultInferenceResults
+  );
+
+  // /////////////
+
+  const [tonePlayer, setTonePlayer] = useState(null);
+
+  useEffect(() => {
+    setTonePlayer(
+      new Tone.Player(defaultInferenceResults[0].audio).toDestination()
+    );
+    // play as soon as the buffer is loaded
+    // player.autostart = true;
+  }, [inferenceResults]);
+
+  useEffect(() => {
+    if (tonePlayer && tonePlayer.loaded) {
+      if (!paused) {
+        tonePlayer.start();
+      } else {
+        tonePlayer.stop();
+      }
+    }
+  }, [paused, tonePlayer]);
+
+  // /////////////
+
+  useEffect(() => {
+    setTimeout(() => {
+      const lastResult = inferenceResults[inferenceResults.length - 1];
+      const newResult = { ...lastResult, counter: lastResult.counter + 1 };
+
+      let results = [...inferenceResults, newResult];
+
+      if (results.length > maxLength) {
+        results = results.slice(1);
+      }
+
+      console.log("Adding to inference results");
+      console.log(results);
+
+      setInferenceResults(results);
+    }, timeout);
+  });
+
   return (
     <>
       <Head>
@@ -32,7 +96,7 @@ export default function Home() {
 
       <div className="bg-sky-900 flex flex-row min-h-screen text-white">
         <div className="w-1/3 min-h-screen">
-          <ThreeCanvas paused={paused} />
+          <ThreeCanvas paused={paused} inferenceResults={inferenceResults} />
         </div>
 
         <PromptPanel

types.ts

@@ -17,6 +17,8 @@ export interface InferenceInput {
 export interface InferenceResult {
     input: InferenceInput;
 
+    counter: number;
+
     // URL of the image
     image: string;