Synchronize time to loop the player and draw the timeline

components/SpectrogramViewer.tsx

@@ -1,6 +1,6 @@
 import { useRef } from "react";
 import { useFrame, useThree } from "@react-three/fiber";
-import { QuadraticBezierLine } from "@react-three/drei";
+import { QuadraticBezierLine, Plane } from "@react-three/drei";
 
 import { InferenceResult } from "../types";
 import HeightMapImage from "./HeightMapImage";
@@ -9,7 +9,9 @@ import ImagePlane from "./ImagePlane";
 interface SpectrogramViewerProps {
   paused: boolean;
   inferenceResults: InferenceResult[];
+  getTime: () => number;
   use_height_map?: boolean;
+  audioLength: number;
 }
 
 /**
@@ -18,27 +20,29 @@ interface SpectrogramViewerProps {
 export default function SpectrogramViewer({
   paused,
   inferenceResults,
+  getTime,
+  audioLength,
   use_height_map = true,
 }: SpectrogramViewerProps) {
   const camera = useThree((state) => state.camera);
 
   const playheadRef = useRef(null);
 
-  // Move the camera and playhead every frame.
+  // Move the camera based on the clock
   useFrame(() => {
-    if (paused) {
-      return;
-    }
+    const time = getTime();
 
-    const framerate = 120.0;
-    camera.position.y -= 1 / framerate;
-    playheadRef.current.position.y -= 1 / framerate;
+    const velocity = -1.0; // [m/s]
+    const position = velocity * time; // [m]
+
+    camera.position.y = position;
+    playheadRef.current.position.y = camera.position.y;
   });
 
   return (
     <group>
       {inferenceResults.map((value: InferenceResult, index: number) => {
-        const height = 5 * (-1 - value.counter) - 2;
+        const height = audioLength * (-0.48 - value.counter);
 
         if (use_height_map) {
           return (
@@ -46,7 +50,7 @@ export default function SpectrogramViewer({
               url={value.image}
               position={[0, height, 0]}
               rotation={[0, 0, -Math.PI / 2]}
-              scale={[5, 5, 5]}
+              scale={[audioLength, 5, 1]}
               key={index}
             />
           );
@@ -57,14 +61,22 @@ export default function SpectrogramViewer({
 
       {/* TODO make into playhead component */}
       <group ref={playheadRef}>
-        <QuadraticBezierLine
+        <Plane
+          args={[5.5, 2.0]}
+          rotation={[Math.PI / 2 - 0.4, 0, 0]}
+          position={[0, 0, -0.5]}
+        >
+          <meshBasicMaterial color="#ee2211" transparent opacity={0.8} />
+        </Plane>
+
+        {/* <QuadraticBezierLine
           start={[-3, 0, 1]} // Starting point, can be an array or a vec3
           end={[3, 0, 1]} // Ending point, can be an array or a vec3
           mid={[0, -0.8, 0.4]} // Optional control point, can be an array or a vec3
           color="#aa3333" // Default
           lineWidth={5} // In pixels (default)
           dashed={false} // Default
-        />
+        /> */}
       </group>
     </group>
   );

components/ThreeCanvas.tsx

@@ -6,19 +6,28 @@ import SpectrogramViewer from "./SpectrogramViewer";
 interface CanvasProps {
   paused: boolean;
   inferenceResults: InferenceResult[];
+  getTime: () => number;
+  audioLength: number;
 }
 
 /**
  * React three fiber canvas with spectrogram drawing.
  */
-export default function ThreeCanvas(props: CanvasProps) {
+export default function ThreeCanvas({
+  paused,
+  inferenceResults,
+  getTime,
+  audioLength,
+}: CanvasProps) {
   return (
     <Canvas camera={{ position: [0, 0, 7], rotation: [0.4, 0, 0] }}>
       <ambientLight intensity={2} />
       <pointLight position={[40, 40, 40]} />
       <SpectrogramViewer
-        paused={props.paused}
-        inferenceResults={props.inferenceResults}
+        paused={paused}
+        inferenceResults={inferenceResults}
+        getTime={getTime}
+        audioLength={audioLength}
       />
     </Canvas>
   );

pages/index.tsx

@@ -11,8 +11,8 @@ import { InferenceResult, PromptInput } from "../types";
 import * as Tone from "tone";
 
 const defaultPromptInputs = [
-  { prompt: "A jazz pianist playing a classical concerto"},
-  { prompt: "Taylor Swift singing with a tropical beat"},
+  { prompt: "A jazz pianist playing a classical concerto" },
+  { prompt: "Taylor Swift singing with a tropical beat" },
 ];
 
 const defaultInferenceResults = [
@@ -28,11 +28,12 @@ const defaultInferenceResults = [
   },
 ];
 
-const timeout = 5000;
+// TODO(hayk): Do this as soon as sample comes back
+const timeout = 5150;
 const maxLength = 10;
 
 export default function Home() {
-  const [paused, setPaused] = useState(false);
+  const [paused, setPaused] = useState(true);
 
   const [promptInputs, setPromptInputs] =
     useState<PromptInput[]>(defaultPromptInputs);
@@ -43,45 +44,75 @@ export default function Home() {
 
   // /////////////
 
-  const [tonePlayer, setTonePlayer] = useState(null);
+  const [tonePlayer, setTonePlayer] = useState<Tone.Player>(null);
 
   useEffect(() => {
-    setTonePlayer(
-      new Tone.Player(defaultInferenceResults[0].audio).toDestination()
-    );
-    // play as soon as the buffer is loaded
-    // player.autostart = true;
+    // HACK(hayk): Kill
+    if (tonePlayer) {
+      return;
+    }
+
+    if (inferenceResults.length == 0) {
+      return;
+    }
+    console.log(inferenceResults);
+
+    const player = new Tone.Player(
+      inferenceResults[inferenceResults.length - 1].audio,
+      () => {
+        console.log("New player loaded.");
+
+        player.sync().start(0);
+
+        // if (tonePlayer) {
+        //   tonePlayer.stop();
+        //   tonePlayer.dispose();
+        // }
+        setTonePlayer(player);
+      }
+    ).toDestination();
+    player.loop = true;
   }, [inferenceResults]);
 
   useEffect(() => {
-    if (tonePlayer && tonePlayer.loaded) {
-      if (!paused) {
-        tonePlayer.start();
-      } else {
-        tonePlayer.stop();
+    if (!paused) {
+      console.log("Play");
+
+      if (Tone.context.state == "suspended") {
+        Tone.context.resume();
+      }
+
+      if (tonePlayer) {
+        Tone.Transport.start();
+      }
+    } else {
+      console.log("Pause");
+
+      if (tonePlayer) {
+        Tone.Transport.pause();
       }
     }
-  }, [paused, tonePlayer]);
+  }, [paused]);
 
   // /////////////
 
   useEffect(() => {
-    setTimeout(() => {
-      const lastResult = inferenceResults[inferenceResults.length - 1];
-      const newResult = { ...lastResult, counter: lastResult.counter + 1 };
+    console.log("setInterval");
+    setInterval(() => {
+      setInferenceResults((prevResults) => {
+        const lastResult = prevResults[prevResults.length - 1];
+        const newResult = { ...lastResult, counter: lastResult.counter + 1 };
 
-      let results = [...inferenceResults, newResult];
+        let results = [...prevResults, newResult];
 
-      if (results.length > maxLength) {
-        results = results.slice(1);
-      }
+        if (results.length > maxLength) {
+          results = results.slice(1);
+        }
 
-      console.log("Adding to inference results");
-      console.log(results);
-
-      setInferenceResults(results);
+        return results;
+      });
     }, timeout);
-  });
+  }, []);
 
   return (
     <>
@@ -96,13 +127,18 @@ export default function Home() {
 
       <div className="bg-sky-900 flex flex-row min-h-screen text-white">
         <div className="w-1/3 min-h-screen">
-          <ThreeCanvas paused={paused} inferenceResults={inferenceResults} />
+          <ThreeCanvas
+            paused={paused}
+            getTime={() => Tone.Transport.seconds}
+            audioLength={tonePlayer ? tonePlayer.sampleTime * tonePlayer.buffer.length : 5}
+            inferenceResults={inferenceResults}
+          />
         </div>
 
         <PromptPanel
           prompts={promptInputs}
           addPrompt={(prompt: string) => {
-            setPromptInputs([...promptInputs, { prompt: prompt}]);
+            setPromptInputs([...promptInputs, { prompt: prompt }]);
           }}
         />