Separate AudioPlayer component

components/AudioPlayer.tsx

@@ -0,0 +1,136 @@
+import { useEffect, useState } from "react";
+
+import * as Tone from "tone";
+
+import { InferenceResult } from "../types";
+
+interface AudioPlayerProps {
+  paused: boolean;
+  inferenceResults: InferenceResult[];
+}
+
+/**
+ * Audio player with Tone.js
+ *
+ * TODO(hayk): Look into https://github.com/E-Kuerschner/useAudioPlayer as an alternative.
+ */
+export default function AudioPlayer({
+  paused,
+  inferenceResults,
+}: AudioPlayerProps) {
+  // TODO(hayk): Rename
+  const [tonePlayer, setTonePlayer] = useState<Tone.Player>(null);
+
+  const [numClipsPlayed, setNumClipsPlayed] = useState(0);
+  const [prevNumClipsPlayed, setPrevNumClipsPlayed] = useState(0);
+
+  // TODO(hayk): What is this?
+  const [resultCounter, setResultCounter] = useState(0);
+
+  // On load, create a player synced to the tone transport
+  useEffect(() => {
+    if (tonePlayer) {
+      return;
+    }
+
+    if (inferenceResults.length === 0) {
+      return;
+    }
+
+    const audioUrl = inferenceResults[0].audio;
+
+    const player = new Tone.Player(audioUrl, () => {
+      console.log("Created player.");
+
+      player.loop = true;
+      player.sync().start(0);
+
+      // Set up a callback to increment numClipsPlayed at the edge of each clip
+      const bufferLength = player.sampleTime * player.buffer.length;
+      Tone.Transport.scheduleRepeat((time) => {
+        // TODO(hayk): Edge of clip callback
+        console.log(
+          "Edge of clip, t = ",
+          Tone.Transport.getSecondsAtTime(time),
+          bufferLength
+        );
+        setNumClipsPlayed((n) => n + 1);
+      }, bufferLength);
+
+      setTonePlayer(player);
+
+      // Make further load callbacks do nothing.
+      player.buffer.onload = () => {};
+    }).toDestination();
+  }, [tonePlayer, inferenceResults]);
+
+  // On play/pause button, play/pause the audio with the tone transport
+  useEffect(() => {
+    if (!paused) {
+      console.log("Play");
+
+      if (Tone.context.state == "suspended") {
+        Tone.context.resume();
+      }
+
+      if (tonePlayer) {
+        Tone.Transport.start();
+      }
+    } else {
+      console.log("Pause");
+
+      if (tonePlayer) {
+        Tone.Transport.pause();
+      }
+    }
+  }, [paused, tonePlayer]);
+
+  // If there is a new clip, switch to it
+  useEffect(() => {
+    if (numClipsPlayed == prevNumClipsPlayed) {
+      return;
+    }
+
+    const maxResultCounter = Math.max(
+      ...inferenceResults.map((r) => r.counter)
+    );
+
+    if (maxResultCounter < resultCounter) {
+      console.info(
+        "not picking a new result, none available",
+        resultCounter,
+        maxResultCounter
+      );
+      return;
+    }
+
+    const result = inferenceResults.find(
+      (r: InferenceResult) => r.counter == resultCounter
+    );
+
+    console.log("Incrementing result counter ", resultCounter);
+    setResultCounter((c) => c + 1);
+
+    tonePlayer.load(result.audio).then(() => {
+      console.log("Loaded new audio");
+
+      // Re-jigger the transport so it stops playing old buffers. It seems like this doesn't
+      // introduce a gap, but watch out for that.
+      Tone.Transport.pause();
+      if (!paused) {
+        Tone.Transport.start();
+      }
+    });
+
+    setPrevNumClipsPlayed(numClipsPlayed);
+  }, [
+    numClipsPlayed,
+    prevNumClipsPlayed,
+    resultCounter,
+    inferenceResults,
+    paused,
+    tonePlayer,
+  ]);
+
+  return null;
+}

components/SpectrogramViewer.tsx

@@ -6,12 +6,21 @@ import { InferenceResult } from "../types";
 import HeightMapImage from "./HeightMapImage";
 import ImagePlane from "./ImagePlane";
 
+// import { Effects } from "@react-three/drei";
+// import { ShaderPass, VerticalTiltShiftShader} from "three-stdlib";
+
+// extend({ ShaderPass });
+
+// Fun shaders:
+// RGBShiftShader
+// VerticalBlurShader
+// VerticalTiltShiftShader
+
 interface SpectrogramViewerProps {
   paused: boolean;
   inferenceResults: InferenceResult[];
   getTime: () => number;
   use_height_map?: boolean;
-  audioLength: number;
 }
 
 /**
@@ -21,7 +30,6 @@ export default function SpectrogramViewer({
   paused,
   inferenceResults,
   getTime,
-  audioLength,
   use_height_map = true,
 }: SpectrogramViewerProps) {
   const { camera } = useThree();
@@ -43,24 +51,35 @@ export default function SpectrogramViewer({
 
   return (
     <group>
-      {inferenceResults.map((value: InferenceResult, index: number) => {
-        const position = audioLength * (-0.6 - value.counter) + playbarShift;
+      {/* <Effects
+        multisamping={8}
+        renderIndex={1}
+        disableGamma={false}
+        disableRenderPass={false}
+        disableRender={false}
+      >
+        <shaderPass attachArray="passes" args={[VerticalTiltShiftShader]} />
+      </Effects> */}
+
+      {inferenceResults.map((result: InferenceResult, index: number) => {
+        const duration_s = result.duration_s;
+        const position = duration_s * (-0.6 - result.counter) + playbarShift;
         if (use_height_map) {
           return (
             <HeightMapImage
-              url={value.image}
+              url={result.image}
               position={[0, position, 0]}
               rotation={[0, 0, -Math.PI / 2]}
-              scale={[audioLength, 5, 1]}
+              scale={[duration_s, 5, 1]}
               key={index}
             />
           );
         } else {
           return (
             <ImagePlane
-              url={value.image}
+              url={result.image}
               height={position}
-              duration={audioLength}
+              duration={duration_s}
               key={index}
             />
           );

components/ThreeCanvas.tsx

@@ -8,7 +8,6 @@ interface CanvasProps {
   paused: boolean;
   inferenceResults: InferenceResult[];
   getTime: () => number;
-  audioLength: number;
 }
 
 /**
@@ -18,7 +17,6 @@ export default function ThreeCanvas({
   paused,
   inferenceResults,
   getTime,
-  audioLength,
 }: CanvasProps) {
   return (
     <Canvas>
@@ -35,7 +33,6 @@ export default function ThreeCanvas({
         paused={paused}
         inferenceResults={inferenceResults}
         getTime={getTime}
-        audioLength={audioLength}
       />
     </Canvas>
   );

pages/index.tsx

@@ -4,6 +4,7 @@ import { useEffect, useState } from "react";
 import { useInterval, useTimeout } from "usehooks-ts";
 import * as Tone from "tone";
 
+import AudioPlayer from "../components/AudioPlayer";
 import ThreeCanvas from "../components/ThreeCanvas";
 import PromptPanel from "../components/PromptPanel";
 import Info from "../components/Info";
@@ -38,12 +39,16 @@ const maxLength = 10;
 const maxNumInferenceResults = 15;
 
 export default function Home() {
+  // Create parameters for the inference request
   const [denoising, setDenoising] = useState(0.75);
   const [guidance, setGuidance] = useState(7.0);
   const [numInferenceSteps, setNumInferenceSteps] = useState(50);
   const [seedImageId, setSeedImageId] = useState("og_beat");
   const [maskImageId, setMaskImageId] = useState(null);
 
+  // Create parameters for the app state
+  const [appState, setAppState] = useState<AppState>(AppState.SamePrompt);
+  const [alpha, setAlpha] = useState(0.0);
   const [alphaVelocity, setAlphaVelocity] = useState(0.25);
   const [seed, setSeed] = useState(getRandomInt(1000000));
   const [paused, setPaused] = useState(true);
@@ -101,122 +106,6 @@ export default function Home() {
     []
   );
 
-  const [tonePlayer, setTonePlayer] = useState<Tone.Player>(null);
-
-  const [numClipsPlayed, setNumClipsPlayed] = useState(0);
-  const [prevNumClipsPlayed, setPrevNumClipsPlayed] = useState(0);
-
-  const [resultCounter, setResultCounter] = useState(0);
-
-  const [alpha, setAlpha] = useState(0.0);
-
-  const [appState, setAppState] = useState<AppState>(AppState.SamePrompt);
-
-  // On load, create a player synced to the tone transport
-  useEffect(() => {
-    if (tonePlayer) {
-      return;
-    }
-
-    if (inferenceResults.length === 0) {
-      return;
-    }
-
-    const audioUrl = inferenceResults[0].audio;
-
-    const player = new Tone.Player(audioUrl, () => {
-      console.log("Created player.");
-
-      player.loop = true;
-      player.sync().start(0);
-
-      // Set up a callback to increment numClipsPlayed at the edge of each clip
-      const bufferLength = player.sampleTime * player.buffer.length;
-      Tone.Transport.scheduleRepeat((time) => {
-        console.log(
-          "Edge of clip, t = ",
-          Tone.Transport.getSecondsAtTime(time),
-          bufferLength
-        );
-        setNumClipsPlayed((n) => n + 1);
-      }, bufferLength);
-
-      setTonePlayer(player);
-
-      // Make further load callbacks do nothing.
-      player.buffer.onload = () => {};
-    }).toDestination();
-  }, [tonePlayer, inferenceResults]);
-
-  // On play/pause button, play/pause the audio with the tone transport
-  useEffect(() => {
-    if (!paused) {
-      console.log("Play");
-
-      if (Tone.context.state == "suspended") {
-        Tone.context.resume();
-      }
-
-      if (tonePlayer) {
-        Tone.Transport.start();
-      }
-    } else {
-      console.log("Pause");
-
-      if (tonePlayer) {
-        Tone.Transport.pause();
-      }
-    }
-  }, [paused, tonePlayer]);
-
-  useEffect(() => {
-    if (numClipsPlayed == prevNumClipsPlayed) {
-      return;
-    }
-
-    const maxResultCounter = Math.max(
-      ...inferenceResults.map((r) => r.counter)
-    );
-
-    if (maxResultCounter < resultCounter) {
-      console.info(
-        "not picking a new result, none available",
-        resultCounter,
-        maxResultCounter
-      );
-      return;
-    }
-
-    const result = inferenceResults.find(
-      (r: InferenceResult) => r.counter == resultCounter
-    );
-
-    console.log("Incrementing result counter ", resultCounter);
-    setResultCounter((c) => c + 1);
-
-    tonePlayer.load(result.audio).then(() => {
-      console.log("Loaded new audio");
-
-      // Re-jigger the transport so it stops playing old buffers. It seems like this doesn't
-      // introduce a gap, but watch out for that.
-      Tone.Transport.pause();
-      if (!paused) {
-        Tone.Transport.start();
-      }
-    });
-
-    setPrevNumClipsPlayed(numClipsPlayed);
-  }, [
-    numClipsPlayed,
-    prevNumClipsPlayed,
-    resultCounter,
-    inferenceResults,
-    paused,
-    tonePlayer,
-  ]);
-
-  // /////////////
-
   // Set the app state based on the prompt inputs array
   useEffect(() => {
     if (alpha <= 1) {
@@ -302,18 +191,12 @@ export default function Home() {
 
       const newResult = {
         input: inferenceInput,
-        image: "data:image/jpeg;base64," + data.image,
-        audio: "data:audio/mpeg;base64," + data.audio,
+        image: data.image,
+        audio: data.audio,
+        duration_s: data.duration_s,
         counter: newCounter,
       };
 
-      // TODO(hayk): Fix up
-      // if (alpha > 1.0) {
-      //   setAlpha(alpha - 0.75);
-      //   setSeed(seed + 1);
-      // } else {
-      //   setAlpha(inferenceInput.alpha + alphaVelocity);
-      // }
       setAlpha(alpha + alphaVelocity);
 
       let results = [...prevResults, newResult];
@@ -358,13 +241,12 @@ export default function Home() {
           <ThreeCanvas
             paused={paused}
             getTime={() => Tone.Transport.seconds}
-            audioLength={
-              tonePlayer ? tonePlayer.sampleTime * tonePlayer.buffer.length : 0
-            }
             inferenceResults={inferenceResults}
           />
         </div>
 
+        <AudioPlayer paused={paused} inferenceResults={inferenceResults} />
+
         <PromptPanel
           prompts={promptInputs}
           changePrompt={(prompt: string, index: number) => {

types.ts

@@ -25,4 +25,7 @@ export interface InferenceResult {
 
     // URL of the audio
     audio: string;
+
+    // Duration of the audio in seconds
+    duration_s: number;
 }