From e33ab10a21ff84b0dfad4c3a4954ea0a45c4b8aa Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Sun, 27 Nov 2022 13:53:22 -0800 Subject: [PATCH] Separate AudioPlayer component --- components/AudioPlayer.tsx | 136 ++++++++++++++++++++++++++++++ components/SpectrogramViewer.tsx | 35 ++++++-- components/ThreeCanvas.tsx | 3 - pages/index.tsx | 138 +++---------------------------- types.ts | 3 + 5 files changed, 176 insertions(+), 139 deletions(-) create mode 100644 components/AudioPlayer.tsx diff --git a/components/AudioPlayer.tsx b/components/AudioPlayer.tsx new file mode 100644 index 0000000..1dbbd24 --- /dev/null +++ b/components/AudioPlayer.tsx @@ -0,0 +1,136 @@ +import { useEffect, useState } from "react"; + +import * as Tone from "tone"; + +import { InferenceResult } from "../types"; + +interface AudioPlayerProps { + paused: boolean; + inferenceResults: InferenceResult[]; +} + +/** + * Audio player with Tone.js + * + * TODO(hayk): Look into https://github.com/E-Kuerschner/useAudioPlayer as an alternative. + */ +export default function AudioPlayer({ + paused, + inferenceResults, +}: AudioPlayerProps) { + // TODO(hayk): Rename + const [tonePlayer, setTonePlayer] = useState(null); + + const [numClipsPlayed, setNumClipsPlayed] = useState(0); + const [prevNumClipsPlayed, setPrevNumClipsPlayed] = useState(0); + + // TODO(hayk): What is this? + const [resultCounter, setResultCounter] = useState(0); + + // On load, create a player synced to the tone transport + useEffect(() => { + if (tonePlayer) { + return; + } + + if (inferenceResults.length === 0) { + return; + } + + const audioUrl = inferenceResults[0].audio; + + const player = new Tone.Player(audioUrl, () => { + console.log("Created player."); + + player.loop = true; + player.sync().start(0); + + // Set up a callback to increment numClipsPlayed at the edge of each clip + const bufferLength = player.sampleTime * player.buffer.length; + Tone.Transport.scheduleRepeat((time) => { + // TODO(hayk): Edge of clip callback + console.log( + "Edge of clip, t = ", + Tone.Transport.getSecondsAtTime(time), + bufferLength + ); + setNumClipsPlayed((n) => n + 1); + }, bufferLength); + + setTonePlayer(player); + + // Make further load callbacks do nothing. + player.buffer.onload = () => {}; + }).toDestination(); + }, [tonePlayer, inferenceResults]); + + // On play/pause button, play/pause the audio with the tone transport + useEffect(() => { + if (!paused) { + console.log("Play"); + + if (Tone.context.state == "suspended") { + Tone.context.resume(); + } + + if (tonePlayer) { + Tone.Transport.start(); + } + } else { + console.log("Pause"); + + if (tonePlayer) { + Tone.Transport.pause(); + } + } + }, [paused, tonePlayer]); + + // If there is a new clip, switch to it + useEffect(() => { + if (numClipsPlayed == prevNumClipsPlayed) { + return; + } + + const maxResultCounter = Math.max( + ...inferenceResults.map((r) => r.counter) + ); + + if (maxResultCounter < resultCounter) { + console.info( + "not picking a new result, none available", + resultCounter, + maxResultCounter + ); + return; + } + + const result = inferenceResults.find( + (r: InferenceResult) => r.counter == resultCounter + ); + + console.log("Incrementing result counter ", resultCounter); + setResultCounter((c) => c + 1); + + tonePlayer.load(result.audio).then(() => { + console.log("Loaded new audio"); + + // Re-jigger the transport so it stops playing old buffers. It seems like this doesn't + // introduce a gap, but watch out for that. + Tone.Transport.pause(); + if (!paused) { + Tone.Transport.start(); + } + }); + + setPrevNumClipsPlayed(numClipsPlayed); + }, [ + numClipsPlayed, + prevNumClipsPlayed, + resultCounter, + inferenceResults, + paused, + tonePlayer, + ]); + + return null; +} diff --git a/components/SpectrogramViewer.tsx b/components/SpectrogramViewer.tsx index 82b8609..ce4f299 100644 --- a/components/SpectrogramViewer.tsx +++ b/components/SpectrogramViewer.tsx @@ -6,12 +6,21 @@ import { InferenceResult } from "../types"; import HeightMapImage from "./HeightMapImage"; import ImagePlane from "./ImagePlane"; +// import { Effects } from "@react-three/drei"; +// import { ShaderPass, VerticalTiltShiftShader} from "three-stdlib"; + +// extend({ ShaderPass }); + +// Fun shaders: +// RGBShiftShader +// VerticalBlurShader +// VerticalTiltShiftShader + interface SpectrogramViewerProps { paused: boolean; inferenceResults: InferenceResult[]; getTime: () => number; use_height_map?: boolean; - audioLength: number; } /** @@ -21,7 +30,6 @@ export default function SpectrogramViewer({ paused, inferenceResults, getTime, - audioLength, use_height_map = true, }: SpectrogramViewerProps) { const { camera } = useThree(); @@ -43,24 +51,35 @@ export default function SpectrogramViewer({ return ( - {inferenceResults.map((value: InferenceResult, index: number) => { - const position = audioLength * (-0.6 - value.counter) + playbarShift; + {/* + + */} + + {inferenceResults.map((result: InferenceResult, index: number) => { + const duration_s = result.duration_s; + const position = duration_s * (-0.6 - result.counter) + playbarShift; if (use_height_map) { return ( ); } else { return ( ); diff --git a/components/ThreeCanvas.tsx b/components/ThreeCanvas.tsx index 48d76fb..7a20816 100644 --- a/components/ThreeCanvas.tsx +++ b/components/ThreeCanvas.tsx @@ -8,7 +8,6 @@ interface CanvasProps { paused: boolean; inferenceResults: InferenceResult[]; getTime: () => number; - audioLength: number; } /** @@ -18,7 +17,6 @@ export default function ThreeCanvas({ paused, inferenceResults, getTime, - audioLength, }: CanvasProps) { return ( @@ -35,7 +33,6 @@ export default function ThreeCanvas({ paused={paused} inferenceResults={inferenceResults} getTime={getTime} - audioLength={audioLength} /> ); diff --git a/pages/index.tsx b/pages/index.tsx index 599c51b..ace739e 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -4,6 +4,7 @@ import { useEffect, useState } from "react"; import { useInterval, useTimeout } from "usehooks-ts"; import * as Tone from "tone"; +import AudioPlayer from "../components/AudioPlayer"; import ThreeCanvas from "../components/ThreeCanvas"; import PromptPanel from "../components/PromptPanel"; import Info from "../components/Info"; @@ -38,12 +39,16 @@ const maxLength = 10; const maxNumInferenceResults = 15; export default function Home() { + // Create parameters for the inference request const [denoising, setDenoising] = useState(0.75); const [guidance, setGuidance] = useState(7.0); const [numInferenceSteps, setNumInferenceSteps] = useState(50); const [seedImageId, setSeedImageId] = useState("og_beat"); const [maskImageId, setMaskImageId] = useState(null); + // Create parameters for the app state + const [appState, setAppState] = useState(AppState.SamePrompt); + const [alpha, setAlpha] = useState(0.0); const [alphaVelocity, setAlphaVelocity] = useState(0.25); const [seed, setSeed] = useState(getRandomInt(1000000)); const [paused, setPaused] = useState(true); @@ -101,122 +106,6 @@ export default function Home() { [] ); - const [tonePlayer, setTonePlayer] = useState(null); - - const [numClipsPlayed, setNumClipsPlayed] = useState(0); - const [prevNumClipsPlayed, setPrevNumClipsPlayed] = useState(0); - - const [resultCounter, setResultCounter] = useState(0); - - const [alpha, setAlpha] = useState(0.0); - - const [appState, setAppState] = useState(AppState.SamePrompt); - - // On load, create a player synced to the tone transport - useEffect(() => { - if (tonePlayer) { - return; - } - - if (inferenceResults.length === 0) { - return; - } - - const audioUrl = inferenceResults[0].audio; - - const player = new Tone.Player(audioUrl, () => { - console.log("Created player."); - - player.loop = true; - player.sync().start(0); - - // Set up a callback to increment numClipsPlayed at the edge of each clip - const bufferLength = player.sampleTime * player.buffer.length; - Tone.Transport.scheduleRepeat((time) => { - console.log( - "Edge of clip, t = ", - Tone.Transport.getSecondsAtTime(time), - bufferLength - ); - setNumClipsPlayed((n) => n + 1); - }, bufferLength); - - setTonePlayer(player); - - // Make further load callbacks do nothing. - player.buffer.onload = () => {}; - }).toDestination(); - }, [tonePlayer, inferenceResults]); - - // On play/pause button, play/pause the audio with the tone transport - useEffect(() => { - if (!paused) { - console.log("Play"); - - if (Tone.context.state == "suspended") { - Tone.context.resume(); - } - - if (tonePlayer) { - Tone.Transport.start(); - } - } else { - console.log("Pause"); - - if (tonePlayer) { - Tone.Transport.pause(); - } - } - }, [paused, tonePlayer]); - - useEffect(() => { - if (numClipsPlayed == prevNumClipsPlayed) { - return; - } - - const maxResultCounter = Math.max( - ...inferenceResults.map((r) => r.counter) - ); - - if (maxResultCounter < resultCounter) { - console.info( - "not picking a new result, none available", - resultCounter, - maxResultCounter - ); - return; - } - - const result = inferenceResults.find( - (r: InferenceResult) => r.counter == resultCounter - ); - - console.log("Incrementing result counter ", resultCounter); - setResultCounter((c) => c + 1); - - tonePlayer.load(result.audio).then(() => { - console.log("Loaded new audio"); - - // Re-jigger the transport so it stops playing old buffers. It seems like this doesn't - // introduce a gap, but watch out for that. - Tone.Transport.pause(); - if (!paused) { - Tone.Transport.start(); - } - }); - - setPrevNumClipsPlayed(numClipsPlayed); - }, [ - numClipsPlayed, - prevNumClipsPlayed, - resultCounter, - inferenceResults, - paused, - tonePlayer, - ]); - - // ///////////// - // Set the app state based on the prompt inputs array useEffect(() => { if (alpha <= 1) { @@ -302,18 +191,12 @@ export default function Home() { const newResult = { input: inferenceInput, - image: "data:image/jpeg;base64," + data.image, - audio: "data:audio/mpeg;base64," + data.audio, + image: data.image, + audio: data.audio, + duration_s: data.duration_s, counter: newCounter, }; - // TODO(hayk): Fix up - // if (alpha > 1.0) { - // setAlpha(alpha - 0.75); - // setSeed(seed + 1); - // } else { - // setAlpha(inferenceInput.alpha + alphaVelocity); - // } setAlpha(alpha + alphaVelocity); let results = [...prevResults, newResult]; @@ -358,13 +241,12 @@ export default function Home() { Tone.Transport.seconds} - audioLength={ - tonePlayer ? tonePlayer.sampleTime * tonePlayer.buffer.length : 0 - } inferenceResults={inferenceResults} /> + + { diff --git a/types.ts b/types.ts index 79dedcb..84fc816 100644 --- a/types.ts +++ b/types.ts @@ -25,4 +25,7 @@ export interface InferenceResult { // URL of the audio audio: string; + + // Duration of the audio in seconds + duration_s: number; }