Separate AudioPlayer component
This commit is contained in:
parent
5e381e59d1
commit
e33ab10a21
|
@ -0,0 +1,136 @@
|
|||
import { useEffect, useState } from "react";
|
||||
|
||||
import * as Tone from "tone";
|
||||
|
||||
import { InferenceResult } from "../types";
|
||||
|
||||
interface AudioPlayerProps {
|
||||
paused: boolean;
|
||||
inferenceResults: InferenceResult[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Audio player with Tone.js
|
||||
*
|
||||
* TODO(hayk): Look into https://github.com/E-Kuerschner/useAudioPlayer as an alternative.
|
||||
*/
|
||||
export default function AudioPlayer({
|
||||
paused,
|
||||
inferenceResults,
|
||||
}: AudioPlayerProps) {
|
||||
// TODO(hayk): Rename
|
||||
const [tonePlayer, setTonePlayer] = useState<Tone.Player>(null);
|
||||
|
||||
const [numClipsPlayed, setNumClipsPlayed] = useState(0);
|
||||
const [prevNumClipsPlayed, setPrevNumClipsPlayed] = useState(0);
|
||||
|
||||
// TODO(hayk): What is this?
|
||||
const [resultCounter, setResultCounter] = useState(0);
|
||||
|
||||
// On load, create a player synced to the tone transport
|
||||
useEffect(() => {
|
||||
if (tonePlayer) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (inferenceResults.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const audioUrl = inferenceResults[0].audio;
|
||||
|
||||
const player = new Tone.Player(audioUrl, () => {
|
||||
console.log("Created player.");
|
||||
|
||||
player.loop = true;
|
||||
player.sync().start(0);
|
||||
|
||||
// Set up a callback to increment numClipsPlayed at the edge of each clip
|
||||
const bufferLength = player.sampleTime * player.buffer.length;
|
||||
Tone.Transport.scheduleRepeat((time) => {
|
||||
// TODO(hayk): Edge of clip callback
|
||||
console.log(
|
||||
"Edge of clip, t = ",
|
||||
Tone.Transport.getSecondsAtTime(time),
|
||||
bufferLength
|
||||
);
|
||||
setNumClipsPlayed((n) => n + 1);
|
||||
}, bufferLength);
|
||||
|
||||
setTonePlayer(player);
|
||||
|
||||
// Make further load callbacks do nothing.
|
||||
player.buffer.onload = () => {};
|
||||
}).toDestination();
|
||||
}, [tonePlayer, inferenceResults]);
|
||||
|
||||
// On play/pause button, play/pause the audio with the tone transport
|
||||
useEffect(() => {
|
||||
if (!paused) {
|
||||
console.log("Play");
|
||||
|
||||
if (Tone.context.state == "suspended") {
|
||||
Tone.context.resume();
|
||||
}
|
||||
|
||||
if (tonePlayer) {
|
||||
Tone.Transport.start();
|
||||
}
|
||||
} else {
|
||||
console.log("Pause");
|
||||
|
||||
if (tonePlayer) {
|
||||
Tone.Transport.pause();
|
||||
}
|
||||
}
|
||||
}, [paused, tonePlayer]);
|
||||
|
||||
// If there is a new clip, switch to it
|
||||
useEffect(() => {
|
||||
if (numClipsPlayed == prevNumClipsPlayed) {
|
||||
return;
|
||||
}
|
||||
|
||||
const maxResultCounter = Math.max(
|
||||
...inferenceResults.map((r) => r.counter)
|
||||
);
|
||||
|
||||
if (maxResultCounter < resultCounter) {
|
||||
console.info(
|
||||
"not picking a new result, none available",
|
||||
resultCounter,
|
||||
maxResultCounter
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const result = inferenceResults.find(
|
||||
(r: InferenceResult) => r.counter == resultCounter
|
||||
);
|
||||
|
||||
console.log("Incrementing result counter ", resultCounter);
|
||||
setResultCounter((c) => c + 1);
|
||||
|
||||
tonePlayer.load(result.audio).then(() => {
|
||||
console.log("Loaded new audio");
|
||||
|
||||
// Re-jigger the transport so it stops playing old buffers. It seems like this doesn't
|
||||
// introduce a gap, but watch out for that.
|
||||
Tone.Transport.pause();
|
||||
if (!paused) {
|
||||
Tone.Transport.start();
|
||||
}
|
||||
});
|
||||
|
||||
setPrevNumClipsPlayed(numClipsPlayed);
|
||||
}, [
|
||||
numClipsPlayed,
|
||||
prevNumClipsPlayed,
|
||||
resultCounter,
|
||||
inferenceResults,
|
||||
paused,
|
||||
tonePlayer,
|
||||
]);
|
||||
|
||||
return null;
|
||||
}
|
|
@ -6,12 +6,21 @@ import { InferenceResult } from "../types";
|
|||
import HeightMapImage from "./HeightMapImage";
|
||||
import ImagePlane from "./ImagePlane";
|
||||
|
||||
// import { Effects } from "@react-three/drei";
|
||||
// import { ShaderPass, VerticalTiltShiftShader} from "three-stdlib";
|
||||
|
||||
// extend({ ShaderPass });
|
||||
|
||||
// Fun shaders:
|
||||
// RGBShiftShader
|
||||
// VerticalBlurShader
|
||||
// VerticalTiltShiftShader
|
||||
|
||||
interface SpectrogramViewerProps {
|
||||
paused: boolean;
|
||||
inferenceResults: InferenceResult[];
|
||||
getTime: () => number;
|
||||
use_height_map?: boolean;
|
||||
audioLength: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -21,7 +30,6 @@ export default function SpectrogramViewer({
|
|||
paused,
|
||||
inferenceResults,
|
||||
getTime,
|
||||
audioLength,
|
||||
use_height_map = true,
|
||||
}: SpectrogramViewerProps) {
|
||||
const { camera } = useThree();
|
||||
|
@ -43,24 +51,35 @@ export default function SpectrogramViewer({
|
|||
|
||||
return (
|
||||
<group>
|
||||
{inferenceResults.map((value: InferenceResult, index: number) => {
|
||||
const position = audioLength * (-0.6 - value.counter) + playbarShift;
|
||||
{/* <Effects
|
||||
multisamping={8}
|
||||
renderIndex={1}
|
||||
disableGamma={false}
|
||||
disableRenderPass={false}
|
||||
disableRender={false}
|
||||
>
|
||||
<shaderPass attachArray="passes" args={[VerticalTiltShiftShader]} />
|
||||
</Effects> */}
|
||||
|
||||
{inferenceResults.map((result: InferenceResult, index: number) => {
|
||||
const duration_s = result.duration_s;
|
||||
const position = duration_s * (-0.6 - result.counter) + playbarShift;
|
||||
if (use_height_map) {
|
||||
return (
|
||||
<HeightMapImage
|
||||
url={value.image}
|
||||
url={result.image}
|
||||
position={[0, position, 0]}
|
||||
rotation={[0, 0, -Math.PI / 2]}
|
||||
scale={[audioLength, 5, 1]}
|
||||
scale={[duration_s, 5, 1]}
|
||||
key={index}
|
||||
/>
|
||||
);
|
||||
} else {
|
||||
return (
|
||||
<ImagePlane
|
||||
url={value.image}
|
||||
url={result.image}
|
||||
height={position}
|
||||
duration={audioLength}
|
||||
duration={duration_s}
|
||||
key={index}
|
||||
/>
|
||||
);
|
||||
|
|
|
@ -8,7 +8,6 @@ interface CanvasProps {
|
|||
paused: boolean;
|
||||
inferenceResults: InferenceResult[];
|
||||
getTime: () => number;
|
||||
audioLength: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -18,7 +17,6 @@ export default function ThreeCanvas({
|
|||
paused,
|
||||
inferenceResults,
|
||||
getTime,
|
||||
audioLength,
|
||||
}: CanvasProps) {
|
||||
return (
|
||||
<Canvas>
|
||||
|
@ -35,7 +33,6 @@ export default function ThreeCanvas({
|
|||
paused={paused}
|
||||
inferenceResults={inferenceResults}
|
||||
getTime={getTime}
|
||||
audioLength={audioLength}
|
||||
/>
|
||||
</Canvas>
|
||||
);
|
||||
|
|
138
pages/index.tsx
138
pages/index.tsx
|
@ -4,6 +4,7 @@ import { useEffect, useState } from "react";
|
|||
import { useInterval, useTimeout } from "usehooks-ts";
|
||||
import * as Tone from "tone";
|
||||
|
||||
import AudioPlayer from "../components/AudioPlayer";
|
||||
import ThreeCanvas from "../components/ThreeCanvas";
|
||||
import PromptPanel from "../components/PromptPanel";
|
||||
import Info from "../components/Info";
|
||||
|
@ -38,12 +39,16 @@ const maxLength = 10;
|
|||
const maxNumInferenceResults = 15;
|
||||
|
||||
export default function Home() {
|
||||
// Create parameters for the inference request
|
||||
const [denoising, setDenoising] = useState(0.75);
|
||||
const [guidance, setGuidance] = useState(7.0);
|
||||
const [numInferenceSteps, setNumInferenceSteps] = useState(50);
|
||||
const [seedImageId, setSeedImageId] = useState("og_beat");
|
||||
const [maskImageId, setMaskImageId] = useState(null);
|
||||
|
||||
// Create parameters for the app state
|
||||
const [appState, setAppState] = useState<AppState>(AppState.SamePrompt);
|
||||
const [alpha, setAlpha] = useState(0.0);
|
||||
const [alphaVelocity, setAlphaVelocity] = useState(0.25);
|
||||
const [seed, setSeed] = useState(getRandomInt(1000000));
|
||||
const [paused, setPaused] = useState(true);
|
||||
|
@ -101,122 +106,6 @@ export default function Home() {
|
|||
[]
|
||||
);
|
||||
|
||||
const [tonePlayer, setTonePlayer] = useState<Tone.Player>(null);
|
||||
|
||||
const [numClipsPlayed, setNumClipsPlayed] = useState(0);
|
||||
const [prevNumClipsPlayed, setPrevNumClipsPlayed] = useState(0);
|
||||
|
||||
const [resultCounter, setResultCounter] = useState(0);
|
||||
|
||||
const [alpha, setAlpha] = useState(0.0);
|
||||
|
||||
const [appState, setAppState] = useState<AppState>(AppState.SamePrompt);
|
||||
|
||||
// On load, create a player synced to the tone transport
|
||||
useEffect(() => {
|
||||
if (tonePlayer) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (inferenceResults.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const audioUrl = inferenceResults[0].audio;
|
||||
|
||||
const player = new Tone.Player(audioUrl, () => {
|
||||
console.log("Created player.");
|
||||
|
||||
player.loop = true;
|
||||
player.sync().start(0);
|
||||
|
||||
// Set up a callback to increment numClipsPlayed at the edge of each clip
|
||||
const bufferLength = player.sampleTime * player.buffer.length;
|
||||
Tone.Transport.scheduleRepeat((time) => {
|
||||
console.log(
|
||||
"Edge of clip, t = ",
|
||||
Tone.Transport.getSecondsAtTime(time),
|
||||
bufferLength
|
||||
);
|
||||
setNumClipsPlayed((n) => n + 1);
|
||||
}, bufferLength);
|
||||
|
||||
setTonePlayer(player);
|
||||
|
||||
// Make further load callbacks do nothing.
|
||||
player.buffer.onload = () => {};
|
||||
}).toDestination();
|
||||
}, [tonePlayer, inferenceResults]);
|
||||
|
||||
// On play/pause button, play/pause the audio with the tone transport
|
||||
useEffect(() => {
|
||||
if (!paused) {
|
||||
console.log("Play");
|
||||
|
||||
if (Tone.context.state == "suspended") {
|
||||
Tone.context.resume();
|
||||
}
|
||||
|
||||
if (tonePlayer) {
|
||||
Tone.Transport.start();
|
||||
}
|
||||
} else {
|
||||
console.log("Pause");
|
||||
|
||||
if (tonePlayer) {
|
||||
Tone.Transport.pause();
|
||||
}
|
||||
}
|
||||
}, [paused, tonePlayer]);
|
||||
|
||||
useEffect(() => {
|
||||
if (numClipsPlayed == prevNumClipsPlayed) {
|
||||
return;
|
||||
}
|
||||
|
||||
const maxResultCounter = Math.max(
|
||||
...inferenceResults.map((r) => r.counter)
|
||||
);
|
||||
|
||||
if (maxResultCounter < resultCounter) {
|
||||
console.info(
|
||||
"not picking a new result, none available",
|
||||
resultCounter,
|
||||
maxResultCounter
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const result = inferenceResults.find(
|
||||
(r: InferenceResult) => r.counter == resultCounter
|
||||
);
|
||||
|
||||
console.log("Incrementing result counter ", resultCounter);
|
||||
setResultCounter((c) => c + 1);
|
||||
|
||||
tonePlayer.load(result.audio).then(() => {
|
||||
console.log("Loaded new audio");
|
||||
|
||||
// Re-jigger the transport so it stops playing old buffers. It seems like this doesn't
|
||||
// introduce a gap, but watch out for that.
|
||||
Tone.Transport.pause();
|
||||
if (!paused) {
|
||||
Tone.Transport.start();
|
||||
}
|
||||
});
|
||||
|
||||
setPrevNumClipsPlayed(numClipsPlayed);
|
||||
}, [
|
||||
numClipsPlayed,
|
||||
prevNumClipsPlayed,
|
||||
resultCounter,
|
||||
inferenceResults,
|
||||
paused,
|
||||
tonePlayer,
|
||||
]);
|
||||
|
||||
// /////////////
|
||||
|
||||
// Set the app state based on the prompt inputs array
|
||||
useEffect(() => {
|
||||
if (alpha <= 1) {
|
||||
|
@ -302,18 +191,12 @@ export default function Home() {
|
|||
|
||||
const newResult = {
|
||||
input: inferenceInput,
|
||||
image: "data:image/jpeg;base64," + data.image,
|
||||
audio: "data:audio/mpeg;base64," + data.audio,
|
||||
image: data.image,
|
||||
audio: data.audio,
|
||||
duration_s: data.duration_s,
|
||||
counter: newCounter,
|
||||
};
|
||||
|
||||
// TODO(hayk): Fix up
|
||||
// if (alpha > 1.0) {
|
||||
// setAlpha(alpha - 0.75);
|
||||
// setSeed(seed + 1);
|
||||
// } else {
|
||||
// setAlpha(inferenceInput.alpha + alphaVelocity);
|
||||
// }
|
||||
setAlpha(alpha + alphaVelocity);
|
||||
|
||||
let results = [...prevResults, newResult];
|
||||
|
@ -358,13 +241,12 @@ export default function Home() {
|
|||
<ThreeCanvas
|
||||
paused={paused}
|
||||
getTime={() => Tone.Transport.seconds}
|
||||
audioLength={
|
||||
tonePlayer ? tonePlayer.sampleTime * tonePlayer.buffer.length : 0
|
||||
}
|
||||
inferenceResults={inferenceResults}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<AudioPlayer paused={paused} inferenceResults={inferenceResults} />
|
||||
|
||||
<PromptPanel
|
||||
prompts={promptInputs}
|
||||
changePrompt={(prompt: string, index: number) => {
|
||||
|
|
Loading…
Reference in New Issue