diff --git a/components/ImagePlane.tsx b/components/ImagePlane.tsx index 3f0d06d..acd99e5 100644 --- a/components/ImagePlane.tsx +++ b/components/ImagePlane.tsx @@ -1,11 +1,10 @@ import * as THREE from "three"; -import React, { useRef, Suspense } from "react"; -import { useLoader, useFrame } from "@react-three/fiber"; +import React, { Suspense } from "react"; +import { useLoader } from "@react-three/fiber"; interface ImagePlaneProps { url: string; height: number; - paused: boolean; } /** @@ -14,20 +13,10 @@ interface ImagePlaneProps { export default function ImagePlane(props: ImagePlaneProps) { const texture = useLoader(THREE.TextureLoader, props.url); - const mesh = useRef(null); - - useFrame(() => { - if (props.paused) { - return; - } - - mesh.current.position.y += 1 / 30.0; - }); - return ( - - + + diff --git a/components/RotatingBox.tsx b/components/RotatingBox.tsx index a75299c..25f7e94 100644 --- a/components/RotatingBox.tsx +++ b/components/RotatingBox.tsx @@ -27,7 +27,7 @@ export default function RotatingBox(props: JSX.IntrinsicElements["mesh"]) { smoothness={4} {...props} ref={mesh} - scale={active ? [3, 3, 3] : [2, 2, 2]} + scale={active ? [.3, .3, .3] : [.2, .2, .2]} onClick={() => setActive(!active)} onPointerOver={() => setHover(true)} onPointerOut={() => setHover(false)} diff --git a/components/SpectrogramViewer.tsx b/components/SpectrogramViewer.tsx new file mode 100644 index 0000000..2275cee --- /dev/null +++ b/components/SpectrogramViewer.tsx @@ -0,0 +1,56 @@ +import ImagePlane from "./ImagePlane"; + +import { useRef } from "react"; +import { useFrame, useThree } from "@react-three/fiber"; + +import { InferenceResult } from "../types"; +import { QuadraticBezierLine } from "@react-three/drei"; + +interface SpectrogramViewerProps { + paused: boolean; + inferenceResults: InferenceResult[]; +} + +/** + * Spectrogram drawing code. + */ +export default function SpectrogramViewer(props: SpectrogramViewerProps) { + const paused = props.paused; + const inferenceResults = props.inferenceResults; + + const camera = useThree((state) => state.camera); + + const playheadRef = useRef(null); + + // Move the camera and playhead every frame. + useFrame(() => { + if (paused) { + return; + } + + const framerate = 120.0; + camera.position.y -= 1 / framerate; + playheadRef.current.position.y -= 1 / framerate; + }); + + return ( + + {inferenceResults.map((value: InferenceResult, index: number) => { + const height = 5 * (-1 - value.counter) - 2; + return ; + })} + + {/* TODO make into playhead component */} + + + + + ); +} diff --git a/components/ThreeCanvas.tsx b/components/ThreeCanvas.tsx index 2dc763a..48d7bbd 100644 --- a/components/ThreeCanvas.tsx +++ b/components/ThreeCanvas.tsx @@ -1,34 +1,25 @@ import { Canvas } from "@react-three/fiber"; -import RotatingBox from "./RotatingBox"; -import ImagePlane from "./ImagePlane"; +import { InferenceResult } from "../types"; +import SpectrogramViewer from "./SpectrogramViewer"; interface CanvasProps { paused: boolean; + inferenceResults: InferenceResult[]; } /** * React three fiber canvas with spectrogram drawing. */ export default function ThreeCanvas(props: CanvasProps) { - // change the image URL - const spectrogram_image = "spectrogram.jpeg"; - - const height = -30.0; - return ( - + - - - - - ); } diff --git a/package-lock.json b/package-lock.json index 03fdc71..9b3df45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,8 @@ "react": "18.2.0", "react-dom": "18.2.0", "react-icons": "^4.6.0", - "three": "^0.146.0" + "three": "^0.146.0", + "tone": "^14.7.77" }, "devDependencies": { "@types/node": "^18.11.9", @@ -938,6 +939,18 @@ "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==" }, + "node_modules/automation-events": { + "version": "4.0.24", + "resolved": "https://registry.npmjs.org/automation-events/-/automation-events-4.0.24.tgz", + "integrity": "sha512-v/c//VCE1TGU3R56cyTFnebXFR3NOOx5Kiuaz7TXwLGpYMp9VgKCwelODRW3ZZ5TP2OFymfrWsvNVuPSi6vVPQ==", + "dependencies": { + "@babel/runtime": "^7.20.1", + "tslib": "^2.4.1" + }, + "engines": { + "node": ">=12.20.1" + } + }, "node_modules/autoprefixer": { "version": "10.4.13", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz", @@ -3642,6 +3655,16 @@ "node": ">=0.10.0" } }, + "node_modules/standardized-audio-context": { + "version": "25.3.35", + "resolved": "https://registry.npmjs.org/standardized-audio-context/-/standardized-audio-context-25.3.35.tgz", + "integrity": "sha512-61KBkor9fOsMEl3NZllnis8KDmTJzonuuW8h6JBmxXXcUDucypGCJp+hlbyKG+olav8wgA+VKlqFmiJEbh4CfA==", + "dependencies": { + "@babel/runtime": "^7.20.1", + "automation-events": "^4.0.24", + "tslib": "^2.4.1" + } + }, "node_modules/stats.js": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/stats.js/-/stats.js-0.17.0.tgz", @@ -3906,6 +3929,15 @@ "node": ">=8.0" } }, + "node_modules/tone": { + "version": "14.7.77", + "resolved": "https://registry.npmjs.org/tone/-/tone-14.7.77.tgz", + "integrity": "sha512-tCfK73IkLHyzoKUvGq47gyDyxiKLFvKiVCOobynGgBB9Dl0NkxTM2p+eRJXyCYrjJwy9Y0XCMqD3uOYsYt2Fdg==", + "dependencies": { + "standardized-audio-context": "^25.1.8", + "tslib": "^2.0.1" + } + }, "node_modules/troika-three-text": { "version": "0.46.4", "resolved": "https://registry.npmjs.org/troika-three-text/-/troika-three-text-0.46.4.tgz", @@ -4770,6 +4802,15 @@ "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==" }, + "automation-events": { + "version": "4.0.24", + "resolved": "https://registry.npmjs.org/automation-events/-/automation-events-4.0.24.tgz", + "integrity": "sha512-v/c//VCE1TGU3R56cyTFnebXFR3NOOx5Kiuaz7TXwLGpYMp9VgKCwelODRW3ZZ5TP2OFymfrWsvNVuPSi6vVPQ==", + "requires": { + "@babel/runtime": "^7.20.1", + "tslib": "^2.4.1" + } + }, "autoprefixer": { "version": "10.4.13", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz", @@ -6645,6 +6686,16 @@ "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==" }, + "standardized-audio-context": { + "version": "25.3.35", + "resolved": "https://registry.npmjs.org/standardized-audio-context/-/standardized-audio-context-25.3.35.tgz", + "integrity": "sha512-61KBkor9fOsMEl3NZllnis8KDmTJzonuuW8h6JBmxXXcUDucypGCJp+hlbyKG+olav8wgA+VKlqFmiJEbh4CfA==", + "requires": { + "@babel/runtime": "^7.20.1", + "automation-events": "^4.0.24", + "tslib": "^2.4.1" + } + }, "stats.js": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/stats.js/-/stats.js-0.17.0.tgz", @@ -6836,6 +6887,15 @@ "is-number": "^7.0.0" } }, + "tone": { + "version": "14.7.77", + "resolved": "https://registry.npmjs.org/tone/-/tone-14.7.77.tgz", + "integrity": "sha512-tCfK73IkLHyzoKUvGq47gyDyxiKLFvKiVCOobynGgBB9Dl0NkxTM2p+eRJXyCYrjJwy9Y0XCMqD3uOYsYt2Fdg==", + "requires": { + "standardized-audio-context": "^25.1.8", + "tslib": "^2.0.1" + } + }, "troika-three-text": { "version": "0.46.4", "resolved": "https://registry.npmjs.org/troika-three-text/-/troika-three-text-0.46.4.tgz", diff --git a/package.json b/package.json index bd45de8..8571a06 100644 --- a/package.json +++ b/package.json @@ -9,16 +9,17 @@ "lint": "next lint" }, "dependencies": { + "@headlessui/react": "^1.5.0", "@react-three/drei": "^9.41.2", "@react-three/fiber": "^8.9.1", "eslint": "8.27.0", "eslint-config-next": "13.0.4", - "@headlessui/react": "^1.5.0", "next": "13.0.4", "react": "18.2.0", "react-dom": "18.2.0", "react-icons": "^4.6.0", - "three": "^0.146.0" + "three": "^0.146.0", + "tone": "^14.7.77" }, "devDependencies": { "@types/node": "^18.11.9", diff --git a/pages/index.tsx b/pages/index.tsx index 3c9d86a..f5bcb15 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -1,5 +1,5 @@ import Head from "next/head"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import ThreeCanvas from "../components/ThreeCanvas"; import PromptPanel from "../components/PromptPanel"; @@ -8,17 +8,81 @@ import Pause from "../components/Pause"; import { InferenceResult, PromptInput } from "../types"; +import * as Tone from "tone"; + const defaultPromptInputs = [ { prompt: "A jazz pianist playing a classical concerto", seed: 10 }, { prompt: "Taylor Swift singing with a tropical beat", seed: 10 }, ]; +const defaultInferenceResults = [ + { + input: { + alpha: 0.0, + start: defaultPromptInputs[0], + end: defaultPromptInputs[1], + }, + image: "rap_sample.jpg", + audio: "rap_sample.mp3", + counter: 0, + }, +]; + +const timeout = 5000; +const maxLength = 10; + export default function Home() { const [paused, setPaused] = useState(false); const [promptInputs, setPromptInputs] = useState(defaultPromptInputs); + const [inferenceResults, setInferenceResults] = useState( + defaultInferenceResults + ); + + // ///////////// + + const [tonePlayer, setTonePlayer] = useState(null); + + useEffect(() => { + setTonePlayer( + new Tone.Player(defaultInferenceResults[0].audio).toDestination() + ); + // play as soon as the buffer is loaded + // player.autostart = true; + }, [inferenceResults]); + + useEffect(() => { + if (tonePlayer && tonePlayer.loaded) { + if (!paused) { + tonePlayer.start(); + } else { + tonePlayer.stop(); + } + } + }, [paused, tonePlayer]); + + // ///////////// + + useEffect(() => { + setTimeout(() => { + const lastResult = inferenceResults[inferenceResults.length - 1]; + const newResult = { ...lastResult, counter: lastResult.counter + 1 }; + + let results = [...inferenceResults, newResult]; + + if (results.length > maxLength) { + results = results.slice(1); + } + + console.log("Adding to inference results"); + console.log(results); + + setInferenceResults(results); + }, timeout); + }); + return ( <> @@ -32,7 +96,7 @@ export default function Home() {
- +