FIRST INFINITE RIFF

package-lock.json

@@ -19,7 +19,8 @@
         "react-icons": "^4.6.0",
         "styled-components": "^5.3.6",
         "three": "^0.146.0",
-        "tone": "^14.7.77"
+        "tone": "^14.7.77",
+        "usehooks-ts": "^2.9.1"
       },
       "devDependencies": {
         "@types/node": "^18.11.9",
@@ -4551,6 +4552,19 @@
         "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       }
     },
+    "node_modules/usehooks-ts": {
+      "version": "2.9.1",
+      "resolved": "https://registry.npmjs.org/usehooks-ts/-/usehooks-ts-2.9.1.tgz",
+      "integrity": "sha512-2FAuSIGHlY+apM9FVlj8/oNhd+1y+Uwv5QNkMQz1oSfdHk4PXo1qoCw9I5M7j0vpH8CSWFJwXbVPeYDjLCx9PA==",
+      "engines": {
+        "node": ">=16.15.0",
+        "npm": ">=8"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0  || ^17.0.0 || ^18.0.0",
+        "react-dom": "^16.8.0  || ^17.0.0 || ^18.0.0"
+      }
+    },
     "node_modules/util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -7816,6 +7830,12 @@
       "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==",
       "requires": {}
     },
+    "usehooks-ts": {
+      "version": "2.9.1",
+      "resolved": "https://registry.npmjs.org/usehooks-ts/-/usehooks-ts-2.9.1.tgz",
+      "integrity": "sha512-2FAuSIGHlY+apM9FVlj8/oNhd+1y+Uwv5QNkMQz1oSfdHk4PXo1qoCw9I5M7j0vpH8CSWFJwXbVPeYDjLCx9PA==",
+      "requires": {}
+    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

package.json

@@ -20,7 +20,8 @@
     "react-icons": "^4.6.0",
     "styled-components": "^5.3.6",
     "three": "^0.146.0",
-    "tone": "^14.7.77"
+    "tone": "^14.7.77",
+    "usehooks-ts": "^2.9.1"
   },
   "devDependencies": {
     "@types/node": "^18.11.9",

pages/index.tsx

@@ -1,7 +1,6 @@
 import Head from "next/head";
-import { useEffect, useState, useRef } from "react";
+import { useEffect, useState } from "react";
-
+import { useInterval } from "usehooks-ts";
-import Image from "next/image";
 
 import ThreeCanvas from "../components/ThreeCanvas";
 import PromptPanel from "../components/PromptPanel";
@@ -11,6 +10,7 @@ import Pause from "../components/Pause";
 import { InferenceResult, PromptInput } from "../types";
 
 import * as Tone from "tone";
+import { start } from "repl";
 
 const SERVER_URL = "http://129.146.52.68:3013/run_inference/";
 
@@ -36,11 +36,10 @@ const urlToBase64 = async (url: string) => {
   });
 };
 
-const defaultInferenceResults = [];
-
 // TODO(hayk): Do this as soon as sample comes back
 const timeout = 5150;
 const maxLength = 10;
+const alphaVelocity = 0.25;
 
 export default function Home() {
   const [paused, setPaused] = useState(true);
@@ -49,7 +48,7 @@ export default function Home() {
     useState<PromptInput[]>(defaultPromptInputs);
 
   const [inferenceResults, setInferenceResults] = useState<InferenceResult[]>(
-    defaultInferenceResults
+    []
   );
 
   // /////////////
@@ -61,10 +60,16 @@ export default function Home() {
 
   const [resultCounter, setResultCounter] = useState(0);
 
-  // const [randomImgSrc, setRandomImgSrc] = useState("");
+  const [alpha, setAlpha] = useState(0.0);
+  const [seed, setSeed] = useState(0);
 
   // On load, populate the first two prompts from checked-in URLs
   useEffect(() => {
+    // NOTE(hayk): not currently populating initial prompts.
+    if (true) {
+      return;
+    }
+
     if (inferenceResults.length > 0) {
       return;
     }
@@ -76,8 +81,8 @@ export default function Home() {
           start: defaultPromptInputs[0],
           end: defaultPromptInputs[1],
         },
-        image: await urlToBase64("rap_sample.jpg") as string,
+        image: (await urlToBase64("rap_sample.jpg")) as string,
-        audio: await urlToBase64("rap_sample.mp3") as string,
+        audio: (await urlToBase64("rap_sample.mp3")) as string,
         counter: 0,
       };
 
@@ -87,12 +92,13 @@ export default function Home() {
           start: defaultPromptInputs[0],
           end: defaultPromptInputs[1],
         },
-        image: await urlToBase64("pop_sample.jpg") as string,
+        image: (await urlToBase64("pop_sample.jpg")) as string,
-        audio: await urlToBase64("pop_sample.mp3") as string,
+        audio: (await urlToBase64("pop_sample.mp3")) as string,
         counter: 1,
       };
 
-      setInferenceResults([...inferenceResults, result1, result2]);
+      console.log(result1);
+      setInferenceResults([...inferenceResults, result1]);
     };
 
     populateDefaults();
@@ -164,7 +170,11 @@ export default function Home() {
     );
 
     if (maxResultCounter < resultCounter) {
-      console.info("not picking a new result, none available", resultCounter, maxResultCounter);
+      console.info(
+        "not picking a new result, none available",
+        resultCounter,
+        maxResultCounter
+      );
       return;
     }
 
@@ -172,7 +182,7 @@ export default function Home() {
       (r: InferenceResult) => r.counter == resultCounter
     );
 
-    console.log("Incrementing result counter ", resultCounter, result);
+    console.log("Incrementing result counter ", resultCounter);
     setResultCounter((c) => c + 1);
 
     tonePlayer.load(result.audio).then(() => {
@@ -198,69 +208,80 @@ export default function Home() {
 
   // /////////////
 
-  // Request a new inference prompt at a regular interval
+  const runInference = async (alpha: number, seed: number, promptInputs: PromptInput[]) => {
-  // TODO(hayk): Rewrite this to request more frequently and max out
+    console.log(promptInputs);
-  // useEffect(() => {
+    const startPrompt = promptInputs[promptInputs.length - 3].prompt;
-  //   console.log("setInterval");
+    const endPrompt = promptInputs[promptInputs.length - 2].prompt;
-  //   setInterval(() => {
-  //     setInferenceResults((prevResults) => {
-  //       const lastResult = prevResults[prevResults.length - 2];
-  //       const newResult = { ...lastResult, counter: lastResult.counter + 2 };
 
-  //       let results = [...prevResults, newResult];
+    const inferenceInput = {
+      alpha: alpha,
+      start: {
+        prompt: startPrompt,
+        seed: seed,
+      },
+      end: {
+        prompt: endPrompt ? endPrompt : startPrompt,
+        seed: endPrompt ? seed : seed + 1,
+      },
+    };
 
-  //       if (results.length > maxLength) {
+    console.log("Running for input: ", inferenceInput);
-  //         results = results.slice(1);
-  //       }
 
-  //       console.log(results);
+    const response = await fetch(SERVER_URL, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Access-Control-Allow-Origin": "*",
+      },
+      body: JSON.stringify(inferenceInput),
+    });
 
-  //       return results;
+    const data = await response.json();
-  //     });
-  //   }, timeout);
-  // }, []);
 
-  // useEffect(() => {
+    setInferenceResults((prevResults) => {
-  //   console.log("FETCHING");
+      const maxResultCounter = Math.max(...prevResults.map((r) => r.counter));
-  //   fetch(SERVER_URL, {
-  //     method: "POST",
-  //     headers: {
-  //       "Content-Type": "application/json",
-  //       "Access-Control-Allow-Origin": "*",
-  //     },
-  //     body: JSON.stringify({
-  //       alpha: 0.0,
-  //       start: {
-  //         prompt: "bieber grunge",
-  //         seed: 42,
-  //       },
-  //       end: {
-  //         prompt: "eminem violin",
-  //         seed: 10,
-  //       },
-  //     }),
-  //   })
-  //     .then((response) => {
-  //       console.log("Got response!");
-  //       response.json().then((data) => {
-  //         const imageB64Bytes = data.image;
-  //         // const imageBytes = Buffer.from(imageB64Bytes, "base64").toString();
-  //         // const image_bytes = new TextDecoder("utf-8").decode(imageB64Bytes);
 
-  //         const audioB64Bytes = data.audio;
+      const lastResult = prevResults.find((r) => r.counter == maxResultCounter);
-  //         var snd = new Audio("data:audio/mp3;base64," + audioB64Bytes);
-  //         snd.play();
 
-  //         // const audio_bytes = new TextDecoder("utf-8").decode(audioB64Bytes);
+      const newCounter = lastResult ? lastResult.counter + 1 : 0;
+      console.log("newCounter", newCounter);
 
-  //         // console.log(imageBytes);
+      const newResult = {
-  //         setRandomImgSrc(imageB64Bytes);
+        input: inferenceInput,
-  //       });
+        // TODO(hayk): Swap for JPG?
-  //     })
+        image: "data:image/png;base64," + data.image,
-  //     .catch((error) => {
+        audio: "data:audio/mpeg;base64," + data.audio,
-  //       console.error(error);
+        counter: newCounter,
-  //     });
+      };
-  // }, []);
+
+      // TODO(hayk): Fix up
+      if (alpha >= 1.0) {
+        setAlpha(alpha - 0.75);
+        setSeed(seed + 1);
+      } else {
+        setAlpha((a) => a + alphaVelocity);
+      }
+
+      let results = [...prevResults, newResult];
+
+      // TODO(hayk): Move this somewhere more reasonable to prune.
+      if (results.length > maxLength) {
+        results = results.slice(1);
+      }
+
+      return results;
+    });
+  };
+
+  useInterval(() => {
+    runInference(alpha, seed, promptInputs);
+  }, timeout);
+
+  // Run inference on a timer.
+  // TODO(hayk): Improve the timing here.
+  useEffect(() => {
+    runInference(alpha, seed, promptInputs);
+  }, []);
 
   return (
     <>
@@ -285,17 +306,6 @@ export default function Home() {
           />
         </div>
 
-        {/* TODO(hayk): Kill this, just for testing. */}
-        {/* {randomImgSrc && (
-          <Image
-            unoptimized
-            src={`data:image/png;base64,${randomImgSrc}`}
-            alt={"random image"}
-            width={256}
-            height={256}
-          />
-        )} */}
-
         <PromptPanel
           prompts={promptInputs}
           addPrompt={(prompt: string) => {