scripts, guides

- auto download popular scripts
- new block to download any url you give it to the scripts folder
- update guide archvies

StableDiffusionUI_Voldemort_paperspace.ipynb

@@ -48,11 +48,11 @@
     "\n",
     "`/notebooks/` is storage for this notebook only.\n",
     "\n",
-    "<mark style=\"background-color:lime\">`/tmp/` is not a persistent directory, meaning your files there will be deleted when the machine turns off.</mark>\n",
+    "`/tmp/` <mark style=\"background-color:lime\">is not a persistent directory, meaning your files there will be deleted when the machine turns off.</mark>\n",
     "\n",
     "<br>\n",
     "\n",
-    "If you are having storage issues, set `repo_storage_dir` to `/tmp/stable-diffusion`.\n",
+    "<mark style=\"background-color: #ff780082\">If you are having storage issues</mark>, set `repo_storage_dir` to `/tmp/stable-diffusion`.\n",
     "\n",
     "<br>\n",
     "\n",
@@ -83,6 +83,13 @@
     "%store free_tier model_storage_dir repo_storage_dir"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**Don't forget, there's a script to update this notebook to [the latest version](https://github.com/Engineer-of-Stuff/stable-diffusion-paperspace/blob/main/StableDiffusionUI_Voldemort_paperspace.ipynb) on GitHub.**"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -112,19 +119,24 @@
     "            print(f'Symlink broken, removing: {file}')\n",
     "            os.unlink(f'{path}/{file}')\n",
     "\n",
+    "def update_repo_if_not_exists(path, repo_clone_url, pre=None):\n",
+    "    if pre is not None:\n",
+    "        pre()    \n",
+    "    if not os.path.exists(path):\n",
+    "        !git clone \"{repo_clone_url}\" \"{path}\"\n",
+    "    else:\n",
+    "        print(f'{repo_clone_url.split(\"/\")[-1]} already downloaded, updating...')\n",
+    "        %cd \"{path}\"\n",
+    "        !git pull\n",
     "\n",
-    "if not os.path.exists(f'{repo_storage_dir}/stable-diffusion-webui'):\n",
+    "def init_free():\n",
-    "    if free_tier:\n",
+    "    if (free_tier and repo_storage_dir != '/notebooks'):\n",
     "        delete_broken_symlinks('/notebooks/') # remove broken symlinks since it might have been installed in a non-persistent directory\n",
     "        if not os.path.exists(repo_storage_dir):\n",
     "            !mkdir -p \"{repo_storage_dir}\"\n",
     "            !ln -s \"{repo_storage_dir}\" /notebooks/\n",
-    "    %cd \"{repo_storage_dir}\"\n",
+    "            !ls -la /notebooks/stable-diffusion\n",
-    "    !git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui\n",
+    "update_repo_if_not_exists(f'{repo_storage_dir}/stable-diffusion-webui', 'https://github.com/AUTOMATIC1111/stable-diffusion-webui', init_free)"
-    "else: # update repo if already exists\n",
-    "    print('stable-diffusion-webui already downloaded, updating...')\n",
-    "    %cd \"{repo_storage_dir}/stable-diffusion-webui\"\n",
-    "    !git pull"
    ]
   },
   {
@@ -154,6 +166,9 @@
     "# They require a few specific external git repo commits so we have to do it their way. \n",
     "import launch\n",
     "\n",
+    "# Install requirements for this notebooks\n",
+    "!pip install requests validators\n",
+    "\n",
     "# latent-diffusion is a requirement but launch.py isn't downloading it so we'll do it manually.\n",
     "if not os.path.exists(f'{repo_storage_dir}/stable-diffusion-webui/repositories/latent-diffusion'):\n",
     "    !git clone https://github.com/crowsonkb/k-diffusion.git \"{repo_storage_dir}/stable-diffusion-webui/repositories/k-diffusion\"\n",
@@ -170,12 +185,71 @@
     "else:\n",
     "    print('GFPGANv1.3.pth already downloaded')\n",
     "\n",
+    "# Download popular custom scripts. This is basically remote code execution so be careful.\n",
+    "# See https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Custom-Scripts\n",
+    "import os\n",
+    "import shutil\n",
+    "!pip install moviepy==1.0.3\n",
+    "!apt update\n",
+    "!apt install -y potrace\n",
+    "\n",
+    "def download_file_dir(url, output_dir):\n",
+    "    # output_dir must have a trailing slash\n",
+    "    local_filename = url.split('/')[-1]\n",
+    "    with requests.get(url, stream=True) as r:\n",
+    "        r.raise_for_status()\n",
+    "        with open(f'{output_dir}{local_filename}', 'wb') as f:\n",
+    "            for chunk in r.iter_content(chunk_size=8192):\n",
+    "                f.write(chunk)\n",
+    "    return local_filename\n",
+    "\n",
+    "custom_scripts = [\n",
+    "    'https://raw.githubusercontent.com/GRMrGecko/stable-diffusion-webui-automatic/advanced_matrix/scripts/advanced_prompt_matrix.py',\n",
+    "    'https://raw.githubusercontent.com/jtkelm2/stable-diffusion-webui-1/master/scripts/wildcards.py',\n",
+    "    'https://raw.githubusercontent.com/ThereforeGames/txt2img2img/main/scripts/txt2img2img.py',\n",
+    "    'https://raw.githubusercontent.com/dfaker/stable-diffusion-webui-cv2-external-masking-script/main/external_masking.py',\n",
+    "    'https://raw.githubusercontent.com/memes-forever/Stable-diffusion-webui-video/main/videos.py',\n",
+    "    'https://raw.githubusercontent.com/yownas/seed_travel/main/scripts/seed_travel.py',\n",
+    "    'https://raw.githubusercontent.com/Animator-Anon/Animator/main/animation.py',\n",
+    "    'https://gist.githubusercontent.com/dfaker/f88aa62e3a14b559fe4e5f6b345db664/raw/791dabfa0ab26399aa2635bcbc1cf6267aa4ffc2/alternate_sampler_noise_schedules.py',\n",
+    "    'https://raw.githubusercontent.com/Filarius/stable-diffusion-webui/master/scripts/vid2vid.py',\n",
+    "    'https://raw.githubusercontent.com/GeorgLegato/Txt2Vectorgraphics/main/txt2vectorgfx.py',\n",
+    "    'https://raw.githubusercontent.com/yownas/shift-attention/main/scripts/shift_attention.py',\n",
+    "    'https://raw.githubusercontent.com/DiceOwl/StableDiffusionStuff/main/loopback_superimpose.py',\n",
+    "    'https://gist.githubusercontent.com/camenduru/9ec5f8141db9902e375967e93250860f/raw/c1a03eb447548adbef1858c0e69d3567a390d2f4/run_n_times.py',\n",
+    "    'https://raw.githubusercontent.com/Engineer-of-Stuff/stable-diffusion-paperspace/main/lfs/save_steps.py',\n",
+    "    'https://raw.githubusercontent.com/Pfaeff/sd-web-ui-scripts/main/moisaic.py'\n",
+    "]\n",
+    "for item in custom_scripts:\n",
+    "    download_file_dir(item, f'{repo_storage_dir}/stable-diffusion-webui/scripts/')\n",
+    "    print(f'{item.split(\"/\")[-1]} downloaded...')\n",
+    "\n",
+    "# Download and set up txt2mask\n",
+    "update_repo_if_not_exists(f'{repo_storage_dir}/stable-diffusion-webui/txt2mask', 'https://github.com/ThereforeGames/txt2mask.git')\n",
+    "!cp -r \"{repo_storage_dir}/stable-diffusion-webui/txt2mask/repositories/clipseg\" \"{repo_storage_dir}/stable-diffusion-webui/repositories\"\n",
+    "txt2mask_source = f'{repo_storage_dir}/stable-diffusion-webui/txt2mask/scripts/'\n",
+    "for file in os.listdir(txt2mask_source):\n",
+    "    src = os.path.join(txt2mask_source, file)\n",
+    "    if os.path.isfile(src):\n",
+    "        shutil.copy(src, f'{repo_storage_dir}/stable-diffusion-webui/scripts/')\n",
+    "\n",
+    "!mkdir -p \"{repo_storage_dir}/stable-diffusion-webui/scripts/wildcards\"\n",
+    "wildcard_lists = [\n",
+    "    'https://raw.githubusercontent.com/jtkelm2/stable-diffusion-webui-1/master/scripts/wildcards/adjective.txt',\n",
+    "    'https://raw.githubusercontent.com/jtkelm2/stable-diffusion-webui-1/master/scripts/wildcards/artist.txt',\n",
+    "    'https://raw.githubusercontent.com/jtkelm2/stable-diffusion-webui-1/master/scripts/wildcards/genre.txt',\n",
+    "    'https://raw.githubusercontent.com/jtkelm2/stable-diffusion-webui-1/master/scripts/wildcards/site.txt',\n",
+    "    'https://raw.githubusercontent.com/jtkelm2/stable-diffusion-webui-1/master/scripts/wildcards/style.txt'\n",
+    "]\n",
+    "for item in wildcard_lists:\n",
+    "    download_file_dir(item, f'{repo_storage_dir}/stable-diffusion-webui/scripts/wildcards/')\n",
+    "\n",
     "# Make sure your models storage directory exists\n",
     "!mkdir -p \"{model_storage_dir}\"\n",
     "\n",
     "# Link the output folder to /notebooks/outputs\n",
     "!mkdir -p \"{repo_storage_dir}/stable-diffusion-webui/outputs\"\n",
-    "!ln -s \"{repo_storage_dir}/stable-diffusion-webui/outputs\" /notebooks/outputs"
+    "!ln -s \"{repo_storage_dir}/stable-diffusion-webui/outputs\" /notebooks/"
    ]
   },
   {
@@ -450,17 +524,20 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "source": [
     "Run this block to launch the WebUI. You will get a link to nnn.gradio.app, that's your WebUI. Follow it.\n",
     "\n",
-    "If you have a lot of VRAM and aren't generating large images you can add the flag `--disable-opt-split-attention` to disable VRAM optimizations for a speed boost.\n",
+    "- See [webui.py](https://github.com/AUTOMATIC1111/stable-diffusion-webui/blob/master/modules/shared.py#L22) to view the code for the launch args. There's a lot of good info in here about exactly what the args do. If you aren't a programmer, [here's the wiki](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Run-with-Custom-Parameters).\n",
-    "\n",
+    "- If you have a lot of VRAM and desire high generation speeds, add `--disable-opt-split-attention` to disable VRAM optimizations.\n",
-    "**Troubleshooting**\n",
+    "- If you have a decent about of VRAM and aren't generating large images you can remove `--medvram` for a speed boost.\n",
     "\n",
+    "#### Troubleshooting\n",
     "- If you have any issues, try restarting the kernel.\n",
-    "- `EOFError: Ran out of input` probably means you ran out of storage space and the model `.ckpt` file wasn't downloaded completely. Try cleaning up your files or you can reset your storage with the _Reset Storage_ block in the Tools section below.\n",
+    "- `EOFError: Ran out of input` probably means you ran out of storage space and the model `.ckpt` file wasn't downloaded completely. Try cleaning up your files. There are some helpful scripts in the Tools section below.\n",
-    "- If you're having issues with your results not loading, try running an ngrok proxy. More details in the Tools section below."
+    "- If you're having issues with your results not loading, that's a known bug. I used to suggest people use ngrok but apparently accounts were getting locked because tunneling proxies are against the terms of service even though the Paperspace terms of service doesn't mention anything like that. [More details in the Tools section below.](https://github.com/anderspitman/awesome-tunneling)"
    ]
   },
   {
@@ -474,7 +551,9 @@
    "source": [
     "%store -r free_tier model_storage_dir repo_storage_dir\n",
     "%cd \"{repo_storage_dir}/stable-diffusion-webui\"\n",
-    "!python webui.py --gradio-debug --share # --gradio-auth me:password1234"
+    "\n",
+    "# Launch args go below:\n",
+    "!python webui.py --gradio-debug --share --medvram # --gradio-auth me:password1234"
    ]
   },
   {
@@ -585,7 +664,9 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "source": [
     "### Reset Repository\n",
     "\n",
@@ -605,6 +686,24 @@
     "!git reset --hard <commit>"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Delete .ipynb_checkpoints\n",
+    "\n",
+    "Jupyter stores temporary files in folders named `.ipynb_checkpoints`. It gets a little excessive sometimes so if you're running low on storage space or getting weird errors about a directory named `.ipynb_checkpoints`, run this block."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!find . -type d -name .ipynb_checkpoints -delete"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -631,6 +730,66 @@
     "# !rm -rf {model_storage_dir}\n",
     "# !rm -rf {repo_storage_dir}"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "### Download and Install Custom Scripts\n",
+    "\n",
+    "This block will prompt you to enter the URL of the script you want to download.\n",
+    "\n",
+    "See [https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Custom-Scripts](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Custom-Scripts)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%store -r free_tier model_storage_dir repo_storage_dir\n",
+    "import validators\n",
+    "import requests\n",
+    "import sys\n",
+    "\n",
+    "def download_file(url, directory):\n",
+    "    print('Checking if text...', end='\\r')\n",
+    "    content_type = requests.head(url).headers['content-type'].split(';')[0]\n",
+    "    if (content_type != 'text/plain'):\n",
+    "        while True:\n",
+    "            print(f'This is not a text file, it\\'s {content_type}! Continue? y/n', end='\\r')\n",
+    "            ask = input().lower()\n",
+    "            if (ask == 'y'):\n",
+    "                break\n",
+    "            if (ask == 'n'):\n",
+    "                sys.exit(0)\n",
+    "    print('\\033[F' + '\\033[K' + 'Downloading...', end='\\r')\n",
+    "    local_filename = f'{directory}{url.split(\"/\")[-1]}'\n",
+    "    with requests.get(url, stream=True) as r:\n",
+    "        try:\n",
+    "           r.raise_for_status()\n",
+    "        except requests.exceptions.HTTPError as err:\n",
+    "            print(err)\n",
+    "            raise Exception(err)\n",
+    "        with open(local_filename, 'wb') as f:\n",
+    "            for chunk in r.iter_content(chunk_size=8192): \n",
+    "                f.write(chunk)\n",
+    "    print('Downloading... done!')\n",
+    "    return local_filename\n",
+    "\n",
+    "print('URL to script then press ENTER: ')\n",
+    "i = input().strip()\n",
+    "valid_url = validators.url(i)\n",
+    "if not valid_url:\n",
+    "    print('Not a valid URL')\n",
+    "else:\n",
+    "    download_file(i, f'{repo_storage_dir}/stable-diffusion-webui/scripts/') # this will overwrite any existing files\n",
+    "    print('\\nScripts folder content:')\n",
+    "    !ls \"{repo_storage_dir}/stable-diffusion-webui/scripts\""
+   ]
   }
  ],
  "metadata": {

docs/archives/update-guides.sh

@@ -20,3 +20,5 @@ wget "https://rentry.org/sdupscale/pdf" -O "Larger resolutions with Stable Diffu
 wget "https://rentry.org/male-to-anime-girl/pdf" -O "How to Turn Yourself Into a Cute Anime Girl.pdf"
 wget "https://rentry.org/sdmodels/pdf" -O "Stable Diffusion Models.pdf"
 wkhtmltopdf "https://wiki.installgentoo.com/wiki/Stable_Diffusion" "wiki.installgentoo.com Stable Diffusion Guide.pdf"
+wkhtmltopdf "https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Run-with-Custom-Parameters" "Run with Custom Parameters - stable-diffusion-webui wiki.pdf"
+# wkhtmltopdf "https://proximacentaurib.notion.site/A-guide-to-getting-started-with-the-paperspace-port-of-AUTOMATIC1111-s-web-UI-for-ppl-who-get-nervou-b83c2213f17e452e8b0e37ba64fe9758" "Getting started with the paperspace port of AUTOMATIC1111s web UI for ppl who get nervous.pdf"