FIX FREE TIER STORAGE ISSUES!!!!!!!!!
Former-commit-id: 40b52bf6d1871c71233926f170a43fb98cefd778 [formerly 529c955783]
Former-commit-id: b52c22a369be432357324cc25cd51498f845c424
This commit is contained in:
frostydad
parent
b4cd485bc1
commit
4390810d29
|
|
@ -50,7 +50,7 @@
|
|||
"\n",
|
||||
"We're going to store models in `/storage/models` and create a symlink.\n",
|
||||
"\n",
|
||||
"_You must run the block below or else the variable won't be accessable in the notebook._"
|
||||
"<mark>You must uncomment the correct section and run the block below or else the notebook won't work!</mark>"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -59,12 +59,22 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model_storage_dir = '/storage/models' # no trailing slash\n",
|
||||
"# Free tier\n",
|
||||
"# free_tier = True # Enables the creation of symlinks back to /notebooks/\n",
|
||||
"# model_storage_dir = '/tmp/stable-diffusion/models' # Where the models will be downloaded to\n",
|
||||
"# repo_storage_dir = '/tmp/stable-diffusion' # Where the repository will be downloaded to\n",
|
||||
"\n",
|
||||
"# Paid Tier\n",
|
||||
"# free_tier = False\n",
|
||||
"# model_storage_dir = '/storage/models'\n",
|
||||
"# repo_storage_dir = '/notebooks'\n",
|
||||
"\n",
|
||||
"# Don't put a trailing slash on directory paths.\n",
|
||||
"# To reset your storage directory, rerun this cell.\n",
|
||||
"\n",
|
||||
"# ===============================================================\n",
|
||||
"# Save the variable to Jupiter's temp storage to access it when the kernel restarts.\n",
|
||||
"# To reset your storage directory, rerun this cell.\n",
|
||||
"%store model_storage_dir"
|
||||
"# Save variables to Jupiter's temp storage so we can access it even if the kernel restarts.\n",
|
||||
"%store free_tier model_storage_dir repo_storage_dir"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -85,14 +95,28 @@
|
|||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os.path\n",
|
||||
"%store -r model_storage_dir\n",
|
||||
"if not os.path.exists('/notebooks/stable-diffusion-webui'):\n",
|
||||
" %cd /notebooks/\n",
|
||||
"import os\n",
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"%cd /notebooks/\n",
|
||||
"\n",
|
||||
"def delete_broken_symlinks(path):\n",
|
||||
" # make sure to pass this function a path without a trailing slash\n",
|
||||
" for file in os.listdir(path):\n",
|
||||
" if os.path.islink(f'{path}/{file}') and not os.path.exists(os.readlink(f'{path}/{file}')):\n",
|
||||
" print(f'Symlink broken, removing: {file}')\n",
|
||||
" os.unlink(f'{path}/{file}')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if not os.path.exists(f'{repo_storage_dir}/stable-diffusion-webui'):\n",
|
||||
" if free_tier:\n",
|
||||
" delete_broken_symlinks('/notebooks/') # remove broken symlinks since it might have been installed in a non-persistent directory\n",
|
||||
" !mkdir -p \"{repo_storage_dir}\"\n",
|
||||
" !ln -s \"{repo_storage_dir}\" /notebooks/\n",
|
||||
" %cd \"{repo_storage_dir}\"\n",
|
||||
" !git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui\n",
|
||||
"else: # update repo if already exists\n",
|
||||
" print('stable-diffusion-webui already downloaded, updating...')\n",
|
||||
" %cd /notebooks/stable-diffusion-webui\n",
|
||||
" %cd \"{repo_storage_dir}/stable-diffusion-webui\"\n",
|
||||
" !git pull"
|
||||
]
|
||||
},
|
||||
|
|
@ -116,25 +140,35 @@
|
|||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%cd /notebooks/stable-diffusion-webui\n",
|
||||
"%store -r model_storage_dir\n",
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"%cd \"{repo_storage_dir}/stable-diffusion-webui\"\n",
|
||||
"\n",
|
||||
"# Import launch.py which will automatically run the install script but not launch the WebUI.\n",
|
||||
"# They require a few specific external git repo commits so we have to do it their way. \n",
|
||||
"import launch\n",
|
||||
"\n",
|
||||
"# latent-diffusion is a requirement but launch.py isn't downloading it so we'll do it manually.\n",
|
||||
"if not os.path.exists(f'{repo_storage_dir}/stable-diffusion-webui/repositories/latent-diffusion'):\n",
|
||||
" !git clone https://github.com/crowsonkb/k-diffusion.git \"{repo_storage_dir}/stable-diffusion-webui/repositories/k-diffusion\"\n",
|
||||
" !git clone https://github.com/Hafiidz/latent-diffusion.git \"{repo_storage_dir}/stable-diffusion-webui/repositories/latent-diffusion\"\n",
|
||||
" # I don't think it's necessary to do this:\n",
|
||||
" # %mkdir \"{repo_storage_dir}/stable-diffusion-webui/repositories/latent-diffusion/experiments/\"\n",
|
||||
" # %mkdir \"{repo_storage_dir}/stable-diffusion-webui/repositories/latent-diffusion/experiments/pretrained_models\"\n",
|
||||
" # !wget https://heibox.uni-heidelberg.de/f/31a76b13ea27482981b4/?dl=1 -O \"{repo_storage_dir}/stable-diffusion-webui/repositories/latent-diffusion/experiments/pretrained_models/project.yaml\"\n",
|
||||
" # !wget https://heibox.uni-heidelberg.de/f/578df07c8fc04ffbadf3/?dl=1 -O \"{repo_storage_dir}/stable-diffusion-webui/repositories/latent-diffusion/experiments/pretrained_models/model.ckpt\"\n",
|
||||
"\n",
|
||||
"# Download the GFPGAN face restorer.\n",
|
||||
"if not os.path.exists('/notebooks/stable-diffusion-webui/GFPGANv1.3.pth'):\n",
|
||||
" !wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -O /notebooks/stable-diffusion-webui/GFPGANv1.3.pth\n",
|
||||
"if not os.path.exists(f'{repo_storage_dir}/stable-diffusion-webui/GFPGANv1.3.pth'):\n",
|
||||
" !wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -O \"{repo_storage_dir}/stable-diffusion-webui/GFPGANv1.3.pth\"\n",
|
||||
"else:\n",
|
||||
" print('GFPGANv1.3.pth already downloaded')\n",
|
||||
" \n",
|
||||
"# Make sure your models storage directory exists\n",
|
||||
"!mkdir -p $model_storage_dir\n",
|
||||
"!mkdir -p \"{model_storage_dir}\"\n",
|
||||
"\n",
|
||||
"# Link the output folder to /notebooks/outputs\n",
|
||||
"!mkdir -p /notebooks/stable-diffusion-webui/outputs\n",
|
||||
"!ln -s /notebooks/stable-diffusion-webui/outputs /notebooks/outputs"
|
||||
"!mkdir -p \"{repo_storage_dir}/stable-diffusion-webui/outputs\"\n",
|
||||
"!ln -s \"{repo_storage_dir}/stable-diffusion-webui/outputs\" /notebooks/outputs"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -160,10 +194,11 @@
|
|||
"\n",
|
||||
"**Filesize and Storage Disclaimer**\n",
|
||||
"\n",
|
||||
"Paperspace free tier has only 5GB of storage space. There's really not a way to get around this limitation, but here's a few suggestions.\n",
|
||||
"1. Add a payment method to your account. Storage overages are billed at \\$0.29/GB and billing occurs monthly and runs at midnight on the first of each month. With a payment method on file, Paperspace will let you use more storage and if you time it right you shouldn't actually be charged for it.\n",
|
||||
"2. Upgrade to a Pro account. They'll give you 15GB and you'll get longer runtimes and more powerful free GPUs.\n",
|
||||
"3. Use my referral code `KQLRH37` You'll get \\$10 credit that you should be able to put towards the storage overage charges. Redeem the code at the bottom of the Billing page.\n",
|
||||
"Paperspace free tier has only 5GB of storage space. If you're having storage issues, here's a few suggestions.\n",
|
||||
"1. Download everything to `/tmp/`\n",
|
||||
"2. Add a payment method to your account. Storage overages are billed at \\$0.29/GB and billing occurs monthly and runs at midnight on the first of each month. With a payment method on file, Paperspace will let you use more storage and if you time it right you shouldn't actually be charged for it.\n",
|
||||
"3. Upgrade to a Pro account. They'll give you 15GB and you'll get longer runtimes and more powerful free GPUs.\n",
|
||||
"4. Use my referral code `KQLRH37` You'll get \\$10 credit that you should be able to put towards the storage overage charges. Redeem the code at the bottom of the Billing page.\n",
|
||||
"\n",
|
||||
"If you're on free tier, only download one model.\n",
|
||||
"\n",
|
||||
|
|
@ -191,10 +226,12 @@
|
|||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r model_storage_dir\n",
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"!apt update\n",
|
||||
"!apt install -y aria2\n",
|
||||
"%cd $model_storage_dir\n",
|
||||
|
|
@ -224,11 +261,11 @@
|
|||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"user_token = \"<enter your user token here>\"\n",
|
||||
"user_token = \"<put your user token here>\"\n",
|
||||
"\n",
|
||||
"# ===============================================================================================\n",
|
||||
"%store -r model_storage_dir\n",
|
||||
"!wget --header=\"'Authorization: Bearer {user_token}'\" https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt -O $model_storage_dir/sd-v1-4.ckpt"
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"!wget --header=\"Authorization: Bearer {user_token}\" https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/main/sd-v1-4.ckpt -O \"{model_storage_dir}/sd-v1-4.ckpt\""
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -248,9 +285,10 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"!apt update\n",
|
||||
"!apt install -y aria2\n",
|
||||
"%cd $model_storage_dir\n",
|
||||
"%cd \"{model_storage_dir}\"\n",
|
||||
"!aria2c --seed-time=0 --max-overall-upload-limit=1K \"magnet:?xt=urn:btih:153590FD7E93EE11D8DB951451056C362E3A9150&dn=wd-v1-2-full-ema-pruned.ckpt&tr=udp://tracker.opentrackr.org:1337/announce\""
|
||||
]
|
||||
},
|
||||
|
|
@ -267,9 +305,9 @@
|
|||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Anime finetune.\n",
|
||||
"Another anime finetune. Designed to nudge SD to an anime/manga style. Seems to be more \"stylized\" and \"artistic\" than Waifu Diffusion, if that makes any sense.\n",
|
||||
"\n",
|
||||
"The 60000 steps version is the original, the 115000 steps is the 60000 with additional training. Use the 60000 step version if the style nudging is too much.\n",
|
||||
"The 60,000 steps version is the original, the 115,000 and 95,000 versions is the 60,000 with additional training. Use the 60,000 step version if the style nudging is too much.\n",
|
||||
"\n",
|
||||
"[See the comparison here.](https://cyberes.github.io/stable-diffusion-models/#model-comparison)"
|
||||
]
|
||||
|
|
@ -287,8 +325,8 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r model_storage_dir\n",
|
||||
"!wget https://huggingface.co/naclbit/trinart_stable_diffusion_v2/resolve/main/trinart2_step60000.ckpt -O $model_storage_dir/trinart2_step60000.ckpt"
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"!wget https://huggingface.co/naclbit/trinart_stable_diffusion_v2/resolve/main/trinart2_step60000.ckpt -O \"{model_storage_dir}/trinart2_step60000.ckpt\""
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -304,8 +342,8 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r model_storage_dir\n",
|
||||
"!wget https://huggingface.co/naclbit/trinart_stable_diffusion_v2/resolve/main/trinart2_step95000.ckpt -O $model_storage_dir/trinart2_step95000.ckpt"
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"!wget https://huggingface.co/naclbit/trinart_stable_diffusion_v2/resolve/main/trinart2_step95000.ckpt -O \"{model_storage_dir}/trinart2_step95000.ckpt\""
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -321,8 +359,8 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r model_storage_dir\n",
|
||||
"!wget https://huggingface.co/naclbit/trinart_stable_diffusion_v2/resolve/main/trinart2_step115000.ckpt -O $model_storage_dir/trinart2_step115000.ckpt"
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"!wget https://huggingface.co/naclbit/trinart_stable_diffusion_v2/resolve/main/trinart2_step115000.ckpt -O \"{model_storage_dir}/trinart2_step115000.ckpt\""
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -340,11 +378,11 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r model_storage_dir\n",
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"\n",
|
||||
"# Get some storage back\n",
|
||||
"!pip cache purge\n",
|
||||
"!rm $model_storage_dir/*.aria2\n",
|
||||
"!cd \"{model_storage_dir}\" && rm *.aria2\n",
|
||||
"!apt remove --purge -y aria2 p7zip-full\n",
|
||||
"!apt autoremove --purge -y\n",
|
||||
"!apt clean\n",
|
||||
|
|
@ -369,24 +407,29 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r model_storage_dir\n",
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Check for broken symlinks and remove them\n",
|
||||
"for file in os.listdir('/notebooks/stable-diffusion-webui/models/Stable-diffusion/'):\n",
|
||||
" path = f'/notebooks/stable-diffusion-webui/models/Stable-diffusion/{file}'\n",
|
||||
"deleted = False\n",
|
||||
"for file in os.listdir(f'{repo_storage_dir}/stable-diffusion-webui/models/Stable-diffusion/'):\n",
|
||||
" path = f'{repo_storage_dir}/stable-diffusion-webui/models/Stable-diffusion/{file}'\n",
|
||||
" if os.path.islink(path) and not os.path.exists(os.readlink(path)):\n",
|
||||
" print(f'Symlink broken, removing: {file}')\n",
|
||||
" os.unlink(path)\n",
|
||||
" deleted = True\n",
|
||||
"if deleted:\n",
|
||||
" print('')\n",
|
||||
"\n",
|
||||
"# Make symlinks for new files\n",
|
||||
"for file in os.listdir(model_storage_dir):\n",
|
||||
" if file.endswith(\"ckpt\"):\n",
|
||||
" path = f'/notebooks/stable-diffusion-webui/models/Stable-diffusion/{file}'\n",
|
||||
" path = f'{repo_storage_dir}/stable-diffusion-webui/models/Stable-diffusion/{file}'\n",
|
||||
" if not os.path.exists(path):\n",
|
||||
" print(f'New model: {file}')\n",
|
||||
" !ln -s $model_storage_dir/$file /notebooks/stable-diffusion-webui/models/Stable-diffusion/$file\n",
|
||||
" !ls -la --block-size=GB /notebooks/stable-diffusion-webui/models/Stable-diffusion/$file"
|
||||
" !ln -s \"{model_storage_dir}/$file\" \"{repo_storage_dir}/stable-diffusion-webui/models/Stable-diffusion/$file\"\n",
|
||||
" !ls -la --block-size=GB \"{repo_storage_dir}/stable-diffusion-webui/models/Stable-diffusion/$file\"\n",
|
||||
" print('')"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -422,9 +465,9 @@
|
|||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r model_storage_dir\n",
|
||||
"%cd /notebooks/stable-diffusion-webui\n",
|
||||
"!python /notebooks/stable-diffusion-webui/webui.py --share # --disable-opt-split-attention"
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"%cd \"{repo_storage_dir}/stable-diffusion-webui\"\n",
|
||||
"!python webui.py --share # --gradio-auth me:password1234"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -453,15 +496,18 @@
|
|||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"!apt update\n",
|
||||
"!apt install -y p7zip-full\n",
|
||||
"from datetime import datetime\n",
|
||||
"datetime_str = datetime.now().strftime('%m-%d-%Y_%H:%M:%S')\n",
|
||||
"%cd /notebooks/\n",
|
||||
"!mkdir $datetime_str\n",
|
||||
"!mv /notebooks/stable-diffusion-webui/log $datetime_str\n",
|
||||
"!mv /notebooks/stable-diffusion-webui/outputs/* $datetime_str\n",
|
||||
"!7z a -t7z -m0=lzma2 -mx=9 -mfb=64 -md=32m -ms=on {datetime_str}.7z {datetime_str}/"
|
||||
"!mkdir \"{datetime_str}\"\n",
|
||||
"!mv \"{repo_storage_dir}/stable-diffusion-webui/log\" \"/notebooks/{datetime_str}\"\n",
|
||||
"!cd \"{repo_storage_dir}/stable-diffusion-webui/outputs/\" && mv * \"/notebooks/{datetime_str}\"\n",
|
||||
"!TEMP=\"/notebooks/{datetime_str}\" # find command has issues with ipynb variables??\n",
|
||||
"# !find $TEMP -name .ipynb_checkpoints -exec rm -rf \"{}\" +\n",
|
||||
"!7z a -t7z -m0=lzma2 -mx=9 -mfb=64 -md=32m -ms=on \"{datetime_str}.7z\" \"/notebooks/{datetime_str}/\""
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -481,8 +527,8 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!rm -rf /notebooks/$datetime_str/\n",
|
||||
"!echo Deleted /notebooks/$datetime_str/"
|
||||
"!rm -rf \"/notebooks/{datetime_str}/\"\n",
|
||||
"!echo Deleted /notebooks/{datetime_str}/"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -536,7 +582,9 @@
|
|||
"source": [
|
||||
"### Reset Repository\n",
|
||||
"\n",
|
||||
"Sometimes AUTOMATIC1111 breaks something. Go to https://github.com/AUTOMATIC1111/stable-diffusion-webui/commits/master and choose a commit to revert to."
|
||||
"Sometimes AUTOMATIC1111 breaks something. Go to https://github.com/AUTOMATIC1111/stable-diffusion-webui/commits/master and choose a commit to revert to.\n",
|
||||
"\n",
|
||||
"**This shouldn't delete your outputs or any changes you've made to files, but I'd back up anything important just to be safe.**"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -545,16 +593,20 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"%cd \"{repo_storage_dir}/stable-diffusion-webui\"\n",
|
||||
"!git reset --hard <commit>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"### Delete all files and reset storage\n",
|
||||
"### Reset storage\n",
|
||||
"\n",
|
||||
"This will delete ALL your files in `/notebooks/` and `/storage/`. Use if you're having issues with zero storage space and you don't want to delete your notebook."
|
||||
"This will delete ALL your files in `/notebooks/`, `/storage/`, `model_storage_dir`, and `repo_storage_dir`. Use if you're having issues with zero storage space and you don't want to delete your notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -564,10 +616,13 @@
|
|||
"outputs": [],
|
||||
"source": [
|
||||
"# Uncomment the lines below to run this block. You can highlight the lines and do ctrl + /\n",
|
||||
"# %store -r free_tier model_storage_dir repo_storage_dir\n",
|
||||
"# !rm -rf /storage/*\n",
|
||||
"# !mv /notebooks/StableDiffusionUI_Voldemort_paperspace.ipynb / # move the notebook out of the directory before we nuke it\n",
|
||||
"# !mv /notebooks/*.ipynb / # move the notebook out of the directory before we nuke it\n",
|
||||
"# !rm -rf /notebooks/*\n",
|
||||
"# !mv /StableDiffusionUI_Voldemort_paperspace.ipynb /notebooks/ # move it back"
|
||||
"# !mv /*.ipynb /notebooks/ # move it back\n",
|
||||
"# !rm -rf {model_storage_dir}\n",
|
||||
"# !rm -rf {repo_storage_dir}"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue