diff --git a/Train-Runpod.ipynb b/Train-Runpod.ipynb new file mode 100644 index 0000000..52ff3c0 --- /dev/null +++ b/Train-Runpod.ipynb @@ -0,0 +1,326 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "676114ae", + "metadata": {}, + "source": [ + "## Every Dream trainer\n", + "\n", + "You will need your data prepared first before starting! Don't waste rental fees if you're not ready to upload your files. Your files should be captioned before you start with either the caption as the filename or in text files for each image alongside the image files. See main README.md for more details. Tools are available to automatically caption your files.\n", + "\n", + "[Instructions](https://github.com/victorchall/EveryDream-trainer/blob/main/README.md)\n", + "\n", + "If you can sign up for Runpod here (shameless referral link): [Runpod](https://runpod.io?ref=oko38cd0)\n", + "\n", + "If you are confused by the wall of text, join the discord here: [EveryDream Discord](https://discord.gg/uheqxU6sXN)\n", + "\n", + "Make sure you have at least 40GB of Runpod **Volume** storage at a minimum so you don't waste training just 1 ckpt that is overtrained and have to start over. Penny pinching on storage is ultimately a waste of your time and money! This is setup to give you more than one ckpt so you don't overtrain.\n", + "\n", + "### Starting model\n", + "Make sure you have your hugging face token ready to download the 1.5 mode. You can get one here: https://huggingface.co/settings/tokens\n", + "If you don't have a User Access Token, create one. Or you can upload a starting checkpoint instead of using the HF download and skip that step, but you'll need to modify the starting model name when you start training (more info below)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb6d14b7-3c37-4ec4-8559-16b4e9b8dd18", + "metadata": {}, + "outputs": [], + "source": [ + "!git clone https://github.com/victorchall/everydream-trainer\n", + "%cd everydream-trainer" + ] + }, + { + "cell_type": "markdown", + "id": "589bfca0", + "metadata": {}, + "source": [ + "## Install dependencies\n", + "You can ignore \"warnings.\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ab559338", + "metadata": {}, + "outputs": [], + "source": [ + "# BUILD ENV\n", + "!pip install -q omegaconf\n", + "!pip install -q einops\n", + "!pip install -q pytorch-lightning==1.6.5\n", + "!pip install -q test-tube\n", + "!pip install -q transformers==4.19.2\n", + "!pip install -q kornia\n", + "!pip install -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers\n", + "!pip install -e git+https://github.com/openai/CLIP.git@main#egg=clip\n", + "!pip install -q setuptools==59.5.0\n", + "!pip install -q pillow==9.0.1\n", + "!pip install -q torchmetrics==0.6.0\n", + "!pip install -e .\n", + "#!pip install -qq diffusers[\"training\"]==0.3.0 transformers ftfy\n", + "!pip install -qq ipywidgets==8.0.2\n", + "!pip install huggingface_hub\n", + "#!pip install ipywidgets==7.7.1\n", + "import ipywidgets as widgets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55716da3-7229-45e0-b8c1-2b25466fd126", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install omegaconf\n", + "!pip install albumentations==1.1.0\n", + "!pip install transformers==4.19.2\n", + "!pip install torchvision==0.13.1\n", + "!pip install pudb==2019.2\n", + "!pip install imageio==2.14.1\n", + "!pip install imageio-ffmpeg==0.4.7\n", + "!pip install test-tube>=0.7.5\n", + "!pip install einops==0.4.1\n", + "!pip install pillow==9.0.1\n", + "!pip install torch-fidelity==0.3.0\n", + "!pip install torchmetrics==0.6.0\n", + "!pip install kornia==0.6\n", + "!pip install huggingface_hub\n", + "!pip install -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers\n", + "!pip install -e git+https://github.com/openai/CLIP.git@main#egg=clip\n", + "!pip install -e ." + ] + }, + { + "cell_type": "markdown", + "id": "c230d91a", + "metadata": {}, + "source": [ + "## Now that dependencies are installed, ready to move on!" + ] + }, + { + "cell_type": "markdown", + "id": "17affc47", + "metadata": {}, + "source": [ + "## Log into huggingface\n", + "Run the cell below and paste your token into the prompt. You can get your token from your huggingface account page.\n", + "\n", + "The token will not show on the screen, just press enter after you paste it.\n", + "\n", + "Then run the following cell to download the base checkpoint (may take a minute)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "02c8583e", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "39a1acc3a2914d9797fc2f8ff11a9a69", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(HTML(value='
500 images to ensure even distribution, but you can set this if you really want... check_val_every_n_epoch: 1 gpus: 0, diff --git a/demo/runpodconnect.png b/demo/runpodconnect.png new file mode 100644 index 0000000..20981d3 Binary files /dev/null and b/demo/runpodconnect.png differ diff --git a/demo/runpodinstances.png b/demo/runpodinstances.png new file mode 100644 index 0000000..3ceb8ba Binary files /dev/null and b/demo/runpodinstances.png differ diff --git a/demo/runpodopenurl.png b/demo/runpodopenurl.png new file mode 100644 index 0000000..069f27b Binary files /dev/null and b/demo/runpodopenurl.png differ diff --git a/demo/runpodsetup.png b/demo/runpodsetup.png new file mode 100644 index 0000000..d963ac2 Binary files /dev/null and b/demo/runpodsetup.png differ diff --git a/demo/runpodstop.png b/demo/runpodstop.png new file mode 100644 index 0000000..e128d30 Binary files /dev/null and b/demo/runpodstop.png differ diff --git a/demo/runpodupload.png b/demo/runpodupload.png new file mode 100644 index 0000000..68bea33 Binary files /dev/null and b/demo/runpodupload.png differ diff --git a/doc/RUNPOD.MD b/doc/RUNPOD.MD new file mode 100644 index 0000000..e56c44d --- /dev/null +++ b/doc/RUNPOD.MD @@ -0,0 +1,50 @@ +# Runpod + +You will need your data prepared first before starting! Don't waste rental fees if you're not ready to upload your files. Your files should be captioned before you start with either the caption as the filename or in text files for each image alongside the image files. See main README.md for more details. Tools are available to automatically caption your files on a 4GB GPU or Colab notebook. + +[Main readme](https://github.com/victorchall/EveryDream-trainer/blob/main/README.md) + +You can sign up for Runpod here (*shameless referral link*): [Runpod](https://runpod.io?ref=oko38cd0) + +If you are confused by the wall of text, join the discord here: [EveryDream Discord](https://discord.gg/uheqxU6sXN) + +Make sure you have at least 50GB of Runpod **Volume** storage at a minimum so you don't waste training just 1 ckpt that is overtrained and have to start over. Penny pinching on storage is ultimately a waste of your time and money! This is setup to give you more than one ckpt so you don't overtrain. + +## Getting started + +1. Pick a 24GB GPU instance (community or secure). You can use 48GB but it is unnecessary. Make sure to use "ON DEMAND" not "SPOT" or your instance may be closed suddently and you will lose your training. Once you click, go "My Pods" + +![r](../demo/runpodinstances.png) + +1. Start a Pytorch instance in Runpod. Make sure to get plenty of volume space! + +![r](../demo/runpodsetup.png) + +2. Launch the Juypyter notebook. +![r](../demo/runpodconnect.png) + +3. Click file, open from URL, and paste in this URL: https://raw.githubusercontent.com/victorchall/EveryDream-trainer/main/Train-Runpod.ipynb + +![r](../demo/runpodopenurl.png) + +4. Rest of the instructions are in the notebook, but you'll upload your files here once ready: + +![r](../demo/runpodupload.png) + +5. Make sure to go back to Runpod.io when you are done (don't forget to download your PRUNED.CKPT files) and STOP your instance and also click the **trash can button** to remove the volume storage. You will be charged for the storage if you don't delete it. + +![r](../demo/runpodstop.png) + +# Advanced mode + +You can train MUCH larger models with more data, potentially unlimited numbers of characters and styles into one model. You will follow the same steps, but the expectation is you are using a much larger data set, say, many hundreds or thousands of images, and are willing to train for many hours. There are many things to consider on each project and different projects have different requirements so it is hard to generalize. + +You can read up on a model trained with 7+ characters and a variety of cityscapes using 1600+ new training images and 1600+ preservation images here: [FF7R Mega Model on Huggingface](https://huggingface.co/panopstor/ff7r-stable-diffusion) + +You will want even more volume space. 100GB is advised so you can get MANY ckpts (many epochs) you can test along the way, download them, stop your instance, test, and resume from them again if needed. This will save you a lot of heartache with undertrained or overtrained models, especially if you are training for 6-10+ hours. + +You will change the yaml in the training step to v1-finetune-everydream.yaml, and also want to consider tweaking the values and a long, careful read of the main README, and training more basic models first. + + !python main.py --base configs/stable-diffusion/v1-finetune_everydream.yaml -t --actual_resume "v1-5-pruned.ckpt" -n test --data_root input + +This file is configured better for very large training sets. Repeats is reduced. \ No newline at end of file diff --git a/input/.gitkeep b/input/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ldm/data/image_train_item.py b/ldm/data/image_train_item.py index 194bb19..954a309 100644 --- a/ldm/data/image_train_item.py +++ b/ldm/data/image_train_item.py @@ -2,6 +2,8 @@ import PIL import numpy as np from torchvision import transforms +import random +import math class ImageTrainItem(): # [image, identifier, target_aspect, closest_aspect_wh[w,h], pathname] def __init__(self, image: PIL.Image, caption: str, target_wh: list, pathname: str, flip_p=0.0): @@ -9,6 +11,7 @@ class ImageTrainItem(): # [image, identifier, target_aspect, closest_aspect_wh[w self.target_wh = target_wh self.pathname = pathname self.flip = transforms.RandomHorizontalFlip(p=flip_p) + self.cropped_img = None if image is None: self.image = PIL.Image.new(mode='RGB',size=(1,1)) @@ -19,11 +22,45 @@ class ImageTrainItem(): # [image, identifier, target_aspect, closest_aspect_wh[w if type(self.image) is not np.ndarray: self.image = PIL.Image.open(self.pathname).convert('RGB') - self.image = self.image.resize((self.target_wh), PIL.Image.BICUBIC) + cropped_img = self.__autocrop(self.image) + + self.image = cropped_img.resize((512,512), PIL.Image.BICUBIC) self.image = self.flip(self.image) + self.image = np.array(self.image).astype(np.uint8) self.image = (self.image / 127.5 - 1.0).astype(np.float32) - return self \ No newline at end of file + return self + + @staticmethod + def __autocrop(image: PIL.Image, q=.404): + x, y = image.size + + if x != y: + if (x>y): + rand_x = x-y + rand_y = 0 + sigma = max(rand_x*q,1) + else: + rand_x = 0 + rand_y = y-x + sigma = max(rand_y*q,1) + + if (x>y): + x_crop_gauss = abs(random.gauss(0, sigma)) + x_crop = min(x_crop_gauss,(x-y)/2) + x_crop = math.trunc(x_crop) + y_crop = 0 + else: + y_crop_gauss = abs(random.gauss(0, sigma)) + x_crop = 0 + y_crop = min(y_crop_gauss,(y-x)/2) + y_crop = math.trunc(y_crop) + + min_xy = min(x, y) + image = image.crop((x_crop, y_crop, x_crop + min_xy, y_crop + min_xy)) + #print(f"crop: {x_crop} {y_crop}, {x} {y} => {image.size}") + + return image \ No newline at end of file