From 234e90cca77b94d1102790773fa7e49b347114f1 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Thu, 8 Sep 2022 17:27:36 +0200 Subject: [PATCH] [Docs] Using diffusers (#428) * [Docs] Using diffusers * up --- .../conditional_image_generation.mdx | 2 - docs/source/using-diffusers/custom.mdx | 21 +-------- docs/source/using-diffusers/img2img.mdx | 42 +++++++++++------ docs/source/using-diffusers/inpaint.mdx | 46 +++++++++++++------ docs/source/using-diffusers/loading.mdx | 21 +-------- 5 files changed, 64 insertions(+), 68 deletions(-) diff --git a/docs/source/using-diffusers/conditional_image_generation.mdx b/docs/source/using-diffusers/conditional_image_generation.mdx index e3c5efca..6273a71d 100644 --- a/docs/source/using-diffusers/conditional_image_generation.mdx +++ b/docs/source/using-diffusers/conditional_image_generation.mdx @@ -10,8 +10,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> - - # Conditional Image Generation The [`DiffusionPipeline`] is the easiest way to use a pre-trained diffusion system for inference diff --git a/docs/source/using-diffusers/custom.mdx b/docs/source/using-diffusers/custom.mdx index 044f3937..2d17adaf 100644 --- a/docs/source/using-diffusers/custom.mdx +++ b/docs/source/using-diffusers/custom.mdx @@ -10,23 +10,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> +# Custom Pipeline - -# Quicktour - -Start using Diffusers🧨 quickly! -To start, use the [`DiffusionPipeline`] for quick inference and sample generations! - -``` -pip install diffusers -``` - -## Main classes - -### Models - -### Schedulers - -### Pipeliens - - +Under construction 🚧 diff --git a/docs/source/using-diffusers/img2img.mdx b/docs/source/using-diffusers/img2img.mdx index 044f3937..c1c8849f 100644 --- a/docs/source/using-diffusers/img2img.mdx +++ b/docs/source/using-diffusers/img2img.mdx @@ -10,23 +10,37 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> +# Text-Guided Image-to-Image Generation +The [`StableDiffusionImg2ImgPipeline`] lets you pass a text prompt and an initial image to condition the generation of new images. -# Quicktour +```python +from torch import autocast +import requests +from PIL import Image +from io import BytesIO -Start using Diffusers🧨 quickly! -To start, use the [`DiffusionPipeline`] for quick inference and sample generations! +from diffusers import StableDiffusionImg2ImgPipeline +# load the pipeline +device = "cuda" +pipe = StableDiffusionImg2ImgPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", revision="fp16", torch_dtype=torch.float16, use_auth_token=True +).to(device) + +# let's download an initial image +url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + +response = requests.get(url) +init_image = Image.open(BytesIO(response.content)).convert("RGB") +init_image = init_image.resize((768, 512)) + +prompt = "A fantasy landscape, trending on artstation" + +with autocast("cuda"): + images = pipe(prompt=prompt, init_image=init_image, strength=0.75, guidance_scale=7.5).images + +images[0].save("fantasy_landscape.png") ``` -pip install diffusers -``` - -## Main classes - -### Models - -### Schedulers - -### Pipeliens - +You can also run this example on colab [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/Notebooks/blob/master/image_2_image_using_diffusers.ipynb) diff --git a/docs/source/using-diffusers/inpaint.mdx b/docs/source/using-diffusers/inpaint.mdx index 044f3937..d4448863 100644 --- a/docs/source/using-diffusers/inpaint.mdx +++ b/docs/source/using-diffusers/inpaint.mdx @@ -10,23 +10,41 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> +# Text-Guided Image-Inpainting + +The [`StableDiffusionInpaintPipeline`] lets you edit specific parts of an image by providing a mask and text prompt. + +```python +from io import BytesIO + +from torch import autocast +import requests +import PIL + +from diffusers import StableDiffusionInpaintPipeline -# Quicktour +def download_image(url): + response = requests.get(url) + return PIL.Image.open(BytesIO(response.content)).convert("RGB") -Start using Diffusers🧨 quickly! -To start, use the [`DiffusionPipeline`] for quick inference and sample generations! -``` -pip install diffusers +img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" +mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" + +init_image = download_image(img_url).resize((512, 512)) +mask_image = download_image(mask_url).resize((512, 512)) + +device = "cuda" +pipe = StableDiffusionInpaintPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", revision="fp16", torch_dtype=torch.float16, use_auth_token=True +).to(device) + +prompt = "a cat sitting on a bench" +with autocast("cuda"): + images = pipe(prompt=prompt, init_image=init_image, mask_image=mask_image, strength=0.75).images + +images[0].save("cat_on_bench.png") ``` -## Main classes - -### Models - -### Schedulers - -### Pipeliens - - +You can also run this example on colab [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/Notebooks/blob/master/in_painting_with_stable_diffusion_using_diffusers.ipynb) diff --git a/docs/source/using-diffusers/loading.mdx b/docs/source/using-diffusers/loading.mdx index 044f3937..44b514bb 100644 --- a/docs/source/using-diffusers/loading.mdx +++ b/docs/source/using-diffusers/loading.mdx @@ -10,23 +10,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> +# Loading - -# Quicktour - -Start using Diffusers🧨 quickly! -To start, use the [`DiffusionPipeline`] for quick inference and sample generations! - -``` -pip install diffusers -``` - -## Main classes - -### Models - -### Schedulers - -### Pipeliens - - +Under construction 🚧