2022-05-31 06:27:59 -06:00
|
|
|
# coding=utf-8
|
|
|
|
# Copyright 2022 HuggingFace Inc.
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
2022-06-07 10:20:14 -06:00
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
import inspect
|
2022-05-31 06:27:59 -06:00
|
|
|
import tempfile
|
|
|
|
import unittest
|
|
|
|
|
2022-06-20 05:06:31 -06:00
|
|
|
import numpy as np
|
2022-05-31 06:27:59 -06:00
|
|
|
import torch
|
|
|
|
|
2022-06-27 09:39:41 -06:00
|
|
|
from diffusers import (
|
2022-06-29 04:34:24 -06:00
|
|
|
AutoencoderKL,
|
2022-06-22 07:40:08 -06:00
|
|
|
DDIMPipeline,
|
2022-06-15 04:35:47 -06:00
|
|
|
DDIMScheduler,
|
2022-06-22 07:40:08 -06:00
|
|
|
DDPMPipeline,
|
2022-06-15 04:35:47 -06:00
|
|
|
DDPMScheduler,
|
2022-06-22 07:40:08 -06:00
|
|
|
GlidePipeline,
|
2022-06-22 06:38:36 -06:00
|
|
|
GlideSuperResUNetModel,
|
|
|
|
GlideTextToImageUNetModel,
|
2022-06-22 07:40:08 -06:00
|
|
|
LatentDiffusionPipeline,
|
2022-06-29 07:25:51 -06:00
|
|
|
LatentDiffusionUncondPipeline,
|
2022-06-25 12:25:43 -06:00
|
|
|
NCSNpp,
|
2022-06-22 07:40:08 -06:00
|
|
|
PNDMPipeline,
|
2022-06-15 04:35:47 -06:00
|
|
|
PNDMScheduler,
|
2022-06-25 12:25:43 -06:00
|
|
|
ScoreSdeVePipeline,
|
|
|
|
ScoreSdeVeScheduler,
|
2022-06-26 17:41:55 -06:00
|
|
|
ScoreSdeVpPipeline,
|
|
|
|
ScoreSdeVpScheduler,
|
2022-06-21 02:43:40 -06:00
|
|
|
UNetLDMModel,
|
2022-07-12 10:34:41 -06:00
|
|
|
UNetUnconditionalModel,
|
2022-06-29 04:34:24 -06:00
|
|
|
VQModel,
|
2022-06-15 04:35:47 -06:00
|
|
|
)
|
2022-06-09 04:36:37 -06:00
|
|
|
from diffusers.configuration_utils import ConfigMixin
|
2022-06-09 06:06:58 -06:00
|
|
|
from diffusers.pipeline_utils import DiffusionPipeline
|
2022-06-12 15:20:39 -06:00
|
|
|
from diffusers.testing_utils import floats_tensor, slow, torch_device
|
2022-07-04 04:36:37 -06:00
|
|
|
from diffusers.training_utils import EMAModel
|
2022-05-31 06:27:59 -06:00
|
|
|
|
|
|
|
|
2022-06-12 11:59:39 -06:00
|
|
|
torch.backends.cuda.matmul.allow_tf32 = False
|
2022-05-31 06:27:59 -06:00
|
|
|
|
|
|
|
|
2022-06-09 04:36:37 -06:00
|
|
|
class ConfigTester(unittest.TestCase):
|
|
|
|
def test_load_not_from_mixin(self):
|
|
|
|
with self.assertRaises(ValueError):
|
|
|
|
ConfigMixin.from_config("dummy_path")
|
|
|
|
|
|
|
|
def test_save_load(self):
|
|
|
|
class SampleObject(ConfigMixin):
|
|
|
|
config_name = "config.json"
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
a=2,
|
|
|
|
b=5,
|
|
|
|
c=(2, 5),
|
|
|
|
d="for diffusion",
|
|
|
|
e=[1, 3],
|
|
|
|
):
|
2022-06-17 02:58:43 -06:00
|
|
|
self.register_to_config(a=a, b=b, c=c, d=d, e=e)
|
2022-06-09 04:36:37 -06:00
|
|
|
|
|
|
|
obj = SampleObject()
|
|
|
|
config = obj.config
|
|
|
|
|
|
|
|
assert config["a"] == 2
|
|
|
|
assert config["b"] == 5
|
|
|
|
assert config["c"] == (2, 5)
|
|
|
|
assert config["d"] == "for diffusion"
|
|
|
|
assert config["e"] == [1, 3]
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdirname:
|
|
|
|
obj.save_config(tmpdirname)
|
|
|
|
new_obj = SampleObject.from_config(tmpdirname)
|
|
|
|
new_config = new_obj.config
|
|
|
|
|
2022-06-17 03:55:02 -06:00
|
|
|
# unfreeze configs
|
|
|
|
config = dict(config)
|
|
|
|
new_config = dict(new_config)
|
|
|
|
|
2022-06-09 04:36:37 -06:00
|
|
|
assert config.pop("c") == (2, 5) # instantiated as tuple
|
|
|
|
assert new_config.pop("c") == [2, 5] # saved & loaded as list because of json
|
|
|
|
assert config == new_config
|
|
|
|
|
|
|
|
|
2022-06-17 05:49:26 -06:00
|
|
|
class ModelTesterMixin:
|
2022-05-31 06:27:59 -06:00
|
|
|
def test_from_pretrained_save_pretrained(self):
|
2022-06-17 05:49:26 -06:00
|
|
|
init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common()
|
|
|
|
|
|
|
|
model = self.model_class(**init_dict)
|
2022-06-12 13:56:13 -06:00
|
|
|
model.to(torch_device)
|
2022-06-17 05:49:26 -06:00
|
|
|
model.eval()
|
2022-05-31 06:27:59 -06:00
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdirname:
|
|
|
|
model.save_pretrained(tmpdirname)
|
2022-06-17 11:04:07 -06:00
|
|
|
new_model = self.model_class.from_pretrained(tmpdirname)
|
2022-06-12 13:56:13 -06:00
|
|
|
new_model.to(torch_device)
|
2022-05-31 06:27:59 -06:00
|
|
|
|
2022-06-17 05:49:26 -06:00
|
|
|
with torch.no_grad():
|
|
|
|
image = model(**inputs_dict)
|
2022-07-15 04:48:30 -06:00
|
|
|
if isinstance(image, dict):
|
|
|
|
image = image["sample"]
|
|
|
|
|
2022-06-17 05:49:26 -06:00
|
|
|
new_image = new_model(**inputs_dict)
|
2022-05-31 06:27:59 -06:00
|
|
|
|
2022-07-15 04:48:30 -06:00
|
|
|
if isinstance(new_image, dict):
|
|
|
|
new_image = new_image["sample"]
|
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
max_diff = (image - new_image).abs().sum().item()
|
2022-06-27 03:07:57 -06:00
|
|
|
self.assertLessEqual(max_diff, 5e-5, "Models give different forward passes")
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 05:36:59 -06:00
|
|
|
def test_determinism(self):
|
2022-06-17 08:36:51 -06:00
|
|
|
init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common()
|
|
|
|
model = self.model_class(**init_dict)
|
|
|
|
model.to(torch_device)
|
|
|
|
model.eval()
|
|
|
|
with torch.no_grad():
|
|
|
|
first = model(**inputs_dict)
|
2022-07-15 04:48:30 -06:00
|
|
|
if isinstance(first, dict):
|
|
|
|
first = first["sample"]
|
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
second = model(**inputs_dict)
|
2022-07-15 04:48:30 -06:00
|
|
|
if isinstance(second, dict):
|
|
|
|
second = second["sample"]
|
2022-06-17 08:36:51 -06:00
|
|
|
|
|
|
|
out_1 = first.cpu().numpy()
|
|
|
|
out_2 = second.cpu().numpy()
|
|
|
|
out_1 = out_1[~np.isnan(out_1)]
|
|
|
|
out_2 = out_2[~np.isnan(out_2)]
|
|
|
|
max_diff = np.amax(np.abs(out_1 - out_2))
|
|
|
|
self.assertLessEqual(max_diff, 1e-5)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 05:36:59 -06:00
|
|
|
def test_output(self):
|
2022-06-17 08:36:51 -06:00
|
|
|
init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common()
|
|
|
|
model = self.model_class(**init_dict)
|
|
|
|
model.to(torch_device)
|
|
|
|
model.eval()
|
|
|
|
|
|
|
|
with torch.no_grad():
|
|
|
|
output = model(**inputs_dict)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-07-15 04:48:30 -06:00
|
|
|
if isinstance(output, dict):
|
|
|
|
output = output["sample"]
|
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
self.assertIsNotNone(output)
|
2022-07-12 10:34:41 -06:00
|
|
|
expected_shape = inputs_dict["sample"].shape
|
2022-06-17 08:36:51 -06:00
|
|
|
self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match")
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 05:36:59 -06:00
|
|
|
def test_forward_signature(self):
|
2022-06-17 08:36:51 -06:00
|
|
|
init_dict, _ = self.prepare_init_args_and_inputs_for_common()
|
|
|
|
|
|
|
|
model = self.model_class(**init_dict)
|
|
|
|
signature = inspect.signature(model.forward)
|
|
|
|
# signature.parameters is an OrderedDict => so arg_names order is deterministic
|
|
|
|
arg_names = [*signature.parameters.keys()]
|
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
expected_arg_names = ["sample", "timestep"]
|
2022-06-17 08:36:51 -06:00
|
|
|
self.assertListEqual(arg_names[:2], expected_arg_names)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 05:36:59 -06:00
|
|
|
def test_model_from_config(self):
|
2022-06-17 08:36:51 -06:00
|
|
|
init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common()
|
|
|
|
|
|
|
|
model = self.model_class(**init_dict)
|
|
|
|
model.to(torch_device)
|
|
|
|
model.eval()
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
# test if the model can be loaded from the config
|
|
|
|
# and has all the expected shape
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdirname:
|
|
|
|
model.save_config(tmpdirname)
|
|
|
|
new_model = self.model_class.from_config(tmpdirname)
|
|
|
|
new_model.to(torch_device)
|
|
|
|
new_model.eval()
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
# check if all paramters shape are the same
|
|
|
|
for param_name in model.state_dict().keys():
|
|
|
|
param_1 = model.state_dict()[param_name]
|
|
|
|
param_2 = new_model.state_dict()[param_name]
|
|
|
|
self.assertEqual(param_1.shape, param_2.shape)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
with torch.no_grad():
|
|
|
|
output_1 = model(**inputs_dict)
|
2022-07-15 04:48:30 -06:00
|
|
|
|
|
|
|
if isinstance(output_1, dict):
|
|
|
|
output_1 = output_1["sample"]
|
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
output_2 = new_model(**inputs_dict)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-07-15 04:48:30 -06:00
|
|
|
if isinstance(output_2, dict):
|
|
|
|
output_2 = output_2["sample"]
|
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
self.assertEqual(output_1.shape, output_2.shape)
|
2022-06-17 05:36:59 -06:00
|
|
|
|
|
|
|
def test_training(self):
|
2022-06-17 08:36:51 -06:00
|
|
|
init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common()
|
2022-06-01 16:25:48 -06:00
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
model = self.model_class(**init_dict)
|
|
|
|
model.to(torch_device)
|
|
|
|
model.train()
|
|
|
|
output = model(**inputs_dict)
|
2022-07-15 04:48:30 -06:00
|
|
|
|
|
|
|
if isinstance(output, dict):
|
|
|
|
output = output["sample"]
|
|
|
|
|
2022-07-12 10:34:41 -06:00
|
|
|
noise = torch.randn((inputs_dict["sample"].shape[0],) + self.output_shape).to(torch_device)
|
2022-06-17 08:36:51 -06:00
|
|
|
loss = torch.nn.functional.mse_loss(output, noise)
|
|
|
|
loss.backward()
|
2022-06-01 16:25:48 -06:00
|
|
|
|
2022-07-04 04:36:37 -06:00
|
|
|
def test_ema_training(self):
|
|
|
|
init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common()
|
|
|
|
|
|
|
|
model = self.model_class(**init_dict)
|
|
|
|
model.to(torch_device)
|
|
|
|
model.train()
|
|
|
|
ema_model = EMAModel(model, device=torch_device)
|
|
|
|
|
|
|
|
output = model(**inputs_dict)
|
2022-07-15 04:48:30 -06:00
|
|
|
|
|
|
|
if isinstance(output, dict):
|
|
|
|
output = output["sample"]
|
|
|
|
|
2022-07-12 10:34:41 -06:00
|
|
|
noise = torch.randn((inputs_dict["sample"].shape[0],) + self.output_shape).to(torch_device)
|
2022-07-04 04:36:37 -06:00
|
|
|
loss = torch.nn.functional.mse_loss(output, noise)
|
|
|
|
loss.backward()
|
|
|
|
ema_model.step(model)
|
|
|
|
|
2022-06-17 05:49:26 -06:00
|
|
|
|
|
|
|
class UnetModelTests(ModelTesterMixin, unittest.TestCase):
|
2022-07-14 11:46:04 -06:00
|
|
|
model_class = UNetUnconditionalModel
|
2022-06-17 05:49:26 -06:00
|
|
|
|
|
|
|
@property
|
|
|
|
def dummy_input(self):
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 3
|
|
|
|
sizes = (32, 32)
|
|
|
|
|
|
|
|
noise = floats_tensor((batch_size, num_channels) + sizes).to(torch_device)
|
|
|
|
time_step = torch.tensor([10]).to(torch_device)
|
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
return {"sample": noise, "timestep": time_step}
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def input_shape(self):
|
2022-06-17 11:04:07 -06:00
|
|
|
return (3, 32, 32)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def output_shape(self):
|
2022-06-17 11:04:07 -06:00
|
|
|
return (3, 32, 32)
|
2022-06-17 05:49:26 -06:00
|
|
|
|
|
|
|
def prepare_init_args_and_inputs_for_common(self):
|
|
|
|
init_dict = {
|
|
|
|
"ch": 32,
|
|
|
|
"ch_mult": (1, 2),
|
2022-07-14 11:46:04 -06:00
|
|
|
"block_channels": (32, 64),
|
|
|
|
"down_blocks": ("UNetResDownBlock2D", "UNetResAttnDownBlock2D"),
|
|
|
|
"up_blocks": ("UNetResAttnUpBlock2D", "UNetResUpBlock2D"),
|
|
|
|
"num_head_channels": None,
|
|
|
|
"out_channels": 3,
|
|
|
|
"in_channels": 3,
|
2022-06-17 05:49:26 -06:00
|
|
|
"num_res_blocks": 2,
|
|
|
|
"attn_resolutions": (16,),
|
|
|
|
"resolution": 32,
|
2022-07-14 11:46:04 -06:00
|
|
|
"image_size": 32,
|
2022-06-17 05:49:26 -06:00
|
|
|
}
|
|
|
|
inputs_dict = self.dummy_input
|
|
|
|
return init_dict, inputs_dict
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 05:49:26 -06:00
|
|
|
def test_from_pretrained_hub(self):
|
2022-07-14 11:46:04 -06:00
|
|
|
model, loading_info = UNetUnconditionalModel.from_pretrained(
|
|
|
|
"fusing/ddpm_dummy", output_loading_info=True, ddpm=True
|
|
|
|
)
|
2022-06-17 08:36:51 -06:00
|
|
|
self.assertIsNotNone(model)
|
2022-07-15 11:22:28 -06:00
|
|
|
# self.assertEqual(len(loading_info["missing_keys"]), 0)
|
2022-06-17 05:49:26 -06:00
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
model.to(torch_device)
|
2022-07-15 04:48:30 -06:00
|
|
|
image = model(**self.dummy_input)["sample"]
|
2022-06-17 05:49:26 -06:00
|
|
|
|
|
|
|
assert image is not None, "Make sure output is not None"
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 08:36:51 -06:00
|
|
|
def test_output_pretrained(self):
|
2022-07-14 04:29:06 -06:00
|
|
|
model = UNetUnconditionalModel.from_pretrained("fusing/ddpm_dummy", ddpm=True)
|
|
|
|
model.eval()
|
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
|
|
|
|
|
|
|
noise = torch.randn(1, model.config.in_channels, model.config.image_size, model.config.image_size)
|
|
|
|
time_step = torch.tensor([10])
|
|
|
|
|
|
|
|
with torch.no_grad():
|
2022-07-15 04:48:30 -06:00
|
|
|
output = model(noise, time_step)["sample"]
|
2022-07-14 04:29:06 -06:00
|
|
|
|
|
|
|
output_slice = output[0, -1, -3:, -3:].flatten()
|
|
|
|
# fmt: off
|
|
|
|
expected_output_slice = torch.tensor([0.2891, -0.1899, 0.2595, -0.6214, 0.0968, -0.2622, 0.4688, 0.1311, 0.0053])
|
|
|
|
# fmt: on
|
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, rtol=1e-2))
|
|
|
|
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-22 06:38:36 -06:00
|
|
|
class GlideSuperResUNetTests(ModelTesterMixin, unittest.TestCase):
|
|
|
|
model_class = GlideSuperResUNetModel
|
2022-06-17 11:04:07 -06:00
|
|
|
|
|
|
|
@property
|
|
|
|
def dummy_input(self):
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 6
|
|
|
|
sizes = (32, 32)
|
|
|
|
low_res_size = (4, 4)
|
|
|
|
|
|
|
|
noise = torch.randn((batch_size, num_channels // 2) + sizes).to(torch_device)
|
|
|
|
low_res = torch.randn((batch_size, 3) + low_res_size).to(torch_device)
|
|
|
|
time_step = torch.tensor([10] * noise.shape[0], device=torch_device)
|
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
return {"sample": noise, "timestep": time_step, "low_res": low_res}
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def input_shape(self):
|
2022-06-17 11:04:07 -06:00
|
|
|
return (3, 32, 32)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def output_shape(self):
|
2022-06-17 11:04:07 -06:00
|
|
|
return (6, 32, 32)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
def prepare_init_args_and_inputs_for_common(self):
|
|
|
|
init_dict = {
|
|
|
|
"attention_resolutions": (2,),
|
2022-06-20 05:06:31 -06:00
|
|
|
"channel_mult": (1, 2),
|
2022-06-17 11:04:07 -06:00
|
|
|
"in_channels": 6,
|
|
|
|
"out_channels": 6,
|
|
|
|
"model_channels": 32,
|
|
|
|
"num_head_channels": 8,
|
|
|
|
"num_heads_upsample": 1,
|
|
|
|
"num_res_blocks": 2,
|
|
|
|
"resblock_updown": True,
|
|
|
|
"resolution": 32,
|
2022-06-20 05:06:31 -06:00
|
|
|
"use_scale_shift_norm": True,
|
2022-06-17 11:04:07 -06:00
|
|
|
}
|
|
|
|
inputs_dict = self.dummy_input
|
|
|
|
return init_dict, inputs_dict
|
|
|
|
|
|
|
|
def test_output(self):
|
|
|
|
init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common()
|
|
|
|
model = self.model_class(**init_dict)
|
|
|
|
model.to(torch_device)
|
|
|
|
model.eval()
|
|
|
|
|
|
|
|
with torch.no_grad():
|
|
|
|
output = model(**inputs_dict)
|
|
|
|
|
|
|
|
output, _ = torch.split(output, 3, dim=1)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
self.assertIsNotNone(output)
|
2022-07-12 10:34:41 -06:00
|
|
|
expected_shape = inputs_dict["sample"].shape
|
2022-06-17 11:04:07 -06:00
|
|
|
self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match")
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
def test_from_pretrained_hub(self):
|
2022-06-22 06:38:36 -06:00
|
|
|
model, loading_info = GlideSuperResUNetModel.from_pretrained(
|
2022-06-20 05:06:31 -06:00
|
|
|
"fusing/glide-super-res-dummy", output_loading_info=True
|
|
|
|
)
|
2022-06-17 11:04:07 -06:00
|
|
|
self.assertIsNotNone(model)
|
2022-07-15 11:22:28 -06:00
|
|
|
# self.assertEqual(len(loading_info["missing_keys"]), 0)
|
2022-06-17 11:04:07 -06:00
|
|
|
|
|
|
|
model.to(torch_device)
|
|
|
|
image = model(**self.dummy_input)
|
|
|
|
|
|
|
|
assert image is not None, "Make sure output is not None"
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
def test_output_pretrained(self):
|
2022-06-22 06:38:36 -06:00
|
|
|
model = GlideSuperResUNetModel.from_pretrained("fusing/glide-super-res-dummy")
|
2022-06-17 11:04:07 -06:00
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-20 06:27:37 -06:00
|
|
|
noise = torch.randn(1, 3, 64, 64)
|
2022-06-17 11:04:07 -06:00
|
|
|
low_res = torch.randn(1, 3, 4, 4)
|
|
|
|
time_step = torch.tensor([42] * noise.shape[0])
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
with torch.no_grad():
|
|
|
|
output = model(noise, time_step, low_res)
|
2022-06-20 05:06:31 -06:00
|
|
|
|
2022-06-17 11:04:07 -06:00
|
|
|
output, _ = torch.split(output, 3, dim=1)
|
|
|
|
output_slice = output[0, -1, -3:, -3:].flatten()
|
|
|
|
# fmt: off
|
2022-06-20 06:27:37 -06:00
|
|
|
expected_output_slice = torch.tensor([-22.8782, -23.2652, -15.3966, -22.8034, -23.3159, -15.5640, -15.3970, -15.4614, - 10.4370])
|
2022-06-17 11:04:07 -06:00
|
|
|
# fmt: on
|
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, atol=1e-3))
|
2022-06-17 05:49:26 -06:00
|
|
|
|
2022-06-21 02:43:40 -06:00
|
|
|
|
2022-06-22 06:38:36 -06:00
|
|
|
class GlideTextToImageUNetModelTests(ModelTesterMixin, unittest.TestCase):
|
|
|
|
model_class = GlideTextToImageUNetModel
|
2022-06-21 04:01:07 -06:00
|
|
|
|
|
|
|
@property
|
|
|
|
def dummy_input(self):
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 3
|
|
|
|
sizes = (32, 32)
|
|
|
|
transformer_dim = 32
|
|
|
|
seq_len = 16
|
|
|
|
|
|
|
|
noise = torch.randn((batch_size, num_channels) + sizes).to(torch_device)
|
|
|
|
emb = torch.randn((batch_size, seq_len, transformer_dim)).to(torch_device)
|
|
|
|
time_step = torch.tensor([10] * noise.shape[0], device=torch_device)
|
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
return {"sample": noise, "timestep": time_step, "transformer_out": emb}
|
2022-06-21 04:01:07 -06:00
|
|
|
|
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def input_shape(self):
|
2022-06-21 04:01:07 -06:00
|
|
|
return (3, 32, 32)
|
|
|
|
|
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def output_shape(self):
|
2022-06-21 04:01:07 -06:00
|
|
|
return (6, 32, 32)
|
|
|
|
|
|
|
|
def prepare_init_args_and_inputs_for_common(self):
|
|
|
|
init_dict = {
|
|
|
|
"attention_resolutions": (2,),
|
|
|
|
"channel_mult": (1, 2),
|
|
|
|
"in_channels": 3,
|
|
|
|
"out_channels": 6,
|
|
|
|
"model_channels": 32,
|
|
|
|
"num_head_channels": 8,
|
|
|
|
"num_heads_upsample": 1,
|
|
|
|
"num_res_blocks": 2,
|
|
|
|
"resblock_updown": True,
|
|
|
|
"resolution": 32,
|
|
|
|
"use_scale_shift_norm": True,
|
|
|
|
"transformer_dim": 32,
|
|
|
|
}
|
|
|
|
inputs_dict = self.dummy_input
|
|
|
|
return init_dict, inputs_dict
|
|
|
|
|
|
|
|
def test_output(self):
|
|
|
|
init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common()
|
|
|
|
model = self.model_class(**init_dict)
|
|
|
|
model.to(torch_device)
|
|
|
|
model.eval()
|
|
|
|
|
|
|
|
with torch.no_grad():
|
|
|
|
output = model(**inputs_dict)
|
|
|
|
|
|
|
|
output, _ = torch.split(output, 3, dim=1)
|
|
|
|
|
|
|
|
self.assertIsNotNone(output)
|
2022-07-12 10:34:41 -06:00
|
|
|
expected_shape = inputs_dict["sample"].shape
|
2022-06-21 04:01:07 -06:00
|
|
|
self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match")
|
|
|
|
|
|
|
|
def test_from_pretrained_hub(self):
|
2022-06-22 06:38:36 -06:00
|
|
|
model, loading_info = GlideTextToImageUNetModel.from_pretrained(
|
2022-06-21 04:01:07 -06:00
|
|
|
"fusing/unet-glide-text2im-dummy", output_loading_info=True
|
|
|
|
)
|
|
|
|
self.assertIsNotNone(model)
|
2022-07-15 11:22:28 -06:00
|
|
|
# self.assertEqual(len(loading_info["missing_keys"]), 0)
|
2022-06-21 04:01:07 -06:00
|
|
|
|
|
|
|
model.to(torch_device)
|
|
|
|
image = model(**self.dummy_input)
|
|
|
|
|
|
|
|
assert image is not None, "Make sure output is not None"
|
|
|
|
|
|
|
|
def test_output_pretrained(self):
|
2022-06-22 06:38:36 -06:00
|
|
|
model = GlideTextToImageUNetModel.from_pretrained("fusing/unet-glide-text2im-dummy")
|
2022-06-21 04:01:07 -06:00
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
|
|
|
|
|
|
|
noise = torch.randn((1, model.config.in_channels, model.config.resolution, model.config.resolution)).to(
|
|
|
|
torch_device
|
|
|
|
)
|
|
|
|
emb = torch.randn((1, 16, model.config.transformer_dim)).to(torch_device)
|
|
|
|
time_step = torch.tensor([10] * noise.shape[0], device=torch_device)
|
|
|
|
|
2022-06-27 03:07:57 -06:00
|
|
|
model.to(torch_device)
|
2022-06-21 04:01:07 -06:00
|
|
|
with torch.no_grad():
|
|
|
|
output = model(noise, time_step, emb)
|
|
|
|
|
|
|
|
output, _ = torch.split(output, 3, dim=1)
|
2022-06-27 03:07:57 -06:00
|
|
|
output_slice = output[0, -1, -3:, -3:].cpu().flatten()
|
2022-06-21 04:01:07 -06:00
|
|
|
# fmt: off
|
2022-06-22 06:38:36 -06:00
|
|
|
expected_output_slice = torch.tensor([2.7766, -10.3558, -14.9149, -0.9376, -14.9175, -17.7679, -5.5565, -12.9521, -12.9845])
|
2022-06-21 04:01:07 -06:00
|
|
|
# fmt: on
|
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, atol=1e-3))
|
|
|
|
|
|
|
|
|
2022-06-20 06:45:58 -06:00
|
|
|
class UNetLDMModelTests(ModelTesterMixin, unittest.TestCase):
|
2022-07-12 10:34:41 -06:00
|
|
|
model_class = UNetUnconditionalModel
|
2022-06-20 06:45:58 -06:00
|
|
|
|
|
|
|
@property
|
|
|
|
def dummy_input(self):
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 4
|
|
|
|
sizes = (32, 32)
|
|
|
|
|
|
|
|
noise = floats_tensor((batch_size, num_channels) + sizes).to(torch_device)
|
|
|
|
time_step = torch.tensor([10]).to(torch_device)
|
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
return {"sample": noise, "timestep": time_step}
|
2022-06-20 06:45:58 -06:00
|
|
|
|
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def input_shape(self):
|
2022-06-20 06:45:58 -06:00
|
|
|
return (4, 32, 32)
|
|
|
|
|
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def output_shape(self):
|
2022-06-20 06:45:58 -06:00
|
|
|
return (4, 32, 32)
|
|
|
|
|
|
|
|
def prepare_init_args_and_inputs_for_common(self):
|
|
|
|
init_dict = {
|
|
|
|
"image_size": 32,
|
|
|
|
"in_channels": 4,
|
|
|
|
"out_channels": 4,
|
|
|
|
"num_res_blocks": 2,
|
|
|
|
"attention_resolutions": (16,),
|
2022-07-14 04:29:06 -06:00
|
|
|
"block_channels": (32, 64),
|
2022-07-12 10:34:41 -06:00
|
|
|
"num_head_channels": 32,
|
2022-06-20 06:45:58 -06:00
|
|
|
"conv_resample": True,
|
2022-07-13 04:21:11 -06:00
|
|
|
"down_blocks": ("UNetResDownBlock2D", "UNetResDownBlock2D"),
|
|
|
|
"up_blocks": ("UNetResUpBlock2D", "UNetResUpBlock2D"),
|
2022-07-14 04:29:06 -06:00
|
|
|
"ldm": True,
|
2022-06-20 06:45:58 -06:00
|
|
|
}
|
|
|
|
inputs_dict = self.dummy_input
|
|
|
|
return init_dict, inputs_dict
|
2022-06-21 02:43:40 -06:00
|
|
|
|
2022-06-20 06:45:58 -06:00
|
|
|
def test_from_pretrained_hub(self):
|
2022-07-14 04:29:06 -06:00
|
|
|
model, loading_info = UNetUnconditionalModel.from_pretrained(
|
|
|
|
"fusing/unet-ldm-dummy", output_loading_info=True, ldm=True
|
|
|
|
)
|
2022-06-20 06:45:58 -06:00
|
|
|
self.assertIsNotNone(model)
|
2022-07-15 11:22:28 -06:00
|
|
|
# self.assertEqual(len(loading_info["missing_keys"]), 0)
|
2022-06-20 06:45:58 -06:00
|
|
|
|
|
|
|
model.to(torch_device)
|
2022-07-15 04:48:30 -06:00
|
|
|
image = model(**self.dummy_input)["sample"]
|
2022-06-20 06:45:58 -06:00
|
|
|
|
|
|
|
assert image is not None, "Make sure output is not None"
|
|
|
|
|
|
|
|
def test_output_pretrained(self):
|
2022-07-14 04:29:06 -06:00
|
|
|
model = UNetUnconditionalModel.from_pretrained("fusing/unet-ldm-dummy", ldm=True)
|
2022-06-20 06:45:58 -06:00
|
|
|
model.eval()
|
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
|
|
|
|
|
|
|
noise = torch.randn(1, model.config.in_channels, model.config.image_size, model.config.image_size)
|
|
|
|
time_step = torch.tensor([10] * noise.shape[0])
|
|
|
|
|
|
|
|
with torch.no_grad():
|
2022-07-15 04:48:30 -06:00
|
|
|
output = model(noise, time_step)["sample"]
|
2022-06-20 06:45:58 -06:00
|
|
|
|
|
|
|
output_slice = output[0, -1, -3:, -3:].flatten()
|
|
|
|
# fmt: off
|
|
|
|
expected_output_slice = torch.tensor([-13.3258, -20.1100, -15.9873, -17.6617, -23.0596, -17.9419, -13.3675, -16.1889, -12.3800])
|
|
|
|
# fmt: on
|
|
|
|
|
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, atol=1e-3))
|
|
|
|
|
2022-06-27 07:25:26 -06:00
|
|
|
def test_output_pretrained_spatial_transformer(self):
|
|
|
|
model = UNetLDMModel.from_pretrained("fusing/unet-ldm-dummy-spatial")
|
|
|
|
model.eval()
|
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
|
|
|
|
|
|
|
noise = torch.randn(1, model.config.in_channels, model.config.image_size, model.config.image_size)
|
|
|
|
context = torch.ones((1, 16, 64), dtype=torch.float32)
|
|
|
|
time_step = torch.tensor([10] * noise.shape[0])
|
|
|
|
|
|
|
|
with torch.no_grad():
|
|
|
|
output = model(noise, time_step, context=context)
|
|
|
|
|
|
|
|
output_slice = output[0, -1, -3:, -3:].flatten()
|
|
|
|
# fmt: off
|
|
|
|
expected_output_slice = torch.tensor([61.3445, 56.9005, 29.4339, 59.5497, 60.7375, 34.1719, 48.1951, 42.6569, 25.0890])
|
|
|
|
# fmt: on
|
|
|
|
|
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, atol=1e-3))
|
|
|
|
|
2022-06-17 05:49:26 -06:00
|
|
|
|
2022-06-27 11:20:15 -06:00
|
|
|
class NCSNppModelTests(ModelTesterMixin, unittest.TestCase):
|
|
|
|
model_class = NCSNpp
|
|
|
|
|
|
|
|
@property
|
|
|
|
def dummy_input(self):
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 3
|
|
|
|
sizes = (32, 32)
|
|
|
|
|
|
|
|
noise = floats_tensor((batch_size, num_channels) + sizes).to(torch_device)
|
|
|
|
time_step = torch.tensor(batch_size * [10]).to(torch_device)
|
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
return {"sample": noise, "timestep": time_step}
|
2022-06-27 11:20:15 -06:00
|
|
|
|
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def input_shape(self):
|
2022-06-27 11:20:15 -06:00
|
|
|
return (3, 32, 32)
|
|
|
|
|
|
|
|
@property
|
2022-06-28 03:50:21 -06:00
|
|
|
def output_shape(self):
|
2022-06-27 11:20:15 -06:00
|
|
|
return (3, 32, 32)
|
|
|
|
|
|
|
|
def prepare_init_args_and_inputs_for_common(self):
|
|
|
|
init_dict = {
|
|
|
|
"image_size": 32,
|
|
|
|
"ch_mult": [1, 2, 2, 2],
|
|
|
|
"nf": 32,
|
|
|
|
"fir": True,
|
|
|
|
"progressive": "output_skip",
|
|
|
|
"progressive_combine": "sum",
|
|
|
|
"progressive_input": "input_skip",
|
|
|
|
"scale_by_sigma": True,
|
|
|
|
"skip_rescale": True,
|
|
|
|
"embedding_type": "fourier",
|
|
|
|
}
|
|
|
|
inputs_dict = self.dummy_input
|
|
|
|
return init_dict, inputs_dict
|
|
|
|
|
|
|
|
def test_from_pretrained_hub(self):
|
|
|
|
model, loading_info = NCSNpp.from_pretrained("fusing/cifar10-ncsnpp-ve", output_loading_info=True)
|
|
|
|
self.assertIsNotNone(model)
|
2022-07-15 11:22:28 -06:00
|
|
|
# self.assertEqual(len(loading_info["missing_keys"]), 0)
|
2022-06-27 11:20:15 -06:00
|
|
|
|
|
|
|
model.to(torch_device)
|
|
|
|
image = model(**self.dummy_input)
|
|
|
|
|
|
|
|
assert image is not None, "Make sure output is not None"
|
|
|
|
|
|
|
|
def test_output_pretrained_ve_small(self):
|
|
|
|
model = NCSNpp.from_pretrained("fusing/ncsnpp-cifar10-ve-dummy")
|
|
|
|
model.eval()
|
|
|
|
model.to(torch_device)
|
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
|
|
|
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 3
|
|
|
|
sizes = (32, 32)
|
|
|
|
|
2022-06-30 09:54:00 -06:00
|
|
|
noise = torch.ones((batch_size, num_channels) + sizes).to(torch_device)
|
|
|
|
time_step = torch.tensor(batch_size * [1e-4]).to(torch_device)
|
2022-06-27 11:20:15 -06:00
|
|
|
|
|
|
|
with torch.no_grad():
|
|
|
|
output = model(noise, time_step)
|
|
|
|
|
|
|
|
output_slice = output[0, -3:, -3:, -1].flatten().cpu()
|
|
|
|
# fmt: off
|
2022-06-30 09:54:00 -06:00
|
|
|
expected_output_slice = torch.tensor([0.1315, 0.0741, 0.0393, 0.0455, 0.0556, 0.0180, -0.0832, -0.0644, -0.0856])
|
2022-06-27 11:20:15 -06:00
|
|
|
# fmt: on
|
|
|
|
|
2022-06-30 09:54:00 -06:00
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, rtol=1e-2))
|
2022-06-27 11:20:15 -06:00
|
|
|
|
|
|
|
def test_output_pretrained_ve_large(self):
|
|
|
|
model = NCSNpp.from_pretrained("fusing/ncsnpp-ffhq-ve-dummy")
|
|
|
|
model.eval()
|
|
|
|
model.to(torch_device)
|
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
|
|
|
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 3
|
|
|
|
sizes = (32, 32)
|
|
|
|
|
2022-06-30 09:54:00 -06:00
|
|
|
noise = torch.ones((batch_size, num_channels) + sizes).to(torch_device)
|
|
|
|
time_step = torch.tensor(batch_size * [1e-4]).to(torch_device)
|
2022-06-27 11:20:15 -06:00
|
|
|
|
|
|
|
with torch.no_grad():
|
|
|
|
output = model(noise, time_step)
|
|
|
|
|
|
|
|
output_slice = output[0, -3:, -3:, -1].flatten().cpu()
|
|
|
|
# fmt: off
|
2022-06-30 09:54:00 -06:00
|
|
|
expected_output_slice = torch.tensor([-0.0325, -0.0900, -0.0869, -0.0332, -0.0725, -0.0270, -0.0101, 0.0227, 0.0256])
|
2022-06-27 11:20:15 -06:00
|
|
|
# fmt: on
|
|
|
|
|
2022-06-30 09:54:00 -06:00
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, rtol=1e-2))
|
2022-06-27 11:20:15 -06:00
|
|
|
|
|
|
|
def test_output_pretrained_vp(self):
|
2022-06-30 09:54:00 -06:00
|
|
|
model = NCSNpp.from_pretrained("fusing/cifar10-ddpmpp-vp")
|
2022-06-27 11:20:15 -06:00
|
|
|
model.eval()
|
|
|
|
model.to(torch_device)
|
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
|
|
|
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 3
|
|
|
|
sizes = (32, 32)
|
|
|
|
|
2022-06-30 09:54:00 -06:00
|
|
|
noise = torch.randn((batch_size, num_channels) + sizes).to(torch_device)
|
2022-06-30 16:24:22 -06:00
|
|
|
time_step = torch.tensor(batch_size * [9.0]).to(torch_device)
|
2022-06-27 11:20:15 -06:00
|
|
|
|
|
|
|
with torch.no_grad():
|
|
|
|
output = model(noise, time_step)
|
|
|
|
|
|
|
|
output_slice = output[0, -3:, -3:, -1].flatten().cpu()
|
|
|
|
# fmt: off
|
2022-06-30 09:54:00 -06:00
|
|
|
expected_output_slice = torch.tensor([0.3303, -0.2275, -2.8872, -0.1309, -1.2861, 3.4567, -1.0083, 2.5325, -1.3866])
|
2022-06-27 11:20:15 -06:00
|
|
|
# fmt: on
|
|
|
|
|
2022-06-30 09:54:00 -06:00
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, rtol=1e-2))
|
2022-06-27 11:20:15 -06:00
|
|
|
|
|
|
|
|
2022-06-29 04:34:24 -06:00
|
|
|
class VQModelTests(ModelTesterMixin, unittest.TestCase):
|
|
|
|
model_class = VQModel
|
|
|
|
|
|
|
|
@property
|
|
|
|
def dummy_input(self):
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 3
|
|
|
|
sizes = (32, 32)
|
|
|
|
|
|
|
|
image = floats_tensor((batch_size, num_channels) + sizes).to(torch_device)
|
|
|
|
|
2022-07-12 10:34:41 -06:00
|
|
|
return {"sample": image}
|
2022-06-29 04:34:24 -06:00
|
|
|
|
|
|
|
@property
|
|
|
|
def input_shape(self):
|
|
|
|
return (3, 32, 32)
|
|
|
|
|
|
|
|
@property
|
|
|
|
def output_shape(self):
|
|
|
|
return (3, 32, 32)
|
|
|
|
|
|
|
|
def prepare_init_args_and_inputs_for_common(self):
|
|
|
|
init_dict = {
|
|
|
|
"ch": 64,
|
|
|
|
"out_ch": 3,
|
|
|
|
"num_res_blocks": 1,
|
|
|
|
"attn_resolutions": [],
|
|
|
|
"in_channels": 3,
|
|
|
|
"resolution": 32,
|
|
|
|
"z_channels": 3,
|
|
|
|
"n_embed": 256,
|
|
|
|
"embed_dim": 3,
|
|
|
|
"sane_index_shape": False,
|
|
|
|
"ch_mult": (1,),
|
|
|
|
"dropout": 0.0,
|
|
|
|
"double_z": False,
|
|
|
|
}
|
|
|
|
inputs_dict = self.dummy_input
|
|
|
|
return init_dict, inputs_dict
|
|
|
|
|
|
|
|
def test_forward_signature(self):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def test_training(self):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def test_from_pretrained_hub(self):
|
|
|
|
model, loading_info = VQModel.from_pretrained("fusing/vqgan-dummy", output_loading_info=True)
|
|
|
|
self.assertIsNotNone(model)
|
2022-07-15 11:22:28 -06:00
|
|
|
# self.assertEqual(len(loading_info["missing_keys"]), 0)
|
2022-06-29 04:34:24 -06:00
|
|
|
|
|
|
|
model.to(torch_device)
|
|
|
|
image = model(**self.dummy_input)
|
|
|
|
|
|
|
|
assert image is not None, "Make sure output is not None"
|
|
|
|
|
|
|
|
def test_output_pretrained(self):
|
|
|
|
model = VQModel.from_pretrained("fusing/vqgan-dummy")
|
|
|
|
model.eval()
|
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
|
|
|
|
|
|
|
image = torch.randn(1, model.config.in_channels, model.config.resolution, model.config.resolution)
|
|
|
|
with torch.no_grad():
|
|
|
|
output = model(image)
|
|
|
|
|
|
|
|
output_slice = output[0, -1, -3:, -3:].flatten()
|
|
|
|
# fmt: off
|
2022-06-30 08:54:31 -06:00
|
|
|
expected_output_slice = torch.tensor([-1.1321, 0.1056, 0.3505, -0.6461, -0.2014, 0.0419, -0.5763, -0.8462, -0.4218])
|
2022-06-29 04:34:24 -06:00
|
|
|
# fmt: on
|
2022-06-30 16:29:18 -06:00
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, rtol=1e-2))
|
2022-06-29 04:34:24 -06:00
|
|
|
|
|
|
|
|
2022-06-29 05:52:04 -06:00
|
|
|
class AutoEncoderKLTests(ModelTesterMixin, unittest.TestCase):
|
|
|
|
model_class = AutoencoderKL
|
|
|
|
|
|
|
|
@property
|
|
|
|
def dummy_input(self):
|
|
|
|
batch_size = 4
|
|
|
|
num_channels = 3
|
|
|
|
sizes = (32, 32)
|
|
|
|
|
|
|
|
image = floats_tensor((batch_size, num_channels) + sizes).to(torch_device)
|
|
|
|
|
2022-07-12 10:34:41 -06:00
|
|
|
return {"sample": image}
|
2022-06-29 05:52:04 -06:00
|
|
|
|
|
|
|
@property
|
|
|
|
def input_shape(self):
|
|
|
|
return (3, 32, 32)
|
|
|
|
|
|
|
|
@property
|
|
|
|
def output_shape(self):
|
|
|
|
return (3, 32, 32)
|
|
|
|
|
|
|
|
def prepare_init_args_and_inputs_for_common(self):
|
|
|
|
init_dict = {
|
|
|
|
"ch": 64,
|
|
|
|
"ch_mult": (1,),
|
|
|
|
"embed_dim": 4,
|
|
|
|
"in_channels": 3,
|
|
|
|
"num_res_blocks": 1,
|
|
|
|
"out_ch": 3,
|
|
|
|
"resolution": 32,
|
|
|
|
"z_channels": 4,
|
2022-06-29 07:25:51 -06:00
|
|
|
"attn_resolutions": [],
|
2022-06-29 05:52:04 -06:00
|
|
|
}
|
|
|
|
inputs_dict = self.dummy_input
|
|
|
|
return init_dict, inputs_dict
|
|
|
|
|
|
|
|
def test_forward_signature(self):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def test_training(self):
|
|
|
|
pass
|
2022-06-29 07:25:51 -06:00
|
|
|
|
2022-06-29 05:52:04 -06:00
|
|
|
def test_from_pretrained_hub(self):
|
|
|
|
model, loading_info = AutoencoderKL.from_pretrained("fusing/autoencoder-kl-dummy", output_loading_info=True)
|
|
|
|
self.assertIsNotNone(model)
|
2022-07-15 11:22:28 -06:00
|
|
|
# self.assertEqual(len(loading_info["missing_keys"]), 0)
|
2022-06-29 05:52:04 -06:00
|
|
|
|
|
|
|
model.to(torch_device)
|
|
|
|
image = model(**self.dummy_input)
|
|
|
|
|
|
|
|
assert image is not None, "Make sure output is not None"
|
|
|
|
|
|
|
|
def test_output_pretrained(self):
|
|
|
|
model = AutoencoderKL.from_pretrained("fusing/autoencoder-kl-dummy")
|
|
|
|
model.eval()
|
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
if torch.cuda.is_available():
|
|
|
|
torch.cuda.manual_seed_all(0)
|
|
|
|
|
|
|
|
image = torch.randn(1, model.config.in_channels, model.config.resolution, model.config.resolution)
|
|
|
|
with torch.no_grad():
|
|
|
|
output = model(image, sample_posterior=True)
|
|
|
|
|
|
|
|
output_slice = output[0, -1, -3:, -3:].flatten()
|
|
|
|
# fmt: off
|
2022-06-30 08:54:31 -06:00
|
|
|
expected_output_slice = torch.tensor([-0.0814, -0.0229, -0.1320, -0.4123, -0.0366, -0.3473, 0.0438, -0.1662, 0.1750])
|
2022-06-29 05:52:04 -06:00
|
|
|
# fmt: on
|
2022-06-30 16:29:18 -06:00
|
|
|
self.assertTrue(torch.allclose(output_slice, expected_output_slice, rtol=1e-2))
|
2022-06-29 05:52:04 -06:00
|
|
|
|
|
|
|
|
2022-06-07 07:39:47 -06:00
|
|
|
class PipelineTesterMixin(unittest.TestCase):
|
|
|
|
def test_from_pretrained_save_pretrained(self):
|
|
|
|
# 1. Load models
|
2022-07-14 11:46:04 -06:00
|
|
|
model = UNetUnconditionalModel(
|
2022-07-15 04:48:30 -06:00
|
|
|
block_channels=(32, 64),
|
|
|
|
num_res_blocks=2,
|
|
|
|
attn_resolutions=(16,),
|
|
|
|
image_size=32,
|
|
|
|
in_channels=3,
|
|
|
|
out_channels=3,
|
|
|
|
down_blocks=("UNetResDownBlock2D", "UNetResAttnDownBlock2D"),
|
|
|
|
up_blocks=("UNetResAttnUpBlock2D", "UNetResUpBlock2D"),
|
|
|
|
ddpm=True,
|
2022-07-14 11:46:04 -06:00
|
|
|
)
|
2022-06-13 02:39:53 -06:00
|
|
|
schedular = DDPMScheduler(timesteps=10)
|
2022-06-07 07:39:47 -06:00
|
|
|
|
2022-06-22 07:40:08 -06:00
|
|
|
ddpm = DDPMPipeline(model, schedular)
|
2022-06-07 07:39:47 -06:00
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdirname:
|
|
|
|
ddpm.save_pretrained(tmpdirname)
|
2022-06-22 07:40:08 -06:00
|
|
|
new_ddpm = DDPMPipeline.from_pretrained(tmpdirname)
|
2022-06-07 10:20:14 -06:00
|
|
|
|
|
|
|
generator = torch.manual_seed(0)
|
2022-06-07 07:39:47 -06:00
|
|
|
|
2022-06-07 07:43:08 -06:00
|
|
|
image = ddpm(generator=generator)
|
2022-06-07 10:20:14 -06:00
|
|
|
generator = generator.manual_seed(0)
|
2022-06-07 07:43:08 -06:00
|
|
|
new_image = new_ddpm(generator=generator)
|
2022-06-07 07:39:47 -06:00
|
|
|
|
|
|
|
assert (image - new_image).abs().sum() < 1e-5, "Models don't give the same forward pass"
|
|
|
|
|
|
|
|
@slow
|
|
|
|
def test_from_pretrained_hub(self):
|
|
|
|
model_path = "fusing/ddpm-cifar10"
|
|
|
|
|
2022-06-22 07:40:08 -06:00
|
|
|
ddpm = DDPMPipeline.from_pretrained(model_path)
|
2022-06-07 07:39:47 -06:00
|
|
|
ddpm_from_hub = DiffusionPipeline.from_pretrained(model_path)
|
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
ddpm.scheduler.num_timesteps = 10
|
|
|
|
ddpm_from_hub.scheduler.num_timesteps = 10
|
2022-06-07 07:39:47 -06:00
|
|
|
|
2022-06-07 10:20:14 -06:00
|
|
|
generator = torch.manual_seed(0)
|
2022-06-07 07:39:47 -06:00
|
|
|
|
2022-06-07 07:43:08 -06:00
|
|
|
image = ddpm(generator=generator)
|
2022-06-07 10:20:14 -06:00
|
|
|
generator = generator.manual_seed(0)
|
2022-06-07 07:43:08 -06:00
|
|
|
new_image = ddpm_from_hub(generator=generator)
|
2022-06-07 07:39:47 -06:00
|
|
|
|
|
|
|
assert (image - new_image).abs().sum() < 1e-5, "Models don't give the same forward pass"
|
2022-06-08 03:42:31 -06:00
|
|
|
|
|
|
|
@slow
|
|
|
|
def test_ddpm_cifar10(self):
|
|
|
|
model_id = "fusing/ddpm-cifar10"
|
|
|
|
|
2022-07-14 11:46:04 -06:00
|
|
|
unet = UNetUnconditionalModel.from_pretrained(model_id, ddpm=True)
|
2022-07-15 07:04:01 -06:00
|
|
|
scheduler = DDPMScheduler.from_config(model_id)
|
|
|
|
scheduler = scheduler.set_format("pt")
|
2022-06-10 05:12:23 -06:00
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
ddpm = DDPMPipeline(unet=unet, scheduler=scheduler)
|
2022-06-28 11:36:56 -06:00
|
|
|
|
|
|
|
generator = torch.manual_seed(0)
|
2022-06-08 03:42:31 -06:00
|
|
|
image = ddpm(generator=generator)
|
|
|
|
|
|
|
|
image_slice = image[0, -1, -3:, -3:].cpu()
|
|
|
|
|
|
|
|
assert image.shape == (1, 3, 32, 32)
|
2022-06-28 16:59:21 -06:00
|
|
|
expected_slice = torch.tensor(
|
2022-07-14 11:46:04 -06:00
|
|
|
[-0.1601, -0.2823, -0.6123, -0.2305, -0.3236, -0.4706, -0.1691, -0.2836, -0.3231]
|
|
|
|
)
|
|
|
|
assert (image_slice.flatten() - expected_slice).abs().max() < 1e-2
|
|
|
|
|
|
|
|
@slow
|
|
|
|
def test_ddim_lsun(self):
|
|
|
|
model_id = "fusing/ddpm-lsun-bedroom-ema"
|
|
|
|
|
|
|
|
unet = UNetUnconditionalModel.from_pretrained(model_id, ddpm=True)
|
2022-07-15 07:04:01 -06:00
|
|
|
scheduler = DDIMScheduler.from_config(model_id)
|
2022-07-14 11:46:04 -06:00
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
ddpm = DDIMPipeline(unet=unet, scheduler=scheduler)
|
2022-07-14 11:46:04 -06:00
|
|
|
|
|
|
|
generator = torch.manual_seed(0)
|
2022-07-15 07:04:01 -06:00
|
|
|
image = ddpm(generator=generator)["sample"]
|
2022-07-14 11:46:04 -06:00
|
|
|
|
|
|
|
image_slice = image[0, -1, -3:, -3:].cpu()
|
|
|
|
|
|
|
|
assert image.shape == (1, 3, 256, 256)
|
|
|
|
expected_slice = torch.tensor(
|
|
|
|
[-0.9879, -0.9598, -0.9312, -0.9953, -0.9963, -0.9995, -0.9957, -1.0000, -0.9863]
|
2022-06-28 16:59:21 -06:00
|
|
|
)
|
2022-06-08 03:42:31 -06:00
|
|
|
assert (image_slice.flatten() - expected_slice).abs().max() < 1e-2
|
|
|
|
|
|
|
|
@slow
|
|
|
|
def test_ddim_cifar10(self):
|
|
|
|
model_id = "fusing/ddpm-cifar10"
|
|
|
|
|
2022-07-14 11:46:04 -06:00
|
|
|
unet = UNetUnconditionalModel.from_pretrained(model_id, ddpm=True)
|
2022-07-15 07:04:01 -06:00
|
|
|
scheduler = DDIMScheduler(tensor_format="pt")
|
2022-06-10 05:12:23 -06:00
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
ddim = DDIMPipeline(unet=unet, scheduler=scheduler)
|
2022-06-28 11:31:44 -06:00
|
|
|
|
|
|
|
generator = torch.manual_seed(0)
|
2022-07-15 07:04:01 -06:00
|
|
|
image = ddim(generator=generator, eta=0.0)["sample"]
|
2022-06-08 03:42:31 -06:00
|
|
|
|
|
|
|
image_slice = image[0, -1, -3:, -3:].cpu()
|
|
|
|
|
|
|
|
assert image.shape == (1, 3, 32, 32)
|
2022-06-09 06:06:58 -06:00
|
|
|
expected_slice = torch.tensor(
|
2022-06-28 11:31:44 -06:00
|
|
|
[-0.6553, -0.6765, -0.6799, -0.6749, -0.7006, -0.6974, -0.6991, -0.7116, -0.7094]
|
2022-06-09 06:06:58 -06:00
|
|
|
)
|
2022-06-08 03:42:31 -06:00
|
|
|
assert (image_slice.flatten() - expected_slice).abs().max() < 1e-2
|
2022-06-10 10:37:45 -06:00
|
|
|
|
2022-06-13 10:29:22 -06:00
|
|
|
@slow
|
|
|
|
def test_pndm_cifar10(self):
|
|
|
|
model_id = "fusing/ddpm-cifar10"
|
|
|
|
|
2022-07-14 11:46:04 -06:00
|
|
|
unet = UNetUnconditionalModel.from_pretrained(model_id, ddpm=True)
|
2022-07-15 07:04:01 -06:00
|
|
|
scheduler = PNDMScheduler(tensor_format="pt")
|
2022-06-13 10:29:22 -06:00
|
|
|
|
2022-07-15 07:04:01 -06:00
|
|
|
pndm = PNDMPipeline(unet=unet, scheduler=scheduler)
|
2022-06-28 11:36:56 -06:00
|
|
|
generator = torch.manual_seed(0)
|
2022-06-13 10:29:22 -06:00
|
|
|
image = pndm(generator=generator)
|
|
|
|
|
|
|
|
image_slice = image[0, -1, -3:, -3:].cpu()
|
|
|
|
|
|
|
|
assert image.shape == (1, 3, 32, 32)
|
|
|
|
expected_slice = torch.tensor(
|
2022-06-28 11:36:56 -06:00
|
|
|
[-0.6872, -0.7071, -0.7188, -0.7057, -0.7515, -0.7191, -0.7377, -0.7565, -0.7500]
|
2022-06-13 10:29:22 -06:00
|
|
|
)
|
|
|
|
assert (image_slice.flatten() - expected_slice).abs().max() < 1e-2
|
|
|
|
|
2022-06-10 10:37:45 -06:00
|
|
|
@slow
|
2022-06-27 03:39:19 -06:00
|
|
|
@unittest.skip("Skipping for now as it takes too long")
|
2022-06-10 10:37:45 -06:00
|
|
|
def test_ldm_text2img(self):
|
|
|
|
model_id = "fusing/latent-diffusion-text2im-large"
|
2022-06-22 07:40:08 -06:00
|
|
|
ldm = LatentDiffusionPipeline.from_pretrained(model_id)
|
2022-06-10 10:37:45 -06:00
|
|
|
|
|
|
|
prompt = "A painting of a squirrel eating a burger"
|
|
|
|
generator = torch.manual_seed(0)
|
|
|
|
image = ldm([prompt], generator=generator, num_inference_steps=20)
|
|
|
|
|
|
|
|
image_slice = image[0, -1, -3:, -3:].cpu()
|
|
|
|
|
|
|
|
assert image.shape == (1, 3, 256, 256)
|
|
|
|
expected_slice = torch.tensor([0.7295, 0.7358, 0.7256, 0.7435, 0.7095, 0.6884, 0.7325, 0.6921, 0.6458])
|
2022-06-12 11:12:01 -06:00
|
|
|
assert (image_slice.flatten() - expected_slice).abs().max() < 1e-2
|
2022-06-14 04:50:40 -06:00
|
|
|
|
2022-06-27 03:42:52 -06:00
|
|
|
@slow
|
|
|
|
def test_ldm_text2img_fast(self):
|
|
|
|
model_id = "fusing/latent-diffusion-text2im-large"
|
|
|
|
ldm = LatentDiffusionPipeline.from_pretrained(model_id)
|
|
|
|
|
|
|
|
prompt = "A painting of a squirrel eating a burger"
|
|
|
|
generator = torch.manual_seed(0)
|
2022-06-27 03:46:50 -06:00
|
|
|
image = ldm([prompt], generator=generator, num_inference_steps=1)
|
2022-06-27 03:42:52 -06:00
|
|
|
|
|
|
|
image_slice = image[0, -1, -3:, -3:].cpu()
|
|
|
|
|
|
|
|
assert image.shape == (1, 3, 256, 256)
|
2022-06-27 03:44:05 -06:00
|
|
|
expected_slice = torch.tensor([0.3163, 0.8670, 0.6465, 0.1865, 0.6291, 0.5139, 0.2824, 0.3723, 0.4344])
|
2022-06-27 03:42:52 -06:00
|
|
|
assert (image_slice.flatten() - expected_slice).abs().max() < 1e-2
|
|
|
|
|
2022-06-15 03:21:02 -06:00
|
|
|
@slow
|
|
|
|
def test_glide_text2img(self):
|
|
|
|
model_id = "fusing/glide-base"
|
2022-06-22 07:40:08 -06:00
|
|
|
glide = GlidePipeline.from_pretrained(model_id)
|
2022-06-15 03:21:02 -06:00
|
|
|
|
|
|
|
prompt = "a pencil sketch of a corgi"
|
|
|
|
generator = torch.manual_seed(0)
|
|
|
|
image = glide(prompt, generator=generator, num_inference_steps_upscale=20)
|
|
|
|
|
|
|
|
image_slice = image[0, :3, :3, -1].cpu()
|
|
|
|
|
|
|
|
assert image.shape == (1, 256, 256, 3)
|
|
|
|
expected_slice = torch.tensor([0.7119, 0.7073, 0.6460, 0.7780, 0.7423, 0.6926, 0.7378, 0.7189, 0.7784])
|
|
|
|
assert (image_slice.flatten() - expected_slice).abs().max() < 1e-2
|
|
|
|
|
2022-06-25 12:25:43 -06:00
|
|
|
@slow
|
|
|
|
def test_score_sde_ve_pipeline(self):
|
|
|
|
model = NCSNpp.from_pretrained("fusing/ffhq_ncsnpp")
|
|
|
|
scheduler = ScoreSdeVeScheduler.from_config("fusing/ffhq_ncsnpp")
|
|
|
|
|
|
|
|
sde_ve = ScoreSdeVePipeline(model=model, scheduler=scheduler)
|
|
|
|
|
2022-06-28 11:31:44 -06:00
|
|
|
torch.manual_seed(0)
|
2022-06-25 12:25:43 -06:00
|
|
|
image = sde_ve(num_inference_steps=2)
|
|
|
|
|
2022-06-28 11:31:44 -06:00
|
|
|
expected_image_sum = 3382849024.0
|
|
|
|
expected_image_mean = 1075.3788
|
2022-06-25 12:25:43 -06:00
|
|
|
|
|
|
|
assert (image.abs().sum() - expected_image_sum).abs().cpu().item() < 1e-2
|
|
|
|
assert (image.abs().mean() - expected_image_mean).abs().cpu().item() < 1e-4
|
|
|
|
|
2022-06-26 17:41:55 -06:00
|
|
|
@slow
|
|
|
|
def test_score_sde_vp_pipeline(self):
|
2022-06-26 18:07:57 -06:00
|
|
|
model = NCSNpp.from_pretrained("fusing/cifar10-ddpmpp-vp")
|
|
|
|
scheduler = ScoreSdeVpScheduler.from_config("fusing/cifar10-ddpmpp-vp")
|
2022-06-26 17:41:55 -06:00
|
|
|
|
|
|
|
sde_vp = ScoreSdeVpPipeline(model=model, scheduler=scheduler)
|
|
|
|
|
|
|
|
torch.manual_seed(0)
|
|
|
|
image = sde_vp(num_inference_steps=10)
|
|
|
|
|
|
|
|
expected_image_sum = 4183.2012
|
|
|
|
expected_image_mean = 1.3617
|
|
|
|
|
|
|
|
assert (image.abs().sum() - expected_image_sum).abs().cpu().item() < 1e-2
|
|
|
|
assert (image.abs().mean() - expected_image_mean).abs().cpu().item() < 1e-4
|
|
|
|
|
2022-06-29 07:25:51 -06:00
|
|
|
@slow
|
|
|
|
def test_ldm_uncond(self):
|
2022-07-15 04:48:30 -06:00
|
|
|
ldm = LatentDiffusionUncondPipeline.from_pretrained("fusing/latent-diffusion-celeba-256", ldm=True)
|
2022-06-29 07:25:51 -06:00
|
|
|
|
|
|
|
generator = torch.manual_seed(0)
|
2022-07-15 07:04:01 -06:00
|
|
|
image = ldm(generator=generator, num_inference_steps=5)["sample"]
|
2022-06-29 07:25:51 -06:00
|
|
|
|
|
|
|
image_slice = image[0, -1, -3:, -3:].cpu()
|
|
|
|
|
|
|
|
assert image.shape == (1, 3, 256, 256)
|
2022-07-01 09:19:26 -06:00
|
|
|
expected_slice = torch.tensor(
|
|
|
|
[-0.1202, -0.1005, -0.0635, -0.0520, -0.1282, -0.0838, -0.0981, -0.1318, -0.1106]
|
|
|
|
)
|
2022-06-29 07:25:51 -06:00
|
|
|
assert (image_slice.flatten() - expected_slice).abs().max() < 1e-2
|