2022-09-30 14:28:37 -06:00
import sys
import PIL . Image
import numpy as np
import torch
2023-04-19 01:35:50 -06:00
from tqdm import tqdm
2022-09-30 14:28:37 -06:00
import modules . upscaler
2023-05-31 10:56:37 -06:00
from modules import devices , modelloader , script_callbacks , errors
2023-05-29 01:38:51 -06:00
from scunet_model_arch import SCUNet
2023-05-28 23:54:13 -06:00
2023-05-29 00:34:26 -06:00
from modules . modelloader import load_file_from_url
2023-04-19 01:35:50 -06:00
from modules . shared import opts
2022-09-30 14:28:37 -06:00
class UpscalerScuNET ( modules . upscaler . Upscaler ) :
def __init__ ( self , dirname ) :
self . name = " ScuNET "
self . model_name = " ScuNET GAN "
self . model_name2 = " ScuNET PSNR "
self . model_url = " https://github.com/cszn/KAIR/releases/download/v1.0/scunet_color_real_gan.pth "
self . model_url2 = " https://github.com/cszn/KAIR/releases/download/v1.0/scunet_color_real_psnr.pth "
self . user_path = dirname
super ( ) . __init__ ( )
model_paths = self . find_models ( ext_filter = [ " .pth " ] )
scalers = [ ]
add_model2 = True
for file in model_paths :
2023-05-29 00:41:36 -06:00
if file . startswith ( " http " ) :
2022-09-30 14:28:37 -06:00
name = self . model_name
else :
name = modelloader . friendly_name ( file )
if name == self . model_name2 or file == self . model_url2 :
add_model2 = False
try :
scaler_data = modules . upscaler . UpscalerData ( name , file , self , 4 )
scalers . append ( scaler_data )
except Exception :
2023-05-31 10:56:37 -06:00
errors . report ( f " Error loading ScuNET model: { file } " , exc_info = True )
2022-09-30 14:28:37 -06:00
if add_model2 :
scaler_data2 = modules . upscaler . UpscalerData ( self . model_name2 , self . model_url2 , self )
scalers . append ( scaler_data2 )
self . scalers = scalers
2023-04-19 01:35:50 -06:00
@staticmethod
@torch.no_grad ( )
def tiled_inference ( img , model ) :
# test the image tile by tile
h , w = img . shape [ 2 : ]
tile = opts . SCUNET_tile
tile_overlap = opts . SCUNET_tile_overlap
if tile == 0 :
return model ( img )
device = devices . get_device_for ( ' scunet ' )
assert tile % 8 == 0 , " tile size should be a multiple of window_size "
sf = 1
stride = tile - tile_overlap
h_idx_list = list ( range ( 0 , h - tile , stride ) ) + [ h - tile ]
w_idx_list = list ( range ( 0 , w - tile , stride ) ) + [ w - tile ]
E = torch . zeros ( 1 , 3 , h * sf , w * sf , dtype = img . dtype , device = device )
W = torch . zeros_like ( E , dtype = devices . dtype , device = device )
with tqdm ( total = len ( h_idx_list ) * len ( w_idx_list ) , desc = " ScuNET tiles " ) as pbar :
for h_idx in h_idx_list :
for w_idx in w_idx_list :
in_patch = img [ . . . , h_idx : h_idx + tile , w_idx : w_idx + tile ]
out_patch = model ( in_patch )
out_patch_mask = torch . ones_like ( out_patch )
E [
. . . , h_idx * sf : ( h_idx + tile ) * sf , w_idx * sf : ( w_idx + tile ) * sf
] . add_ ( out_patch )
W [
. . . , h_idx * sf : ( h_idx + tile ) * sf , w_idx * sf : ( w_idx + tile ) * sf
] . add_ ( out_patch_mask )
pbar . update ( 1 )
output = E . div_ ( W )
return output
def do_upscale ( self , img : PIL . Image . Image , selected_file ) :
2023-07-08 08:13:18 -06:00
devices . torch_gc ( )
2022-09-30 14:28:37 -06:00
2023-05-29 01:38:51 -06:00
try :
model = self . load_model ( selected_file )
except Exception as e :
print ( f " ScuNET: Unable to load model from { selected_file } : { e } " , file = sys . stderr )
2022-09-30 14:28:37 -06:00
return img
2022-12-03 08:06:33 -07:00
device = devices . get_device_for ( ' scunet ' )
2023-04-19 01:35:50 -06:00
tile = opts . SCUNET_tile
h , w = img . height , img . width
np_img = np . array ( img )
np_img = np_img [ : , : , : : - 1 ] # RGB to BGR
np_img = np_img . transpose ( ( 2 , 0 , 1 ) ) / 255 # HWC to CHW
torch_img = torch . from_numpy ( np_img ) . float ( ) . unsqueeze ( 0 ) . to ( device ) # type: ignore
if tile > h or tile > w :
_img = torch . zeros ( 1 , 3 , max ( h , tile ) , max ( w , tile ) , dtype = torch_img . dtype , device = torch_img . device )
_img [ : , : , : h , : w ] = torch_img # pad image
torch_img = _img
torch_output = self . tiled_inference ( torch_img , model ) . squeeze ( 0 )
torch_output = torch_output [ : , : h * 1 , : w * 1 ] # remove padding, if any
np_output : np . ndarray = torch_output . float ( ) . cpu ( ) . clamp_ ( 0 , 1 ) . numpy ( )
del torch_img , torch_output
2023-07-08 08:13:18 -06:00
devices . torch_gc ( )
2023-04-19 01:35:50 -06:00
output = np_output . transpose ( ( 1 , 2 , 0 ) ) # CHW to HWC
output = output [ : , : , : : - 1 ] # BGR to RGB
return PIL . Image . fromarray ( ( output * 255 ) . astype ( np . uint8 ) )
2022-09-30 14:28:37 -06:00
def load_model ( self , path : str ) :
2022-12-03 08:06:33 -07:00
device = devices . get_device_for ( ' scunet ' )
2023-05-29 00:41:36 -06:00
if path . startswith ( " http " ) :
2023-05-29 00:45:07 -06:00
# TODO: this doesn't use `path` at all?
2023-05-29 00:34:26 -06:00
filename = load_file_from_url ( self . model_url , model_dir = self . model_download_path , file_name = f " { self . name } .pth " )
2022-09-30 14:28:37 -06:00
else :
filename = path
2023-05-29 01:38:51 -06:00
model = SCUNet ( in_nc = 3 , config = [ 4 , 4 , 4 , 4 , 4 , 4 , 4 ] , dim = 64 )
2022-09-30 14:28:37 -06:00
model . load_state_dict ( torch . load ( filename ) , strict = True )
model . eval ( )
2023-05-10 02:37:18 -06:00
for _ , v in model . named_parameters ( ) :
2022-09-30 14:28:37 -06:00
v . requires_grad = False
model = model . to ( device )
return model
2023-05-14 02:04:21 -06:00
def on_ui_settings ( ) :
import gradio as gr
from modules import shared
shared . opts . add_option ( " SCUNET_tile " , shared . OptionInfo ( 256 , " Tile size for SCUNET upscalers. " , gr . Slider , { " minimum " : 0 , " maximum " : 512 , " step " : 16 } , section = ( ' upscaling ' , " Upscaling " ) ) . info ( " 0 = no tiling " ) )
shared . opts . add_option ( " SCUNET_tile_overlap " , shared . OptionInfo ( 8 , " Tile overlap for SCUNET upscalers. " , gr . Slider , { " minimum " : 0 , " maximum " : 64 , " step " : 1 } , section = ( ' upscaling ' , " Upscaling " ) ) . info ( " Low values = visible seam " ) )
script_callbacks . on_ui_settings ( on_ui_settings )