ensure dataloader workers exit cleanly on ctrl-c

This commit is contained in:
Damian Stewart 2023-01-30 14:19:18 +01:00
parent 4b5654452c
commit c270dbf6a8
1 changed files with 4 additions and 1 deletions

View File

@ -691,7 +691,10 @@ def main(args):
logging.error(f"{Fore.LIGHTRED_EX} ************************************************************************{Style.RESET_ALL}")
time.sleep(2) # give opportunity to ctrl-C again to cancel save
__save_model(interrupted_checkpoint_path, unet, text_encoder, tokenizer, noise_scheduler, vae, args.save_ckpt_dir, args.save_full_precision)
exit(_SIGTERM_EXIT_CODE)
exit(_SIGTERM_EXIT_CODE)
else:
# non-main threads (i.e. dataloader workers) should exit cleanly
exit(0)
signal.signal(signal.SIGINT, sigterm_handler)