diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml new file mode 100644 index 000000000..49dc92bd9 --- /dev/null +++ b/.github/workflows/run_tests.yaml @@ -0,0 +1,31 @@ +name: Run basic features tests on CPU with empty SD model + +on: + - push + - pull_request + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: 3.10.6 + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: ${{ runner.os }}-pip- + - name: Run tests + run: python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test + - name: Upload main app stdout-stderr + uses: actions/upload-artifact@v3 + if: always() + with: + name: stdout-stderr + path: | + test/stdout.txt + test/stderr.txt diff --git a/launch.py b/launch.py index b1626cb5c..c426e6739 100644 --- a/launch.py +++ b/launch.py @@ -17,6 +17,19 @@ def extract_arg(args, name): return [x for x in args if x != name], name in args +def extract_opt(args, name): + opt = None + is_present = False + if name in args: + is_present = True + idx = args.index(name) + del args[idx] + if idx < len(args) and args[idx][0] != "-": + opt = args[idx] + del args[idx] + return args, is_present, opt + + def run(command, desc=None, errdesc=None, custom_env=None): if desc is not None: print(desc) @@ -151,12 +164,11 @@ def prepare_enviroment(): blip_commit_hash = os.environ.get('BLIP_COMMIT_HASH', "48211a1594f1321b00f14c9f7a5b4813144b2fb9") sys.argv += shlex.split(commandline_args) - test_argv = [x for x in sys.argv if x != '--tests'] sys.argv, skip_torch_cuda_test = extract_arg(sys.argv, '--skip-torch-cuda-test') sys.argv, reinstall_xformers = extract_arg(sys.argv, '--reinstall-xformers') sys.argv, update_check = extract_arg(sys.argv, '--update-check') - sys.argv, run_tests = extract_arg(sys.argv, '--tests') + sys.argv, run_tests, test_dir = extract_opt(sys.argv, '--tests') xformers = '--xformers' in sys.argv ngrok = '--ngrok' in sys.argv @@ -221,24 +233,30 @@ def prepare_enviroment(): exit(0) if run_tests: - tests(test_argv) - exit(0) + exitcode = tests(test_dir) + exit(exitcode) -def tests(argv): - if "--api" not in argv: - argv.append("--api") +def tests(test_dir): + if "--api" not in sys.argv: + sys.argv.append("--api") + if "--ckpt" not in sys.argv: + sys.argv.append("--ckpt") + sys.argv.append("./test/test_files/empty.pt") + if "--skip-torch-cuda-test" not in sys.argv: + sys.argv.append("--skip-torch-cuda-test") - print(f"Launching Web UI in another process for testing with arguments: {' '.join(argv[1:])}") + print(f"Launching Web UI in another process for testing with arguments: {' '.join(sys.argv[1:])}") with open('test/stdout.txt', "w", encoding="utf8") as stdout, open('test/stderr.txt', "w", encoding="utf8") as stderr: - proc = subprocess.Popen([sys.executable, *argv], stdout=stdout, stderr=stderr) + proc = subprocess.Popen([sys.executable, *sys.argv], stdout=stdout, stderr=stderr) import test.server_poll - test.server_poll.run_tests() + exitcode = test.server_poll.run_tests(proc, test_dir) print(f"Stopping Web UI process with id {proc.pid}") proc.kill() + return exitcode def start(): diff --git a/test/advanced_features/__init__.py b/test/advanced_features/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/extras_test.py b/test/advanced_features/extras_test.py similarity index 90% rename from test/extras_test.py rename to test/advanced_features/extras_test.py index 9b8ce0f03..8763f8ed1 100644 --- a/test/extras_test.py +++ b/test/advanced_features/extras_test.py @@ -11,8 +11,8 @@ class TestExtrasWorking(unittest.TestCase): "codeformer_visibility": 0, "codeformer_weight": 0, "upscaling_resize": 2, - "upscaling_resize_w": 512, - "upscaling_resize_h": 512, + "upscaling_resize_w": 128, + "upscaling_resize_h": 128, "upscaling_crop": True, "upscaler_1": "None", "upscaler_2": "None", diff --git a/test/advanced_features/txt2img_test.py b/test/advanced_features/txt2img_test.py new file mode 100644 index 000000000..36ed7b9a9 --- /dev/null +++ b/test/advanced_features/txt2img_test.py @@ -0,0 +1,47 @@ +import unittest +import requests + + +class TestTxt2ImgWorking(unittest.TestCase): + def setUp(self): + self.url_txt2img = "http://localhost:7860/sdapi/v1/txt2img" + self.simple_txt2img = { + "enable_hr": False, + "denoising_strength": 0, + "firstphase_width": 0, + "firstphase_height": 0, + "prompt": "example prompt", + "styles": [], + "seed": -1, + "subseed": -1, + "subseed_strength": 0, + "seed_resize_from_h": -1, + "seed_resize_from_w": -1, + "batch_size": 1, + "n_iter": 1, + "steps": 3, + "cfg_scale": 7, + "width": 64, + "height": 64, + "restore_faces": False, + "tiling": False, + "negative_prompt": "", + "eta": 0, + "s_churn": 0, + "s_tmax": 0, + "s_tmin": 0, + "s_noise": 1, + "sampler_index": "Euler a" + } + + def test_txt2img_with_restore_faces_performed(self): + self.simple_txt2img["restore_faces"] = True + self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200) + + +class TestTxt2ImgCorrectness(unittest.TestCase): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/test/basic_features/__init__.py b/test/basic_features/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/img2img_test.py b/test/basic_features/img2img_test.py similarity index 96% rename from test/img2img_test.py rename to test/basic_features/img2img_test.py index 012a95809..0a9c1e8ad 100644 --- a/test/img2img_test.py +++ b/test/basic_features/img2img_test.py @@ -51,9 +51,5 @@ class TestImg2ImgWorking(unittest.TestCase): self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200) -class TestImg2ImgCorrectness(unittest.TestCase): - pass - - if __name__ == "__main__": unittest.main() diff --git a/test/txt2img_test.py b/test/basic_features/txt2img_test.py similarity index 91% rename from test/txt2img_test.py rename to test/basic_features/txt2img_test.py index 1936e07e2..1c2674b2a 100644 --- a/test/txt2img_test.py +++ b/test/basic_features/txt2img_test.py @@ -49,26 +49,20 @@ class TestTxt2ImgWorking(unittest.TestCase): self.simple_txt2img["enable_hr"] = True self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200) - def test_txt2img_with_restore_faces_performed(self): - self.simple_txt2img["restore_faces"] = True - self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200) - - def test_txt2img_with_tiling_faces_performed(self): + def test_txt2img_with_tiling_performed(self): self.simple_txt2img["tiling"] = True self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200) def test_txt2img_with_vanilla_sampler_performed(self): self.simple_txt2img["sampler_index"] = "PLMS" self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200) + self.simple_txt2img["sampler_index"] = "DDIM" + self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200) def test_txt2img_multiple_batches_performed(self): self.simple_txt2img["n_iter"] = 2 self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200) -class TestTxt2ImgCorrectness(unittest.TestCase): - pass - - if __name__ == "__main__": unittest.main() diff --git a/test/utils_test.py b/test/basic_features/utils_test.py similarity index 79% rename from test/utils_test.py rename to test/basic_features/utils_test.py index 65d3d177d..765470c90 100644 --- a/test/utils_test.py +++ b/test/basic_features/utils_test.py @@ -18,20 +18,6 @@ class UtilsTests(unittest.TestCase): def test_options_get(self): self.assertEqual(requests.get(self.url_options).status_code, 200) - def test_options_write(self): - response = requests.get(self.url_options) - self.assertEqual(response.status_code, 200) - - pre_value = response.json()["send_seed"] - - self.assertEqual(requests.post(self.url_options, json={"send_seed":not pre_value}).status_code, 200) - - response = requests.get(self.url_options) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["send_seed"], not pre_value) - - requests.post(self.url_options, json={"send_seed": pre_value}) - def test_cmd_flags(self): self.assertEqual(requests.get(self.url_cmd_flags).status_code, 200) @@ -60,4 +46,8 @@ class UtilsTests(unittest.TestCase): self.assertEqual(requests.get(self.url_artist_categories).status_code, 200) def test_artists(self): - self.assertEqual(requests.get(self.url_artists).status_code, 200) \ No newline at end of file + self.assertEqual(requests.get(self.url_artists).status_code, 200) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/server_poll.py b/test/server_poll.py index eeefb7ebc..d4df697b2 100644 --- a/test/server_poll.py +++ b/test/server_poll.py @@ -3,7 +3,7 @@ import requests import time -def run_tests(): +def run_tests(proc, test_dir): timeout_threshold = 240 start_time = time.time() while time.time()-start_time < timeout_threshold: @@ -11,9 +11,14 @@ def run_tests(): requests.head("http://localhost:7860/") break except requests.exceptions.ConnectionError: - pass - if time.time()-start_time < timeout_threshold: - suite = unittest.TestLoader().discover('', pattern='*_test.py') + if proc.poll() is not None: + break + if proc.poll() is None: + if test_dir is None: + test_dir = "" + suite = unittest.TestLoader().discover(test_dir, pattern="*_test.py", top_level_dir="test") result = unittest.TextTestRunner(verbosity=2).run(suite) + return len(result.failures) + len(result.errors) else: print("Launch unsuccessful") + return 1 diff --git a/test/test_files/empty.pt b/test/test_files/empty.pt new file mode 100644 index 000000000..c6ac59eb0 Binary files /dev/null and b/test/test_files/empty.pt differ