From 3d0f2434af830e38a46e13423c4274891595550a Mon Sep 17 00:00:00 2001 From: Sean Sube Date: Wed, 8 Mar 2023 23:17:54 -0600 Subject: [PATCH] fix(tests): make release tests more strict, update DEIS reference --- .../test-refs/txt2img-sd-v1-5-512-muffin-deis-0.png | 4 ++-- api/scripts/test-release.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-deis-0.png b/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-deis-0.png index c3b23071..84d7abfd 100644 --- a/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-deis-0.png +++ b/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-deis-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a543d18a572d24c983f30fbae535409cc743932fdb20abcdd765aa04436193d8 -size 432524 +oid sha256:ce7ac5d1989ba5997b23d15adbcfd2f4565da47d67e746f2d1f75c780d090f8c +size 498915 diff --git a/api/scripts/test-release.py b/api/scripts/test-release.py index cb832a61..f0e522e7 100644 --- a/api/scripts/test-release.py +++ b/api/scripts/test-release.py @@ -46,7 +46,7 @@ class TestCase: name: str, query: str, max_attempts: int = FAST_TEST, - mse_threshold: float = 0.001, + mse_threshold: float = 1e-4, source: Union[Image.Image, List[Image.Image]] = None, mask: Image.Image = None, ) -> None: @@ -86,6 +86,7 @@ TEST_DATA = [ TestCase( "txt2img-sd-v2-1-768-muffin", "txt2img?prompt=a+giant+muffin&seed=0&scheduler=ddim&model=stable-diffusion-onnx-v2-1&width=768&height=768", + max_attempts=SLOW_TEST, ), TestCase( "txt2img-openjourney-512-muffin", @@ -312,9 +313,9 @@ def run_test( mse = find_mse(result, ref) if mse < test.mse_threshold: - logger.info("MSE within threshold: %.4f < %.4f", mse, test.mse_threshold) + logger.info("MSE within threshold: %.5f < %.5f", mse, test.mse_threshold) else: - logger.warning("MSE above threshold: %.4f > %.4f", mse, test.mse_threshold) + logger.warning("MSE above threshold: %.5f > %.5f", mse, test.mse_threshold) passed = False return passed