From 74cb1002365163b918cc046ee67742ca052e56cc Mon Sep 17 00:00:00 2001 From: Sean Sube Date: Thu, 30 Nov 2023 19:21:05 -0600 Subject: [PATCH] fix(scripts): update test MSE and references for recent tiling changes --- .../test-refs/blend-512-muffin-white-0.png | 4 +-- api/scripts/test-refs/outpaint-even-256-0.png | 4 +-- .../test-refs/outpaint-horizontal-512-0.png | 4 +-- .../test-refs/outpaint-vertical-512-0.png | 4 +-- .../txt2img-sd-v1-5-512-muffin-dpm-0.png | 4 +-- .../txt2img-sd-v1-5-512-muffin-heun-0.png | 4 +-- .../txt2img-sd-v2-1-512-muffin-0.png | 4 +-- .../test-refs/upscale-sd-x4-2048-muffin-0.png | 4 +-- ...upscale-sd-x4-codeformer-2048-muffin-0.png | 4 +-- .../upscale-sd-x4-gfpgan-2048-muffin-0.png | 4 +-- api/scripts/test-release.py | 31 +++++++++++++------ 11 files changed, 41 insertions(+), 30 deletions(-) diff --git a/api/scripts/test-refs/blend-512-muffin-white-0.png b/api/scripts/test-refs/blend-512-muffin-white-0.png index 103f582c..f218fb5a 100644 --- a/api/scripts/test-refs/blend-512-muffin-white-0.png +++ b/api/scripts/test-refs/blend-512-muffin-white-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:be25e2a6252de2cd830c6421f75313067bf3ef29904138536615899c7169ac57 -size 573669 +oid sha256:14833dae2dafa6eb2fe9184087a4ba0781b2be881e2fdedb5dca09baf1843799 +size 572108 diff --git a/api/scripts/test-refs/outpaint-even-256-0.png b/api/scripts/test-refs/outpaint-even-256-0.png index fdc925e8..c69f41b5 100644 --- a/api/scripts/test-refs/outpaint-even-256-0.png +++ b/api/scripts/test-refs/outpaint-even-256-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d7d666f57ef942b5b7706b93df8831c06a6872287b44e012bb5a18959d38a63 -size 2018986 +oid sha256:60f019055396a71d387f9c575acac1802b639cdf8ae95ea0287278fa68f051ac +size 1968972 diff --git a/api/scripts/test-refs/outpaint-horizontal-512-0.png b/api/scripts/test-refs/outpaint-horizontal-512-0.png index e6669885..311c20fa 100644 --- a/api/scripts/test-refs/outpaint-horizontal-512-0.png +++ b/api/scripts/test-refs/outpaint-horizontal-512-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0679d0198ec0f321a83c214a1fa178c64aa1f9139d2ba560dac76b653d94fc01 -size 1580841 +oid sha256:b1d1b3417ce63553a15c80e601cbccde7b2ef4af05eef624cfc0d82d5e6a8b35 +size 1579738 diff --git a/api/scripts/test-refs/outpaint-vertical-512-0.png b/api/scripts/test-refs/outpaint-vertical-512-0.png index 2933eb8e..d5f52320 100644 --- a/api/scripts/test-refs/outpaint-vertical-512-0.png +++ b/api/scripts/test-refs/outpaint-vertical-512-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8caa4f99dc1520a5d30d563090aa6cc02db89c0abb3eec7f8c3f8750c9dbf20f -size 1514972 +oid sha256:e59324850e153e768c7f849026fb0753440906a54223d3583f0b18caec6c8ff0 +size 1523624 diff --git a/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-dpm-0.png b/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-dpm-0.png index 05c93b7d..7919d7d7 100644 --- a/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-dpm-0.png +++ b/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-dpm-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d4f5506f9f3836dc3383f695c533b1ff2873f60c76b65a37f0850e2231a29495 -size 522478 +oid sha256:93def1c9b1355ed33d25916df16f037a11ba4ba3ee6bcd487d58818371fc7ad5 +size 526093 diff --git a/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-heun-0.png b/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-heun-0.png index dca24a75..00408050 100644 --- a/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-heun-0.png +++ b/api/scripts/test-refs/txt2img-sd-v1-5-512-muffin-heun-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ce2c5e5510c93d5d4e2e7688cd132e1c6e167e7715173e1250e86f341ac47172 -size 503943 +oid sha256:bb9043d076c9084fb3f74fa5f58ec734c781ef403ae9723be261dff27e191f5e +size 494092 diff --git a/api/scripts/test-refs/txt2img-sd-v2-1-512-muffin-0.png b/api/scripts/test-refs/txt2img-sd-v2-1-512-muffin-0.png index 7b1a789a..59c02b8a 100644 --- a/api/scripts/test-refs/txt2img-sd-v2-1-512-muffin-0.png +++ b/api/scripts/test-refs/txt2img-sd-v2-1-512-muffin-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5dee66e2a5b003c6e892e2f5e7b7ba95c471ee301cdad34e11199083544a46d8 -size 502780 +oid sha256:582669cbcc215d32728c18166a46312c583462a7f438ff69108e4854b6c5edf7 +size 501499 diff --git a/api/scripts/test-refs/upscale-sd-x4-2048-muffin-0.png b/api/scripts/test-refs/upscale-sd-x4-2048-muffin-0.png index 0b713be1..7e52eeb7 100644 --- a/api/scripts/test-refs/upscale-sd-x4-2048-muffin-0.png +++ b/api/scripts/test-refs/upscale-sd-x4-2048-muffin-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1a982c32e952f9fe1d9f0ba1b93f177f04f11b23603736ddc157ece2d45d915e -size 7313660 +oid sha256:30c2845fc375cd965a574c88f1da9e9c6b1d679514e5f3c4578a5d962726d4dd +size 6498264 diff --git a/api/scripts/test-refs/upscale-sd-x4-codeformer-2048-muffin-0.png b/api/scripts/test-refs/upscale-sd-x4-codeformer-2048-muffin-0.png index c37edc5e..a6f36771 100644 --- a/api/scripts/test-refs/upscale-sd-x4-codeformer-2048-muffin-0.png +++ b/api/scripts/test-refs/upscale-sd-x4-codeformer-2048-muffin-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a7c381c34884e1c1304fd7f7a7b631c6ab4ba3694ececd99c3b5f43cba9ff5b6 -size 7313667 +oid sha256:9153a0b9b12a0c9c7309b9023477d74fc824e3820cb19aaadaf99cb80be72ace +size 6711175 diff --git a/api/scripts/test-refs/upscale-sd-x4-gfpgan-2048-muffin-0.png b/api/scripts/test-refs/upscale-sd-x4-gfpgan-2048-muffin-0.png index cda047d7..a6f36771 100644 --- a/api/scripts/test-refs/upscale-sd-x4-gfpgan-2048-muffin-0.png +++ b/api/scripts/test-refs/upscale-sd-x4-gfpgan-2048-muffin-0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f3f34f029ca3b7324b2f7ff851e5af86454cd02c0b7445676655bfd486b8c2c0 -size 7313651 +oid sha256:9153a0b9b12a0c9c7309b9023477d74fc824e3820cb19aaadaf99cb80be72ace +size 6711175 diff --git a/api/scripts/test-release.py b/api/scripts/test-release.py index fbd5a6b8..dd8443d5 100644 --- a/api/scripts/test-release.py +++ b/api/scripts/test-release.py @@ -30,6 +30,10 @@ FAST_TEST = 10 SLOW_TEST = 25 VERY_SLOW_TEST = 75 +STRICT_TEST = 1e-4 +LOOSE_TEST = 1e-2 +VERY_LOOSE_TEST = 0.025 + def test_path(relpath: str) -> str: return path.join(path.dirname(__file__), relpath) @@ -41,7 +45,7 @@ class TestCase: name: str, query: str, max_attempts: int = FAST_TEST, - mse_threshold: float = 1e-4, + mse_threshold: float = STRICT_TEST, source: Union[Image.Image, List[Image.Image]] = None, mask: Image.Image = None, ) -> None: @@ -73,10 +77,12 @@ TEST_DATA = [ TestCase( "txt2img-sd-v1-5-512-muffin-heun", "txt2img?prompt=a+giant+muffin&seed=0&scheduler=heun", + mse_threshold=LOOSE_TEST, ), TestCase( "txt2img-sd-v1-5-512-muffin-unipc", "txt2img?prompt=a+giant+muffin&seed=0&scheduler=unipc-multi", + mse_threshold=LOOSE_TEST, ), TestCase( "txt2img-sd-v2-1-512-muffin", @@ -84,7 +90,7 @@ TEST_DATA = [ ), TestCase( "txt2img-sd-v2-1-768-muffin", - "txt2img?prompt=a+giant+muffin&seed=0&scheduler=ddim&model=stable-diffusion-onnx-v2-1&width=768&height=768", + "txt2img?prompt=a+giant+muffin&seed=0&scheduler=ddim&model=stable-diffusion-onnx-v2-1&width=768&height=768&unet_tile=768", max_attempts=SLOW_TEST, ), TestCase( @@ -106,7 +112,7 @@ TEST_DATA = [ ), TestCase( "img2img-sd-v1-5-256-pumpkin", - "img2img?prompt=a+giant+pumpkin&seed=0&scheduler=ddim&sourceFilter=none", + "img2img?prompt=a+giant+pumpkin&seed=0&scheduler=ddim&sourceFilter=none&unet_tile=256", source="txt2img-sd-v1-5-256-muffin-0", ), TestCase( @@ -130,7 +136,7 @@ TEST_DATA = [ source="txt2img-sd-v1-5-512-muffin-0", mask="mask-black", max_attempts=SLOW_TEST, - mse_threshold=0.025, + mse_threshold=VERY_LOOSE_TEST, ), TestCase( "outpaint-vertical-512", @@ -141,7 +147,7 @@ TEST_DATA = [ source="txt2img-sd-v1-5-512-muffin-0", mask="mask-black", max_attempts=SLOW_TEST, - mse_threshold=0.010, + mse_threshold=LOOSE_TEST, ), TestCase( "outpaint-horizontal-512", @@ -152,7 +158,7 @@ TEST_DATA = [ source="txt2img-sd-v1-5-512-muffin-0", mask="mask-black", max_attempts=SLOW_TEST, - mse_threshold=0.010, + mse_threshold=LOOSE_TEST, ), TestCase( "upscale-resrgan-x2-1024-muffin", @@ -229,7 +235,7 @@ TEST_DATA = [ source="txt2img-sd-v1-5-512-muffin-0", mask="mask-black", max_attempts=VERY_SLOW_TEST, - mse_threshold=0.025, + mse_threshold=VERY_LOOSE_TEST, ), TestCase( "outpaint-panorama-vertical-512", @@ -240,7 +246,7 @@ TEST_DATA = [ source="txt2img-sd-v1-5-512-muffin-0", mask="mask-black", max_attempts=VERY_SLOW_TEST, - mse_threshold=0.025, + mse_threshold=VERY_LOOSE_TEST, ), TestCase( "outpaint-panorama-horizontal-512", @@ -251,7 +257,7 @@ TEST_DATA = [ source="txt2img-sd-v1-5-512-muffin-0", mask="mask-black", max_attempts=VERY_SLOW_TEST, - mse_threshold=0.025, + mse_threshold=VERY_LOOSE_TEST, ), TestCase( "upscale-resrgan-x4-codeformer-2048-muffin", @@ -260,6 +266,7 @@ TEST_DATA = [ "&correction=correction-codeformer&faces=true&faceOutscale=1&faceStrength=1.0" ), source="txt2img-sd-v1-5-512-muffin-0", + max_attempts=SLOW_TEST, ), TestCase( "upscale-resrgan-x4-gfpgan-2048-muffin", @@ -268,6 +275,7 @@ TEST_DATA = [ "&correction=correction-gfpgan&faces=true&faceOutscale=1&faceStrength=1.0" ), source="txt2img-sd-v1-5-512-muffin-0", + max_attempts=SLOW_TEST, ), TestCase( "upscale-swinir-x4-codeformer-2048-muffin", @@ -276,6 +284,7 @@ TEST_DATA = [ "&correction=correction-codeformer&faces=true&faceOutscale=1&faceStrength=1.0" ), source="txt2img-sd-v1-5-512-muffin-0", + max_attempts=SLOW_TEST, ), TestCase( "upscale-swinir-x4-gfpgan-2048-muffin", @@ -284,6 +293,7 @@ TEST_DATA = [ "&correction=correction-gfpgan&faces=true&faceOutscale=1&faceStrength=1.0" ), source="txt2img-sd-v1-5-512-muffin-0", + max_attempts=SLOW_TEST, ), TestCase( "upscale-sd-x4-codeformer-2048-muffin", @@ -316,7 +326,7 @@ TEST_DATA = [ ), TestCase( "txt2img-sd-v1-5-tall-muffin", - "txt2img?prompt=a+giant+muffin&seed=0&scheduler=ddim&width=512&height=768", + "txt2img?prompt=a+giant+muffin&seed=0&scheduler=ddim&width=512&height=768&unet_tile=768", ), TestCase( "upscale-resrgan-x4-tall-muffin", @@ -325,6 +335,7 @@ TEST_DATA = [ "&correction=correction-gfpgan&faces=false&faceOutscale=1&faceStrength=1.0" ), source="txt2img-sd-v1-5-tall-muffin-0", + max_attempts=SLOW_TEST, ), # TODO: non-square controlnet ]