diff --git a/api/onnx_web/worker/pool.py b/api/onnx_web/worker/pool.py index 7df861c3..907658df 100644 --- a/api/onnx_web/worker/pool.py +++ b/api/onnx_web/worker/pool.py @@ -268,18 +268,18 @@ class DevicePoolExecutor: logger.debug("stopping device workers") for device, worker in self.workers.items(): if worker.is_alive(): - logger.debug("stopping worker for device %s", device) + logger.debug("stopping worker %s for device %s", worker.pid, device) worker.join(self.join_timeout) if worker.is_alive(): logger.warning( - "worker for device %s could not be stopped in time", device + "worker %s for device %s could not be stopped in time", worker.pid, device ) self.leaking.append((device, worker)) else: logger.debug("worker for device %s has died", device) for name, thread in self.threads.items(): - logger.debug("stopping worker thread: %s", name) + logger.debug("stopping worker %s for thread %s", thread.ident, name) thread.join(self.join_timeout) logger.debug("worker pool stopped") @@ -288,11 +288,11 @@ class DevicePoolExecutor: if len(self.leaking) > 0: logger.warning("cleaning up %s leaking workers", len(self.leaking)) for device, worker in self.leaking: - logger.debug("shutting down worker for device %s", device) + logger.debug("shutting down worker %s for device %s", worker.pid, device) worker.join(self.join_timeout) if worker.is_alive(): logger.error( - "leaking worker for device %s could not be shut down", device + "leaking worker %s for device %s could not be shut down", worker.pid, device ) self.leaking[:] = [dw for dw in self.leaking if dw[1].is_alive()] @@ -315,7 +315,7 @@ class DevicePoolExecutor: worker.join(self.join_timeout) if worker.is_alive(): logger.warning( - "worker for device %s could not be recycled in time", device + "worker %s for device %s could not be recycled in time", worker.pid, device ) self.leaking.append((device, worker)) else: @@ -325,7 +325,7 @@ class DevicePoolExecutor: needs_restart.append(device) else: logger.debug( - "worker for device %s does not need to be recycled", device + "worker %s for device %s does not need to be recycled", worker.pid, device ) logger.debug("starting new workers") diff --git a/api/scripts/test-memory-leak.sh b/api/scripts/test-memory-leak.sh index e1ca5f1f..6c576519 100755 --- a/api/scripts/test-memory-leak.sh +++ b/api/scripts/test-memory-leak.sh @@ -4,7 +4,7 @@ test_images=0 while true; do curl "${test_host}/api/txt2img?"\ -'cfg=16.00&steps=3&scheduler=ddim&seed=-1&'\ +'cfg=16.00&steps=30&scheduler=ddim&seed=-1&'\ 'prompt=an+astronaut+eating+a+hamburger&negativePrompt=&'\ 'model=stable-diffusion-onnx-v1-5&platform=any&'\ 'upscaling=upscaling-real-esrgan-x2-plus&correction=correction-codeformer&'\ @@ -14,5 +14,5 @@ do --insecure || break; ((test_images++)); echo "waiting after $test_images"; - sleep 30; + sleep 3; done