apextoaster
/
onnx-web
1
0
Fork 0
Code Actions

fix(api): add worker PID to log messages

This commit is contained in:
Sean Sube 2023-03-05 20:25:02 -06:00
parent 4ae4ce176c
commit cb460a0c59
Signed by: ssube
GPG Key ID: 3EED7B957D362AF1
2 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
api/onnx_web/worker/pool.py
View File

@ -268,18 +268,18 @@ class DevicePoolExecutor:
logger.debug("stopping device workers") logger.debug("stopping device workers")
for device, worker in self.workers.items(): for device, worker in self.workers.items():
if worker.is_alive(): if worker.is_alive():
logger.debug("stopping worker for device %s", device) logger.debug("stopping worker %s for device %s", worker.pid, device)
worker.join(self.join_timeout) worker.join(self.join_timeout)
if worker.is_alive(): if worker.is_alive():
logger.warning( logger.warning(
"worker for device %s could not be stopped in time", device "worker %s for device %s could not be stopped in time", worker.pid, device
) )
self.leaking.append((device, worker)) self.leaking.append((device, worker))
else: else:
logger.debug("worker for device %s has died", device) logger.debug("worker for device %s has died", device)
for name, thread in self.threads.items(): for name, thread in self.threads.items():
logger.debug("stopping worker thread: %s", name) logger.debug("stopping worker %s for thread %s", thread.ident, name)
thread.join(self.join_timeout) thread.join(self.join_timeout)
logger.debug("worker pool stopped") logger.debug("worker pool stopped")
@ -288,11 +288,11 @@ class DevicePoolExecutor:
if len(self.leaking) > 0: if len(self.leaking) > 0:
logger.warning("cleaning up %s leaking workers", len(self.leaking)) logger.warning("cleaning up %s leaking workers", len(self.leaking))
for device, worker in self.leaking: for device, worker in self.leaking:
logger.debug("shutting down worker for device %s", device) logger.debug("shutting down worker %s for device %s", worker.pid, device)
worker.join(self.join_timeout) worker.join(self.join_timeout)
if worker.is_alive(): if worker.is_alive():
logger.error( logger.error(
"leaking worker for device %s could not be shut down", device "leaking worker %s for device %s could not be shut down", worker.pid, device
) )
self.leaking[:] = [dw for dw in self.leaking if dw[1].is_alive()] self.leaking[:] = [dw for dw in self.leaking if dw[1].is_alive()]
@ -315,7 +315,7 @@ class DevicePoolExecutor:
worker.join(self.join_timeout) worker.join(self.join_timeout)
if worker.is_alive(): if worker.is_alive():
logger.warning( logger.warning(
"worker for device %s could not be recycled in time", device "worker %s for device %s could not be recycled in time", worker.pid, device
) )
self.leaking.append((device, worker)) self.leaking.append((device, worker))
else: else:
@ -325,7 +325,7 @@ class DevicePoolExecutor:
needs_restart.append(device) needs_restart.append(device)
else: else:
logger.debug( logger.debug(
"worker for device %s does not need to be recycled", device "worker %s for device %s does not need to be recycled", worker.pid, device
) )
logger.debug("starting new workers") logger.debug("starting new workers")

4
api/scripts/test-memory-leak.sh
View File

@ -4,7 +4,7 @@ test_images=0
while true; while true;
do do
curl "${test_host}/api/txt2img?"\ curl "${test_host}/api/txt2img?"\
'cfg=16.00&steps=3&scheduler=ddim&seed=-1&'\ 'cfg=16.00&steps=30&scheduler=ddim&seed=-1&'\
'prompt=an+astronaut+eating+a+hamburger&negativePrompt=&'\ 'prompt=an+astronaut+eating+a+hamburger&negativePrompt=&'\
'model=stable-diffusion-onnx-v1-5&platform=any&'\ 'model=stable-diffusion-onnx-v1-5&platform=any&'\
'upscaling=upscaling-real-esrgan-x2-plus&correction=correction-codeformer&'\ 'upscaling=upscaling-real-esrgan-x2-plus&correction=correction-codeformer&'\
@ -14,5 +14,5 @@ do
--insecure || break; --insecure || break;
((test_images++)); ((test_images++));
echo "waiting after $test_images"; echo "waiting after $test_images";
sleep 30; sleep 3;
done done