apextoaster
/
onnx-web
1
0
Fork 0
Code Actions

fix(api): watch for progress events from leaking workers

This commit is contained in:
Sean Sube 2023-03-26 18:22:16 -05:00
parent e1219cca90
commit 4ddd69ba07
Signed by: ssube
GPG Key ID: 3EED7B957D362AF1
1 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
api/onnx_web/worker/pool.py
View File

@ -277,7 +277,7 @@ class DevicePoolExecutor:
def join_leaking(self): def join_leaking(self):
if len(self.leaking) > 0: if len(self.leaking) > 0:
for device, worker, _context in self.leaking: for device, worker, context in self.leaking:
logger.warning( logger.warning(
"shutting down leaking worker %s for device %s", worker.pid, device "shutting down leaking worker %s for device %s", worker.pid, device
) )
@ -289,6 +289,21 @@ class DevicePoolExecutor:
device, device,
) )
try:
progress = context.progress.get_nowait()
while progress is not None:
self.update_job(progress)
progress = context.progress.get_nowait()
except Empty:
logger.trace("empty queue in leaking worker for device %s", device)
except ValueError as e:
logger.debug("value error in leaking worker for device %s: %s", device, e)
break
except Exception:
logger.exception("error in leaking worker for device %s", device)
self.leaking[:] = [dw for dw in self.leaking if dw[1].is_alive()] self.leaking[:] = [dw for dw in self.leaking if dw[1].is_alive()]
def recycle(self): def recycle(self):