apextoaster
/
onnx-web
1
0
Fork 0
Code Actions

fix(api): attempt to recycle leaking workers when a job finishes

This commit is contained in:
Sean Sube 2023-03-05 20:13:28 -06:00
parent 3a4928e59b
commit 4ae4ce176c
Signed by: ssube
GPG Key ID: 3EED7B957D362AF1
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
api/onnx_web/worker/pool.py
View File

@ -171,6 +171,7 @@ class DevicePoolExecutor:
_device, progress = self.active_jobs[job] _device, progress = self.active_jobs[job]
self.finished_jobs.append((job, progress, context.cancel.value)) self.finished_jobs.append((job, progress, context.cancel.value))
del self.active_jobs[job] del self.active_jobs[job]
self.join_leaking()
except Empty: except Empty:
pass pass
except ValueError: except ValueError:
@ -262,6 +263,7 @@ class DevicePoolExecutor:
queue.close() queue.close()
self.pending.clear() self.pending.clear()
self.join_leaking()
logger.debug("stopping device workers") logger.debug("stopping device workers")
for device, worker in self.workers.items(): for device, worker in self.workers.items():
@ -282,9 +284,7 @@ class DevicePoolExecutor:
logger.debug("worker pool stopped") logger.debug("worker pool stopped")
def recycle(self): def join_leaking(self):
logger.debug("recycling worker pool")
if len(self.leaking) > 0: if len(self.leaking) > 0:
logger.warning("cleaning up %s leaking workers", len(self.leaking)) logger.warning("cleaning up %s leaking workers", len(self.leaking))
for device, worker in self.leaking: for device, worker in self.leaking:
@ -297,6 +297,10 @@ class DevicePoolExecutor:
self.leaking[:] = [dw for dw in self.leaking if dw[1].is_alive()] self.leaking[:] = [dw for dw in self.leaking if dw[1].is_alive()]
def recycle(self):
logger.debug("recycling worker pool")
self.join_leaking()
needs_restart = [] needs_restart = []
for device, worker in self.workers.items(): for device, worker in self.workers.items():