fix(api): watch for progress events from leaking workers
This commit is contained in:
parent
e1219cca90
commit
4ddd69ba07
|
@ -277,7 +277,7 @@ class DevicePoolExecutor:
|
||||||
|
|
||||||
def join_leaking(self):
|
def join_leaking(self):
|
||||||
if len(self.leaking) > 0:
|
if len(self.leaking) > 0:
|
||||||
for device, worker, _context in self.leaking:
|
for device, worker, context in self.leaking:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"shutting down leaking worker %s for device %s", worker.pid, device
|
"shutting down leaking worker %s for device %s", worker.pid, device
|
||||||
)
|
)
|
||||||
|
@ -289,6 +289,21 @@ class DevicePoolExecutor:
|
||||||
device,
|
device,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
progress = context.progress.get_nowait()
|
||||||
|
while progress is not None:
|
||||||
|
self.update_job(progress)
|
||||||
|
progress = context.progress.get_nowait()
|
||||||
|
except Empty:
|
||||||
|
logger.trace("empty queue in leaking worker for device %s", device)
|
||||||
|
except ValueError as e:
|
||||||
|
logger.debug("value error in leaking worker for device %s: %s", device, e)
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
logger.exception("error in leaking worker for device %s", device)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
self.leaking[:] = [dw for dw in self.leaking if dw[1].is_alive()]
|
self.leaking[:] = [dw for dw in self.leaking if dw[1].is_alive()]
|
||||||
|
|
||||||
def recycle(self):
|
def recycle(self):
|
||||||
|
|
Loading…
Reference in New Issue