From 4dd68ea6b64113431910d157e7b735908fab57a7 Mon Sep 17 00:00:00 2001 From: Sean Sube Date: Wed, 22 Mar 2023 19:58:46 -0500 Subject: [PATCH] fix(api): restart worker threads if they crash --- api/onnx_web/worker/pool.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/onnx_web/worker/pool.py b/api/onnx_web/worker/pool.py index b704767f..ae386893 100644 --- a/api/onnx_web/worker/pool.py +++ b/api/onnx_web/worker/pool.py @@ -333,6 +333,14 @@ class DevicePoolExecutor: self.create_device_worker(device) self.total_jobs[device.device] = 0 + if not self.threads["logger"].is_alive(): + logger.warning("restarting crashed logger worker") + self.create_logger_worker() + + if not self.threads["progress"].is_alive(): + logger.warning("restarting crashed progress worker") + self.create_progress_worker() + logger.debug("worker pool recycled") def submit(