From 95a61f3573ff621262daea890c476044bc679c1c Mon Sep 17 00:00:00 2001 From: Sean Sube Date: Sat, 25 Mar 2023 13:46:12 -0500 Subject: [PATCH] fix(api): restart worker threads when their respective queues are full --- api/onnx_web/worker/pool.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/onnx_web/worker/pool.py b/api/onnx_web/worker/pool.py index af99611b..c465b1fd 100644 --- a/api/onnx_web/worker/pool.py +++ b/api/onnx_web/worker/pool.py @@ -320,12 +320,20 @@ class DevicePoolExecutor: if self.threads["logger"].is_alive(): logger.debug("logger worker is running") + if self.logs.full(): + logger.warning("logger queue is full, restarting worker") + self.threads["logger"].join(self.join_timeout) + self.create_logger_worker() else: logger.warning("restarting logger worker") self.create_logger_worker() if self.threads["progress"].is_alive(): logger.debug("progress worker is running") + if self.progress.full(): + logger.warning("progress queue is full, restarting worker") + self.threads["progress"].join(self.join_timeout) + self.create_progress_worker() else: logger.warning("restarting progress worker") self.create_progress_worker()