apextoaster
/
onnx-web
1
0
Fork 0
Code Actions

exit from chain pipeline when retries are exhausted

This commit is contained in:
Sean Sube 2023-07-15 12:18:25 -05:00
parent 7761aea28b
commit 4f0c6d292d
Signed by: ssube
GPG Key ID: 3EED7B957D362AF1
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/onnx_web/chain/base.py
View File

@ -167,7 +167,7 @@ class ChainPipeline:
if is_debug():
save_image(server, "last-tile.png", output_tile)
job.retries = job.retries - i
job.retries = job.retries - (i + 1)
return output_tile
except Exception:
logger.exception(
@ -176,6 +176,8 @@ class ChainPipeline:
)
run_gc([job.get_device()])
raise RuntimeError("exhausted retries on tile")
output = process_tile_order(
stage_params.tile_order,
source,
@ -203,7 +205,7 @@ class ChainPipeline:
# doing this on the same line as stage_pipe.run can leave sources as None, which the pipeline
# does not like, so it throws
stage_sources = stage_outputs
job.retries = job.retries - i
job.retries = job.retries - (i + 1)
break
except Exception:
logger.exception(
@ -211,6 +213,9 @@ class ChainPipeline:
)
run_gc([job.get_device()])
if job.retries <= 0:
raise RuntimeError("exhausted retries on stage")
logger.debug(
"finished stage %s with %s results",
name,