exit from chain pipeline when retries are exhausted
This commit is contained in:
parent
7761aea28b
commit
4f0c6d292d
|
@ -167,7 +167,7 @@ class ChainPipeline:
|
|||
if is_debug():
|
||||
save_image(server, "last-tile.png", output_tile)
|
||||
|
||||
job.retries = job.retries - i
|
||||
job.retries = job.retries - (i + 1)
|
||||
return output_tile
|
||||
except Exception:
|
||||
logger.exception(
|
||||
|
@ -176,6 +176,8 @@ class ChainPipeline:
|
|||
)
|
||||
run_gc([job.get_device()])
|
||||
|
||||
raise RuntimeError("exhausted retries on tile")
|
||||
|
||||
output = process_tile_order(
|
||||
stage_params.tile_order,
|
||||
source,
|
||||
|
@ -203,7 +205,7 @@ class ChainPipeline:
|
|||
# doing this on the same line as stage_pipe.run can leave sources as None, which the pipeline
|
||||
# does not like, so it throws
|
||||
stage_sources = stage_outputs
|
||||
job.retries = job.retries - i
|
||||
job.retries = job.retries - (i + 1)
|
||||
break
|
||||
except Exception:
|
||||
logger.exception(
|
||||
|
@ -211,6 +213,9 @@ class ChainPipeline:
|
|||
)
|
||||
run_gc([job.get_device()])
|
||||
|
||||
if job.retries <= 0:
|
||||
raise RuntimeError("exhausted retries on stage")
|
||||
|
||||
logger.debug(
|
||||
"finished stage %s with %s results",
|
||||
name,
|
||||
|
|
Loading…
Reference in New Issue