only remove retries when an error occurs
This commit is contained in:
parent
4f0c6d292d
commit
822dfeab01
|
@ -167,7 +167,6 @@ class ChainPipeline:
|
||||||
if is_debug():
|
if is_debug():
|
||||||
save_image(server, "last-tile.png", output_tile)
|
save_image(server, "last-tile.png", output_tile)
|
||||||
|
|
||||||
job.retries = job.retries - (i + 1)
|
|
||||||
return output_tile
|
return output_tile
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
|
@ -175,6 +174,7 @@ class ChainPipeline:
|
||||||
i,
|
i,
|
||||||
)
|
)
|
||||||
run_gc([job.get_device()])
|
run_gc([job.get_device()])
|
||||||
|
job.retries = job.retries - (i + 1)
|
||||||
|
|
||||||
raise RuntimeError("exhausted retries on tile")
|
raise RuntimeError("exhausted retries on tile")
|
||||||
|
|
||||||
|
@ -205,13 +205,13 @@ class ChainPipeline:
|
||||||
# doing this on the same line as stage_pipe.run can leave sources as None, which the pipeline
|
# doing this on the same line as stage_pipe.run can leave sources as None, which the pipeline
|
||||||
# does not like, so it throws
|
# does not like, so it throws
|
||||||
stage_sources = stage_outputs
|
stage_sources = stage_outputs
|
||||||
job.retries = job.retries - (i + 1)
|
|
||||||
break
|
break
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"error while running stage pipeline, retry %s of 3", i
|
"error while running stage pipeline, retry %s of 3", i
|
||||||
)
|
)
|
||||||
run_gc([job.get_device()])
|
run_gc([job.get_device()])
|
||||||
|
job.retries = job.retries - (i + 1)
|
||||||
|
|
||||||
if job.retries <= 0:
|
if job.retries <= 0:
|
||||||
raise RuntimeError("exhausted retries on stage")
|
raise RuntimeError("exhausted retries on stage")
|
||||||
|
|
Loading…
Reference in New Issue