1
0
Fork 0

clear model cache on stage errors

This commit is contained in:
Sean Sube 2023-07-15 12:33:07 -05:00
parent 822dfeab01
commit c7e1fce557
Signed by: ssube
GPG Key ID: 3EED7B957D362AF1
1 changed files with 2 additions and 0 deletions

View File

@ -173,6 +173,7 @@ class ChainPipeline:
"error while running stage pipeline for tile, retry %s of 3",
i,
)
server.cache.clear()
run_gc([job.get_device()])
job.retries = job.retries - (i + 1)
@ -210,6 +211,7 @@ class ChainPipeline:
logger.exception(
"error while running stage pipeline, retry %s of 3", i
)
server.cache.clear()
run_gc([job.get_device()])
job.retries = job.retries - (i + 1)