2023-01-28 18:42:22 +00:00
|
|
|
from io import BytesIO
|
2024-01-15 02:03:16 +00:00
|
|
|
from json import dumps
|
2023-01-28 23:09:19 +00:00
|
|
|
from logging import getLogger
|
2024-01-15 02:03:16 +00:00
|
|
|
from typing import Optional
|
2023-02-05 13:53:26 +00:00
|
|
|
|
|
|
|
from boto3 import Session
|
2023-01-28 18:42:22 +00:00
|
|
|
from PIL import Image
|
|
|
|
|
2024-01-15 02:03:16 +00:00
|
|
|
from ..output import make_output_names
|
2023-02-05 13:53:26 +00:00
|
|
|
from ..params import ImageParams, StageParams
|
2023-02-26 05:49:39 +00:00
|
|
|
from ..server import ServerContext
|
2023-02-26 20:15:30 +00:00
|
|
|
from ..worker import WorkerContext
|
2023-11-18 23:18:23 +00:00
|
|
|
from .base import BaseStage
|
2023-11-19 00:08:38 +00:00
|
|
|
from .result import StageResult
|
2023-01-28 18:42:22 +00:00
|
|
|
|
2023-01-28 23:09:19 +00:00
|
|
|
logger = getLogger(__name__)
|
|
|
|
|
2023-01-28 18:42:22 +00:00
|
|
|
|
2023-07-02 23:21:21 +00:00
|
|
|
class PersistS3Stage(BaseStage):
|
2023-07-01 12:10:53 +00:00
|
|
|
def run(
|
|
|
|
self,
|
2024-01-15 02:03:16 +00:00
|
|
|
worker: WorkerContext,
|
2023-07-01 12:10:53 +00:00
|
|
|
server: ServerContext,
|
|
|
|
_stage: StageParams,
|
|
|
|
_params: ImageParams,
|
2023-11-19 00:08:38 +00:00
|
|
|
sources: StageResult,
|
2023-07-01 12:10:53 +00:00
|
|
|
*,
|
|
|
|
bucket: str,
|
|
|
|
endpoint_url: Optional[str] = None,
|
|
|
|
profile_name: Optional[str] = None,
|
|
|
|
stage_source: Optional[Image.Image] = None,
|
|
|
|
**kwargs,
|
2023-11-19 00:08:38 +00:00
|
|
|
) -> StageResult:
|
2023-07-01 12:10:53 +00:00
|
|
|
session = Session(profile_name=profile_name)
|
|
|
|
s3 = session.client("s3", endpoint_url=endpoint_url)
|
|
|
|
|
2024-01-15 02:03:16 +00:00
|
|
|
image_names = make_output_names(server, worker.job, len(sources))
|
|
|
|
for source, name in zip(sources.as_images(), image_names):
|
2023-07-04 18:29:58 +00:00
|
|
|
data = BytesIO()
|
|
|
|
source.save(data, format=server.image_format)
|
|
|
|
data.seek(0)
|
2023-07-01 12:10:53 +00:00
|
|
|
|
2023-07-04 18:29:58 +00:00
|
|
|
try:
|
2023-11-28 03:45:32 +00:00
|
|
|
s3.upload_fileobj(data, bucket, name)
|
|
|
|
logger.info("saved image to s3://%s/%s", bucket, name)
|
2023-07-04 18:29:58 +00:00
|
|
|
except Exception:
|
|
|
|
logger.exception("error saving image to S3")
|
2023-07-01 12:10:53 +00:00
|
|
|
|
2024-01-15 02:03:16 +00:00
|
|
|
metadata_names = make_output_names(
|
|
|
|
server, worker.job, len(sources), extension="json"
|
|
|
|
)
|
|
|
|
for metadata, name in zip(sources.metadata, metadata_names):
|
|
|
|
data = BytesIO()
|
|
|
|
data.write(dumps(metadata.tojson(server, [name])))
|
|
|
|
data.seek(0)
|
|
|
|
|
|
|
|
try:
|
|
|
|
s3.upload_fileobj(data, bucket, name)
|
|
|
|
logger.info("saved metadata to s3://%s/%s", bucket, name)
|
|
|
|
except Exception:
|
|
|
|
logger.exception("error saving metadata to S3")
|
|
|
|
|
2023-07-04 18:29:58 +00:00
|
|
|
return sources
|