From fc00133ee301a01231ca4037981183d9026ecab1 Mon Sep 17 00:00:00 2001 From: "fox.cpp" Date: Sat, 28 Aug 2021 15:24:35 +0300 Subject: [PATCH] storage/blob/s3: Force a smaller PartSize when blob size is unknown Blob size would be unknown ahead of time if message store compression is used (e.g. in imapsql). Part of #395 fix --- internal/storage/blob/s3/s3.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/storage/blob/s3/s3.go b/internal/storage/blob/s3/s3.go index d9ead7d..87cf859 100644 --- a/internal/storage/blob/s3/s3.go +++ b/internal/storage/blob/s3/s3.go @@ -116,7 +116,16 @@ func (s *Store) Create(ctx context.Context, key string, blobSize int64) (module. errCh := make(chan error, 1) go func() { - _, err := s.cl.PutObject(ctx, s.bucketName, s.objectPrefix+key, pr, blobSize, minio.PutObjectOptions{}) + partSize := uint64(0) + if blobSize == module.UnknownBlobSize { + // Without this, minio-go will allocate 500 MiB buffer which + // is a little too much. + // https://github.com/minio/minio-go/issues/1478 + partSize = 1 * 1024 * 1024 /* 1 MiB */ + } + _, err := s.cl.PutObject(ctx, s.bucketName, s.objectPrefix+key, pr, blobSize, minio.PutObjectOptions{ + PartSize: partSize, + }) if err != nil { pr.CloseWithError(fmt.Errorf("s3 PutObject: %w", err)) }