storage/blob: Implement S3-compatible storage support

Closes #304.

docs/man/maddy-blob.5.scd

@@ -39,3 +39,75 @@ storage.blob.fs <directory>
 Path to the FS directory. Must be readable and writable by the server process.
 If it does not exist - it will be created (parent directory should be writable
 for this). Relative paths are interpreted relatively to server state directory.
+
+# Amazon S3 storage (storage.blob.s3)
+
+This modules stores messages bodies in a bucket on S3-compatible storage.
+
+```
+storage.blob.s3 {
+    endpoint play.min.io
+    secure yes
+    access_key "Q3AM3UQ867SPQQA43P2F"
+    secret_key "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
+    bucket maddy-test
+
+    # optional
+    region eu-central-1
+    object_prefix maddy/
+}
+```
+
+Example:
+```
+storage.imapsql local_mailboxes {
+    ...
+    msg_store s3 {
+        endpoint s3.amazonaws.com
+        access_key "..."
+        secret_key "..."
+        bucket maddy-messages
+        region us-west-2
+    }
+}
+```
+
+## Configuration directives
+
+*Syntax:* endpoint _address:port_
+
+REQUIRED.
+
+Root S3 endpoint. e.g. s3.amazonaws.com
+
+*Syntax:* secure _boolean_ ++
+*Default:* yes
+
+Whether TLS should be used.
+
+*Syntax:* access_key _string_ ++
+*Syntax:* secret_key _string_
+
+REQUIRED.
+
+Static S3 credentials.
+
+*Syntax:* bucket _name_
+
+REQUIRED.
+
+S3 bucket name. The bucket must exist and
+be read-writable.
+
+*Syntax:* region _string_ ++
+*Default:* not set
+
+S3 bucket location. May be called "endpoint"
+in some manuals.
+
+*Syntax:* object_prefix _string_ ++
+*Default:* empty string
+
+String to add to all keys stored by maddy.
+
+Can be useful when S3 is used as a file system.

framework/module/blob_store.go

@@ -7,7 +7,6 @@ import (
 
 type Blob interface {
 	Sync() error
-	io.Reader
 	io.Writer
 	io.Closer
 }

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/go-ldap/ldap/v3 v3.3.0
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/google/uuid v1.2.0
+	github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c
 	github.com/klauspost/compress v1.11.13 // indirect
 	github.com/lib/pq v1.10.0
 	github.com/libdns/alidns v1.0.2
@@ -47,6 +48,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
 	github.com/miekg/dns v1.1.42
+	github.com/minio/minio-go/v7 v7.0.12
 	github.com/pierrec/lz4 v2.6.0+incompatible // indirect
 	github.com/prometheus/client_golang v1.10.0
 	github.com/prometheus/common v0.20.0 // indirect

go.sum

@@ -64,6 +64,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
+github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.30.27 h1:9gPjZWVDSoQrBO2AvqrWObS6KAZByfEJxQoCYo4ZfK0=
 github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
@@ -332,6 +333,8 @@ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c h1:lx/uPI+mUWlqEQ9e6CtNvaK/zD64s/mQ9+yMh16PgY0=
+github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -518,8 +521,12 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
+github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0=
+github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -530,6 +537,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
+github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
@@ -558,6 +566,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -647,6 +656,7 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190310074541-c10a0554eabf/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -791,6 +801,7 @@ golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -977,6 +988,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=

internal/storage/blob/s3/s3.go

@@ -0,0 +1,144 @@
+package s3
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/foxcpp/maddy/framework/config"
+	"github.com/foxcpp/maddy/framework/log"
+	"github.com/foxcpp/maddy/framework/module"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+const modName = "storage.blob.s3"
+
+type Store struct {
+	instName string
+	log      log.Logger
+
+	endpoint string
+	cl       *minio.Client
+
+	bucketName   string
+	objectPrefix string
+}
+
+func New(_, instName string, _, inlineArgs []string) (module.Module, error) {
+	if len(inlineArgs) != 0 {
+		return nil, fmt.Errorf("%s: expected 0 arguments", modName)
+	}
+
+	return &Store{
+		instName: instName,
+		log:      log.Logger{Name: modName},
+	}, nil
+}
+
+func (s *Store) Init(cfg *config.Map) error {
+	var (
+		secure          bool
+		accessKeyID     string
+		secretAccessKey string
+		location        string
+	)
+	cfg.String("endpoint", false, true, "", &s.endpoint)
+	cfg.Bool("secure", false, true, &secure)
+	cfg.String("access_key", false, true, "", &accessKeyID)
+	cfg.String("secret_key", false, true, "", &secretAccessKey)
+	cfg.String("bucket", false, true, "", &s.bucketName)
+	cfg.String("region", false, false, "", &location)
+	cfg.String("object_prefix", false, false, "", &s.objectPrefix)
+
+	if _, err := cfg.Process(); err != nil {
+		return err
+	}
+	if s.endpoint == "" {
+		return fmt.Errorf("%s: endpoint not set", modName)
+	}
+
+	cl, err := minio.New(s.endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: secure,
+		Region: location,
+	})
+	if err != nil {
+		return fmt.Errorf("%s: %w", modName, err)
+	}
+
+	s.cl = cl
+	return nil
+}
+
+func (s *Store) Name() string {
+	return modName
+}
+
+func (s *Store) InstanceName() string {
+	return s.instName
+}
+
+type s3blob struct {
+	pw      *io.PipeWriter
+	didSync bool
+	errCh   chan error
+}
+
+func (b *s3blob) Sync() error {
+	// We do this in Sync instead of Close because
+	// backend may not actually check the error of Close.
+
+	// The problematic restriction is that Sync can now be called
+	// only once.
+	b.pw.Close()
+	return <-b.errCh
+}
+
+func (b *s3blob) Write(p []byte) (n int, err error) {
+	return b.pw.Write(p)
+}
+
+func (b *s3blob) Close() error {
+	return nil
+}
+
+func (s *Store) Create(key string) (module.Blob, error) {
+	pr, pw := io.Pipe()
+	errCh := make(chan error, 1)
+
+	go func() {
+		_, err := s.cl.PutObject(context.TODO(), s.bucketName, s.objectPrefix+key, pr, -1, minio.PutObjectOptions{})
+		errCh <- err
+	}()
+
+	return &s3blob{pw: pw, errCh: errCh}, nil
+}
+
+func (s *Store) Open(key string) (io.ReadCloser, error) {
+	obj, err := s.cl.GetObject(context.TODO(), s.bucketName, s.objectPrefix+key, minio.GetObjectOptions{})
+	if err != nil {
+		resp := minio.ToErrorResponse(err)
+		if resp.StatusCode == http.StatusNotFound {
+			return nil, module.ErrNoSuchBlob
+		}
+		return nil, err
+	}
+	return obj, nil
+}
+
+func (s *Store) Delete(keys []string) error {
+	var lastErr error
+	for _, k := range keys {
+		lastErr = s.cl.RemoveObject(context.TODO(), s.bucketName, s.objectPrefix+k, minio.RemoveObjectOptions{})
+		if lastErr != nil {
+			s.log.Error("failed to delete object", lastErr, s.objectPrefix+k)
+		}
+	}
+	return lastErr
+}
+
+func init() {
+	module.Register(modName, New)
+}

internal/storage/blob/s3/s3_test.go

@@ -0,0 +1,71 @@
+package s3
+
+import (
+	"net/http/httptest"
+	"testing"
+
+	"github.com/foxcpp/maddy/framework/config"
+	"github.com/foxcpp/maddy/framework/module"
+	"github.com/foxcpp/maddy/internal/storage/blob"
+	"github.com/johannesboyne/gofakes3"
+	"github.com/johannesboyne/gofakes3/backend/s3mem"
+)
+
+func TestFS(t *testing.T) {
+	var (
+		backend gofakes3.Backend
+		faker   *gofakes3.GoFakeS3
+		ts      *httptest.Server
+	)
+
+	blob.TestStore(t, func() module.BlobStore {
+		backend = s3mem.New()
+		faker = gofakes3.New(backend)
+		ts = httptest.NewServer(faker.Server())
+
+		if err := backend.CreateBucket("maddy-test"); err != nil {
+			panic(err)
+		}
+
+		st := &Store{instName: "test"}
+		err := st.Init(config.NewMap(map[string]interface{}{}, config.Node{
+			Children: []config.Node{
+				{
+					Name: "endpoint",
+					Args: []string{ts.Listener.Addr().String()},
+				},
+				{
+					Name: "secure",
+					Args: []string{"false"},
+				},
+				{
+					Name: "access_key",
+					Args: []string{"access-key"},
+				},
+				{
+					Name: "secret_key",
+					Args: []string{"secret-key"},
+				},
+				{
+					Name: "bucket",
+					Args: []string{"maddy-test"},
+				},
+			},
+		}))
+		if err != nil {
+			panic(err)
+		}
+
+		return st
+	}, func(store module.BlobStore) {
+		ts.Close()
+
+		backend = s3mem.New()
+		faker = gofakes3.New(backend)
+		ts = httptest.NewServer(faker.Server())
+	})
+
+	if ts != nil {
+		ts.Close()
+	}
+}

internal/storage/imapsql/external_blob_store.go

@@ -19,6 +19,14 @@ func (e ExtBlob) Write(p []byte) (n int, err error) {
 	panic("not implemented")
 }
 
+type WriteExtBlob struct {
+	module.Blob
+}
+
+func (w WriteExtBlob) Read(p []byte) (n int, err error) {
+	panic("not implemented")
+}
+
 type ExtBlobStore struct {
 	Base module.BlobStore
 }
@@ -32,7 +40,7 @@ func (e ExtBlobStore) Create(key string) (imapsql.ExtStoreObj, error) {
 			Err:         err,
 		}
 	}
-	return blob, nil
+	return WriteExtBlob{Blob: blob}, nil
 }
 
 func (e ExtBlobStore) Open(key string) (imapsql.ExtStoreObj, error) {

maddy.go

@@ -65,6 +65,7 @@ import (
 	_ "github.com/foxcpp/maddy/internal/modify"
 	_ "github.com/foxcpp/maddy/internal/modify/dkim"
 	_ "github.com/foxcpp/maddy/internal/storage/blob/fs"
+	_ "github.com/foxcpp/maddy/internal/storage/blob/s3"
 	_ "github.com/foxcpp/maddy/internal/storage/imapsql"
 	_ "github.com/foxcpp/maddy/internal/table"
 	_ "github.com/foxcpp/maddy/internal/target/queue"