From ef63383248ea08a0483e29f665bc13eb96fdefce Mon Sep 17 00:00:00 2001 From: "fox.cpp" Date: Thu, 15 Jul 2021 20:31:50 +0300 Subject: [PATCH] storage/blob: Implement S3-compatible storage support Closes #304. --- docs/man/maddy-blob.5.scd | 74 ++++++++- framework/module/blob_store.go | 1 - go.mod | 2 + go.sum | 12 ++ internal/storage/blob/s3/s3.go | 144 ++++++++++++++++++ internal/storage/blob/s3/s3_test.go | 71 +++++++++ .../storage/imapsql/external_blob_store.go | 10 +- maddy.go | 1 + 8 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 internal/storage/blob/s3/s3.go create mode 100644 internal/storage/blob/s3/s3_test.go diff --git a/docs/man/maddy-blob.5.scd b/docs/man/maddy-blob.5.scd index 712c7a4..dcc9a11 100644 --- a/docs/man/maddy-blob.5.scd +++ b/docs/man/maddy-blob.5.scd @@ -38,4 +38,76 @@ storage.blob.fs Path to the FS directory. Must be readable and writable by the server process. If it does not exist - it will be created (parent directory should be writable -for this). Relative paths are interpreted relatively to server state directory. \ No newline at end of file +for this). Relative paths are interpreted relatively to server state directory. + +# Amazon S3 storage (storage.blob.s3) + +This modules stores messages bodies in a bucket on S3-compatible storage. + +``` +storage.blob.s3 { + endpoint play.min.io + secure yes + access_key "Q3AM3UQ867SPQQA43P2F" + secret_key "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + bucket maddy-test + + # optional + region eu-central-1 + object_prefix maddy/ +} +``` + +Example: +``` +storage.imapsql local_mailboxes { + ... + msg_store s3 { + endpoint s3.amazonaws.com + access_key "..." + secret_key "..." + bucket maddy-messages + region us-west-2 + } +} +``` + +## Configuration directives + +*Syntax:* endpoint _address:port_ + +REQUIRED. + +Root S3 endpoint. e.g. s3.amazonaws.com + +*Syntax:* secure _boolean_ ++ +*Default:* yes + +Whether TLS should be used. + +*Syntax:* access_key _string_ ++ +*Syntax:* secret_key _string_ + +REQUIRED. + +Static S3 credentials. + +*Syntax:* bucket _name_ + +REQUIRED. + +S3 bucket name. The bucket must exist and +be read-writable. + +*Syntax:* region _string_ ++ +*Default:* not set + +S3 bucket location. May be called "endpoint" +in some manuals. + +*Syntax:* object_prefix _string_ ++ +*Default:* empty string + +String to add to all keys stored by maddy. + +Can be useful when S3 is used as a file system. \ No newline at end of file diff --git a/framework/module/blob_store.go b/framework/module/blob_store.go index bd30ed1..81d8bae 100644 --- a/framework/module/blob_store.go +++ b/framework/module/blob_store.go @@ -7,7 +7,6 @@ import ( type Blob interface { Sync() error - io.Reader io.Writer io.Closer } diff --git a/go.mod b/go.mod index ff92236..28db408 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/go-ldap/ldap/v3 v3.3.0 github.com/go-sql-driver/mysql v1.6.0 github.com/google/uuid v1.2.0 + github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c github.com/klauspost/compress v1.11.13 // indirect github.com/lib/pq v1.10.0 github.com/libdns/alidns v1.0.2 @@ -47,6 +48,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/miekg/dns v1.1.42 + github.com/minio/minio-go/v7 v7.0.12 github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/prometheus/client_golang v1.10.0 github.com/prometheus/common v0.20.0 // indirect diff --git a/go.sum b/go.sum index 1bf6c6d..06fd49f 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.30.27 h1:9gPjZWVDSoQrBO2AvqrWObS6KAZByfEJxQoCYo4ZfK0= github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= @@ -332,6 +333,8 @@ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c h1:lx/uPI+mUWlqEQ9e6CtNvaK/zD64s/mQ9+yMh16PgY0= +github.com/johannesboyne/gofakes3 v0.0.0-20210704111953-6a9f95c2941c/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -518,8 +521,12 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0= +github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -530,6 +537,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -558,6 +566,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -647,6 +656,7 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190310074541-c10a0554eabf/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -791,6 +801,7 @@ golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -977,6 +988,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= diff --git a/internal/storage/blob/s3/s3.go b/internal/storage/blob/s3/s3.go new file mode 100644 index 0000000..a4d23b5 --- /dev/null +++ b/internal/storage/blob/s3/s3.go @@ -0,0 +1,144 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/log" + "github.com/foxcpp/maddy/framework/module" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const modName = "storage.blob.s3" + +type Store struct { + instName string + log log.Logger + + endpoint string + cl *minio.Client + + bucketName string + objectPrefix string +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + if len(inlineArgs) != 0 { + return nil, fmt.Errorf("%s: expected 0 arguments", modName) + } + + return &Store{ + instName: instName, + log: log.Logger{Name: modName}, + }, nil +} + +func (s *Store) Init(cfg *config.Map) error { + var ( + secure bool + accessKeyID string + secretAccessKey string + location string + ) + cfg.String("endpoint", false, true, "", &s.endpoint) + cfg.Bool("secure", false, true, &secure) + cfg.String("access_key", false, true, "", &accessKeyID) + cfg.String("secret_key", false, true, "", &secretAccessKey) + cfg.String("bucket", false, true, "", &s.bucketName) + cfg.String("region", false, false, "", &location) + cfg.String("object_prefix", false, false, "", &s.objectPrefix) + + if _, err := cfg.Process(); err != nil { + return err + } + if s.endpoint == "" { + return fmt.Errorf("%s: endpoint not set", modName) + } + + cl, err := minio.New(s.endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: secure, + Region: location, + }) + if err != nil { + return fmt.Errorf("%s: %w", modName, err) + } + + s.cl = cl + return nil +} + +func (s *Store) Name() string { + return modName +} + +func (s *Store) InstanceName() string { + return s.instName +} + +type s3blob struct { + pw *io.PipeWriter + didSync bool + errCh chan error +} + +func (b *s3blob) Sync() error { + // We do this in Sync instead of Close because + // backend may not actually check the error of Close. + + // The problematic restriction is that Sync can now be called + // only once. + b.pw.Close() + return <-b.errCh +} + +func (b *s3blob) Write(p []byte) (n int, err error) { + return b.pw.Write(p) +} + +func (b *s3blob) Close() error { + return nil +} + +func (s *Store) Create(key string) (module.Blob, error) { + pr, pw := io.Pipe() + errCh := make(chan error, 1) + + go func() { + _, err := s.cl.PutObject(context.TODO(), s.bucketName, s.objectPrefix+key, pr, -1, minio.PutObjectOptions{}) + errCh <- err + }() + + return &s3blob{pw: pw, errCh: errCh}, nil +} + +func (s *Store) Open(key string) (io.ReadCloser, error) { + obj, err := s.cl.GetObject(context.TODO(), s.bucketName, s.objectPrefix+key, minio.GetObjectOptions{}) + if err != nil { + resp := minio.ToErrorResponse(err) + if resp.StatusCode == http.StatusNotFound { + return nil, module.ErrNoSuchBlob + } + return nil, err + } + return obj, nil +} + +func (s *Store) Delete(keys []string) error { + var lastErr error + for _, k := range keys { + lastErr = s.cl.RemoveObject(context.TODO(), s.bucketName, s.objectPrefix+k, minio.RemoveObjectOptions{}) + if lastErr != nil { + s.log.Error("failed to delete object", lastErr, s.objectPrefix+k) + } + } + return lastErr +} + +func init() { + module.Register(modName, New) +} diff --git a/internal/storage/blob/s3/s3_test.go b/internal/storage/blob/s3/s3_test.go new file mode 100644 index 0000000..98dd228 --- /dev/null +++ b/internal/storage/blob/s3/s3_test.go @@ -0,0 +1,71 @@ +package s3 + +import ( + "net/http/httptest" + "testing" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" + "github.com/foxcpp/maddy/internal/storage/blob" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" +) + +func TestFS(t *testing.T) { + var ( + backend gofakes3.Backend + faker *gofakes3.GoFakeS3 + ts *httptest.Server + ) + + blob.TestStore(t, func() module.BlobStore { + backend = s3mem.New() + faker = gofakes3.New(backend) + ts = httptest.NewServer(faker.Server()) + + if err := backend.CreateBucket("maddy-test"); err != nil { + panic(err) + } + + st := &Store{instName: "test"} + err := st.Init(config.NewMap(map[string]interface{}{}, config.Node{ + Children: []config.Node{ + { + Name: "endpoint", + Args: []string{ts.Listener.Addr().String()}, + }, + { + Name: "secure", + Args: []string{"false"}, + }, + { + Name: "access_key", + Args: []string{"access-key"}, + }, + { + Name: "secret_key", + Args: []string{"secret-key"}, + }, + { + Name: "bucket", + Args: []string{"maddy-test"}, + }, + }, + })) + if err != nil { + panic(err) + } + + return st + }, func(store module.BlobStore) { + ts.Close() + + backend = s3mem.New() + faker = gofakes3.New(backend) + ts = httptest.NewServer(faker.Server()) + }) + + if ts != nil { + ts.Close() + } +} diff --git a/internal/storage/imapsql/external_blob_store.go b/internal/storage/imapsql/external_blob_store.go index 743765f..b1d707b 100644 --- a/internal/storage/imapsql/external_blob_store.go +++ b/internal/storage/imapsql/external_blob_store.go @@ -19,6 +19,14 @@ func (e ExtBlob) Write(p []byte) (n int, err error) { panic("not implemented") } +type WriteExtBlob struct { + module.Blob +} + +func (w WriteExtBlob) Read(p []byte) (n int, err error) { + panic("not implemented") +} + type ExtBlobStore struct { Base module.BlobStore } @@ -32,7 +40,7 @@ func (e ExtBlobStore) Create(key string) (imapsql.ExtStoreObj, error) { Err: err, } } - return blob, nil + return WriteExtBlob{Blob: blob}, nil } func (e ExtBlobStore) Open(key string) (imapsql.ExtStoreObj, error) { diff --git a/maddy.go b/maddy.go index 181bb3a..795048e 100644 --- a/maddy.go +++ b/maddy.go @@ -65,6 +65,7 @@ import ( _ "github.com/foxcpp/maddy/internal/modify" _ "github.com/foxcpp/maddy/internal/modify/dkim" _ "github.com/foxcpp/maddy/internal/storage/blob/fs" + _ "github.com/foxcpp/maddy/internal/storage/blob/s3" _ "github.com/foxcpp/maddy/internal/storage/imapsql" _ "github.com/foxcpp/maddy/internal/table" _ "github.com/foxcpp/maddy/internal/target/queue"