Refactor imapsql ExternalStore to use modules

Closes #303

.mkdocs.yml

@@ -29,6 +29,7 @@ nav:
     - man/_generated_maddy.1.md
     - man/_generated_maddy.5.md
     - man/_generated_maddy-auth.5.md
+    - man/_generated_maddy-blob.5.md
     - man/_generated_maddy-config.5.md
     - man/_generated_maddy-filters.5.md
     - man/_generated_maddy-imap.5.md

docs/man/maddy-blob.5.scd

@@ -0,0 +1,41 @@
+maddy-blob(5) "maddy mail server" "maddy reference documentation"
+
+; TITLE Message blob storage
+
+Some IMAP storage backends support pluggable message storage that allows
+message contents to be stored separately from IMAP index.
+
+Modules described in this page are what can be used with such storage backends.
+In most cases they have to be specified using the 'msg_store' directive, like
+this:
+```
+storage.imapsql local_mailboxes {
+    msg_store fs /var/lib/email
+}
+```
+
+Unless explicitly configured, storage backends with pluggable storage will
+store messages in state_dir/messages (e.g. /var/lib/maddy/messages) FS
+directory.
+
+# FS directory storage (storage.blob.fs)
+
+This module stores message bodies in a file system directory.
+
+```
+storage.blob.fs {
+    root <directory>
+}
+```
+```
+storage.blob.fs <directory>
+```
+
+## Configuration directives
+
+*Syntax:* root _path_ ++
+*Default:* not set
+
+Path to the FS directory. Must be readable and writable by the server process.
+If it does not exist - it will be created (parent directory should be writable
+for this). Relative paths are interpreted relatively to server state directory.

docs/man/maddy-storage.5.scd

@@ -21,12 +21,11 @@ created.
 
 # SQL-based database module (storage.imapsql)
 
-The imapsql module implements unified database for IMAP index and message
+The imapsql module implements database for IMAP index and message
 metadata using SQL-based relational database.
 
-Message contents are stored in an "external store", currently the only
+Message contents are stored in an "external store" defined by msg_store
-supported "external store" is a filesystem directory, used by default.
+directive. By default this is a file system directory under /var/lib/maddy.
-By default, all messages are stored in StateDirectory/messages under random IDs.
 
 Supported RDBMS:
 - SQLite 3.25.0
@@ -40,6 +39,7 @@ PRECIS UsernameCaseMapped profile.
 storage.imapsql {
 	driver sqlite3
 	dsn imapsql.db
+	msg_store fs messages/
 }
 ```
 
@@ -88,10 +88,12 @@ For PostgreSQL: https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parame
 
 Should be specified either via an argument or via this directive.
 
-*Syntax*: fsstore _directory_ ++
+*Syntax*: msg_store _store_ ++
-*Default*: messages/
+*Default*: fs messages/
 
-Directory to store message contents in.
+Module to use for message bodies storage.
+
+See *maddy-blob*(5) for details.
 
 *Syntax*: ++
     compression off ++

framework/module/blob_store.go

@@ -0,0 +1,30 @@
+package module
+
+import (
+	"errors"
+	"io"
+)
+
+type Blob interface {
+	Sync() error
+	io.Reader
+	io.Writer
+	io.Closer
+}
+
+var ErrNoSuchBlob = errors.New("blob_store: no such object")
+
+// BlobStore is the interface used by modules providing large binary object
+// storage.
+type BlobStore interface {
+	Create(key string) (Blob, error)
+
+	// Open returns the reader for the object specified by
+	// passed key.
+	//
+	// If no such object exists - ErrNoSuchBlob is returned.
+	Open(key string) (io.ReadCloser, error)
+
+	// Delete removes a set of keys from store. Non-existent keys are ignored.
+	Delete(keys []string) error
+}

internal/storage/blob/fs/fs.go

@@ -0,0 +1,89 @@
+package fs
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/foxcpp/maddy/framework/config"
+	"github.com/foxcpp/maddy/framework/module"
+)
+
+// FSStore struct represents directory on FS used to store blobs.
+type FSStore struct {
+	instName string
+	root     string
+}
+
+func New(_, instName string, _, inlineArgs []string) (module.Module, error) {
+	switch len(inlineArgs) {
+	case 0:
+		return &FSStore{instName: instName}, nil
+	case 1:
+		return &FSStore{instName: instName, root: inlineArgs[0]}, nil
+	default:
+		return nil, fmt.Errorf("storage.blob.fs: 1 or 0 arguments expected")
+	}
+}
+
+func (s FSStore) Name() string {
+	return "storage.blob.fs"
+}
+
+func (s FSStore) InstanceName() string {
+	return s.instName
+}
+
+func (s *FSStore) Init(cfg *config.Map) error {
+	cfg.String("root", false, false, s.root, &s.root)
+	if _, err := cfg.Process(); err != nil {
+		return err
+	}
+
+	if s.root == "" {
+		return config.NodeErr(cfg.Block, "storage.blob.fs: directory not set")
+	}
+
+	if err := os.MkdirAll(s.root, os.ModeDir|os.ModePerm); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *FSStore) Open(key string) (io.ReadCloser, error) {
+	f, err := os.Open(filepath.Join(s.root, key))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, module.ErrNoSuchBlob
+		}
+		return nil, err
+	}
+	return f, nil
+}
+
+func (s *FSStore) Create(key string) (module.Blob, error) {
+	f, err := os.Create(filepath.Join(s.root, key))
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+func (s *FSStore) Delete(keys []string) error {
+	for _, key := range keys {
+		if err := os.Remove(filepath.Join(s.root, key)); err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+func init() {
+	var _ module.BlobStore = &FSStore{}
+	module.Register(FSStore{}.Name(), New)
+}

internal/storage/imapsql/external_blob_store.go

@@ -0,0 +1,59 @@
+package imapsql
+
+import (
+	"io"
+
+	imapsql "github.com/foxcpp/go-imap-sql"
+	"github.com/foxcpp/maddy/framework/module"
+)
+
+type ExtBlob struct {
+	io.ReadCloser
+}
+
+func (e ExtBlob) Sync() error {
+	panic("not implemented")
+}
+
+func (e ExtBlob) Write(p []byte) (n int, err error) {
+	panic("not implemented")
+}
+
+type ExtBlobStore struct {
+	base module.BlobStore
+}
+
+func (e ExtBlobStore) Create(key string) (imapsql.ExtStoreObj, error) {
+	blob, err := e.base.Create(key)
+	if err != nil {
+		return nil, imapsql.ExternalError{
+			NonExistent: err == module.ErrNoSuchBlob,
+			Key:         key,
+			Err:         err,
+		}
+	}
+	return blob, nil
+}
+
+func (e ExtBlobStore) Open(key string) (imapsql.ExtStoreObj, error) {
+	blob, err := e.base.Open(key)
+	if err != nil {
+		return nil, imapsql.ExternalError{
+			NonExistent: err == module.ErrNoSuchBlob,
+			Key:         key,
+			Err:         err,
+		}
+	}
+	return ExtBlob{ReadCloser: blob}, nil
+}
+
+func (e ExtBlobStore) Delete(keys []string) error {
+	err := e.base.Delete(keys)
+	if err != nil {
+		return imapsql.ExternalError{
+			Key: "",
+			Err: err,
+		}
+	}
+	return nil
+}

internal/storage/imapsql/imapsql.go

@@ -31,7 +31,6 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
 	"runtime/debug"
 	"strconv"
@@ -103,12 +102,13 @@ func New(_, instName string, _, inlineArgs []string) (module.Module, error) {
 
 func (store *Storage) Init(cfg *config.Map) error {
 	var (
-		driver          string
+		driver         string
-		dsn             []string
+		dsn            []string
-		fsstoreLocation string
+		appendlimitVal = -1
-		appendlimitVal  = -1
+		compression    []string
-		compression     []string
+		authNormalize  string
-		authNormalize   string
+
+		blobStore module.BlobStore
 	)
 
 	opts := imapsql.Opts{
@@ -118,14 +118,22 @@ func (store *Storage) Init(cfg *config.Map) error {
 	}
 	cfg.String("driver", false, false, store.driver, &driver)
 	cfg.StringList("dsn", false, false, store.dsn, &dsn)
-	cfg.Custom("fsstore", false, false, func() (interface{}, error) {
+	cfg.Callback("fsstore", func(m *config.Map, node config.Node) error {
-		return "messages", nil
+		store.Log.Msg("'fsstore' directive is deprecated, use 'msg_store fs' instead")
+		return modconfig.ModuleFromNode("storage.blob", append([]string{"fs"}, node.Args...),
+			node, m.Globals, &blobStore)
+	})
+	cfg.Custom("msg_store", false, false, func() (interface{}, error) {
+		var store module.BlobStore
+		err := modconfig.ModuleFromNode("storage.blob", []string{"fs", "messages"},
+			config.Node{}, nil, &store)
+		return store, err
 	}, func(m *config.Map, node config.Node) (interface{}, error) {
-		if len(node.Args) != 1 {
+		var store module.BlobStore
-			return nil, config.NodeErr(node, "expected 0 or 1 arguments")
+		err := modconfig.ModuleFromNode("storage.blob", node.Args,
-		}
+			node, m.Globals, &store)
-		return node.Args[0], nil
+		return store, err
-	}, &fsstoreLocation)
+	}, &blobStore)
 	cfg.StringList("compression", false, false, []string{"off"}, &compression)
 	cfg.DataSize("appendlimit", false, false, 32*1024*1024, &appendlimitVal)
 	cfg.Bool("debug", true, false, &store.Log.Debug)
@@ -195,11 +203,6 @@ func (store *Storage) Init(cfg *config.Map) error {
 
 	dsnStr := strings.Join(dsn, " ")
 
-	if err := os.MkdirAll(fsstoreLocation, os.ModeDir|os.ModePerm); err != nil {
-		return err
-	}
-	extStore := &imapsql.FSStore{Root: fsstoreLocation}
-
 	if len(compression) != 0 {
 		switch compression[0] {
 		case "zstd", "lz4":
@@ -222,7 +225,7 @@ func (store *Storage) Init(cfg *config.Map) error {
 		}
 	}
 
-	store.Back, err = imapsql.New(driver, dsnStr, extStore, opts)
+	store.Back, err = imapsql.New(driver, dsnStr, ExtBlobStore{base: blobStore}, opts)
 	if err != nil {
 		return fmt.Errorf("imapsql: %s", err)
 	}

maddy.go

@@ -61,6 +61,7 @@ import (
 	_ "github.com/foxcpp/maddy/internal/imap_filter/command"
 	_ "github.com/foxcpp/maddy/internal/modify"
 	_ "github.com/foxcpp/maddy/internal/modify/dkim"
+	_ "github.com/foxcpp/maddy/internal/storage/blob/fs"
 	_ "github.com/foxcpp/maddy/internal/storage/imapsql"
 	_ "github.com/foxcpp/maddy/internal/table"
 	_ "github.com/foxcpp/maddy/internal/target/queue"
@@ -344,6 +345,8 @@ func RegisterModules(globals map[string]interface{}, nodes []config.Node) (endpo
 			}
 			module.RegisterAlias(alias, instName)
 		}
+
+		log.Debugf("%v:%v: register config block %v %v", block.File, block.Line, instName, modAliases)
 		mods = append(mods, ModInfo{Instance: inst, Cfg: block})
 	}