storage/imapsql: Implement auth_map

docs/man/maddy-storage.5.scd

@@ -153,5 +153,48 @@ Ex.
 imap_filters {
 	command /etc/maddy/sieve.sh {account_name}
 }
-}
 ```
+
+*Syntax*: auth_map *table* ++
+*Default*: not set
+
+Use specified table module (*maddy-tables*(5)) to map authentication
+usernames to mailbox names.
+
+Normalization algorithm specified in auth_normalize is applied before
+auth_map.
+
+imapsql unconditionally requires a full email as an account name as they are
+used to determine the right mailboxes for inbound messages. If you do not
+want domain-specific accounts then you can use a dummy domain like
+foxcpp@maddy.invalid. But you will need to ensure that on delivery recipient
+addresses are rewritten appropriately (e.g.
+foxcpp@example.org => foxcpp@maddy.invalid).
+
+*Syntax*: auth_normalize precis_email|precis|casefold|no ++
+*Default*: precis_email
+
+Normalization algorithm to apply to authentication usernames before mapping
+them to mailboxes.
+
+Options are:
+
+- precis_email
+
+RFC 8265 "UsernameCaseMapped" profile + additional logic to normalize domain
+part of a full email (converted to U-labels, case-folded and NFC-normalized).
+
+- precis
+
+RFC 8265 "UsernameCaseMapped" profile applied to the whole string.
+
+- casefold
+
+Username is just converted to lower case.
+
+- no
+
+No normalization.
+
+Note: On message delivery, recipient address is unconditionally normalized
+using precis_email algorithm.

internal/storage/imapsql/delivery.go

@@ -21,7 +21,6 @@ package imapsql
 import (
 	"context"
 	"runtime/trace"
-	"strings"
 
 	specialuse "github.com/emersion/go-imap-specialuse"
 	"github.com/emersion/go-imap/backend"
@@ -46,21 +45,24 @@ func (d *delivery) String() string {
 	return d.store.Name() + ":" + d.store.InstanceName()
 }
 
+func userDoesNotExist(actual error) error {
+	return &exterrors.SMTPError{
+		Code:         501,
+		EnhancedCode: exterrors.EnhancedCode{5, 1, 1},
+		Message:      "User does not exist",
+		TargetName:   "imapsql",
+		Err:          actual,
+	}
+}
+
 func (d *delivery) AddRcpt(ctx context.Context, rcptTo string) error {
 	defer trace.StartRegion(ctx, "sql/AddRcpt").End()
 
-	accountName, err := prepareUsername(rcptTo)
+	accountName, err := d.store.deliveryNormalize(rcptTo)
 	if err != nil {
-		return &exterrors.SMTPError{
+		return userDoesNotExist(err)
-			Code:         501,
-			EnhancedCode: exterrors.EnhancedCode{5, 1, 1},
-			Message:      "User does not exist",
-			TargetName:   "imapsql",
-			Err:          err,
-		}
 	}
 
-	accountName = strings.ToLower(accountName)
 	if _, ok := d.addedRcpts[accountName]; ok {
 		return nil
 	}
@@ -73,19 +75,13 @@ func (d *delivery) AddRcpt(ctx context.Context, rcptTo string) error {
 
 	if err := d.d.AddRcpt(accountName, userHeader); err != nil {
 		if err == imapsql.ErrUserDoesntExists || err == backend.ErrNoSuchMailbox {
-			return &exterrors.SMTPError{
+			return userDoesNotExist(err)
-				Code:         550,
-				EnhancedCode: exterrors.EnhancedCode{5, 1, 1},
-				Message:      "User does not exist",
-				TargetName:   "imapsql",
-				Err:          err,
-			}
 		}
 		if _, ok := err.(imapsql.SerializationError); ok {
 			return &exterrors.SMTPError{
 				Code:         453,
 				EnhancedCode: exterrors.EnhancedCode{4, 3, 2},
-				Message:      "Storage access serialiation problem, try again later",
+				Message:      "Internal server error, try again later",
 				TargetName:   "imapsql",
 				Err:          err,
 			}

internal/storage/imapsql/imapsql.go

@@ -40,14 +40,12 @@ import (
 	sortthread "github.com/emersion/go-imap-sortthread"
 	"github.com/emersion/go-imap/backend"
 	imapsql "github.com/foxcpp/go-imap-sql"
-	"github.com/foxcpp/maddy/framework/address"
 	"github.com/foxcpp/maddy/framework/config"
 	modconfig "github.com/foxcpp/maddy/framework/config/module"
 	"github.com/foxcpp/maddy/framework/dns"
 	"github.com/foxcpp/maddy/framework/log"
 	"github.com/foxcpp/maddy/framework/module"
 	"github.com/foxcpp/maddy/internal/updatepipe"
-	"golang.org/x/text/secure/precis"
 
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/lib/pq"
@@ -70,6 +68,10 @@ type Storage struct {
 	updPushStop chan struct{}
 
 	filters module.IMAPFilter
+
+	deliveryNormalize func(string) (string, error)
+	authMap           module.Table
+	authNormalize     func(string) (string, error)
 }
 
 func (store *Storage) Name() string {
@@ -104,6 +106,7 @@ func (store *Storage) Init(cfg *config.Map) error {
 		fsstoreLocation string
 		appendlimitVal  = -1
 		compression     []string
+		authNormalize   string
 	)
 
 	opts := imapsql.Opts{
@@ -135,6 +138,11 @@ func (store *Storage) Init(cfg *config.Map) error {
 		err := modconfig.GroupFromNode("imap_filters", node.Args, node, m.Globals, &filter)
 		return filter, err
 	}, &store.filters)
+	cfg.Custom("auth_map", false, false, func() (interface{}, error) {
+		return nil, nil
+	}, modconfig.TableDirective, &store.authMap)
+	cfg.Enum("auth_normalize", false, false,
+		[]string{"precis_email", "precis", "casefold", "no"}, "precis_email", &authNormalize)
 
 	if _, err := cfg.Process(); err != nil {
 		return err
@@ -147,6 +155,23 @@ func (store *Storage) Init(cfg *config.Map) error {
 		return errors.New("imapsql: driver is required")
 	}
 
+	store.deliveryNormalize = normalizeFuncs["precis_email"]
+	authNormFunc := normalizeFuncs[authNormalize]
+	store.authNormalize = authNormFunc
+	if store.authMap != nil {
+		store.authNormalize = func(username string) (string, error) {
+			username, err := authNormFunc(username)
+			if err != nil {
+				return "", err
+			}
+			mapped, ok, err := store.authMap.Lookup(username)
+			if err != nil || !ok {
+				return "", userDoesNotExist(err)
+			}
+			return mapped, nil
+		}
+	}
+
 	opts.Log = &store.Log
 
 	if appendlimitVal == -1 {
@@ -307,34 +332,8 @@ func (store *Storage) EnableChildrenExt() bool {
 	return store.Back.EnableChildrenExt()
 }
 
-func prepareUsername(username string) (string, error) {
-	mbox, domain, err := address.Split(username)
-	if err != nil {
-		return "", fmt.Errorf("imapsql: username prepare: %w", err)
-	}
-
-	// PRECIS is not included in the regular address.ForLookup since it reduces
-	// the range of valid addresses to a subset of actually valid values.
-	// PRECIS is a matter of our own local policy, not a general rule for all
-	// email addresses.
-
-	// Side note: For used profiles, there is no practical difference between
-	// CompareKey and String.
-	mbox, err = precis.UsernameCaseMapped.CompareKey(mbox)
-	if err != nil {
-		return "", fmt.Errorf("imapsql: username prepare: %w", err)
-	}
-
-	domain, err = dns.ForLookup(domain)
-	if err != nil {
-		return "", fmt.Errorf("imapsql: username prepare: %w", err)
-	}
-
-	return mbox + "@" + domain, nil
-}
-
 func (store *Storage) GetOrCreateIMAPAcct(username string) (backend.User, error) {
-	accountName, err := prepareUsername(username)
+	accountName, err := store.authNormalize(username)
 	if err != nil {
 		return nil, backend.ErrInvalidCredentials
 	}
@@ -343,7 +342,7 @@ func (store *Storage) GetOrCreateIMAPAcct(username string) (backend.User, error)
 }
 
 func (store *Storage) Lookup(key string) (string, bool, error) {
-	accountName, err := prepareUsername(key)
+	accountName, err := store.authNormalize(key)
 	if err != nil {
 		return "", false, nil
 	}

internal/storage/imapsql/maddyctl.go

@@ -29,29 +29,14 @@ func (store *Storage) ListIMAPAccts() ([]string, error) {
 	return store.Back.ListUsers()
 }
 
-func (store *Storage) CreateIMAPAcct(username string) error {
+func (store *Storage) CreateIMAPAcct(accountName string) error {
-	accountName, err := prepareUsername(username)
-	if err != nil {
-		return err
-	}
-
 	return store.Back.CreateUser(accountName)
 }
 
-func (store *Storage) DeleteIMAPAcct(username string) error {
+func (store *Storage) DeleteIMAPAcct(accountName string) error {
-	accountName, err := prepareUsername(username)
-	if err != nil {
-		return err
-	}
-
 	return store.Back.DeleteUser(accountName)
 }
 
-func (store *Storage) GetIMAPAcct(username string) (backend.User, error) {
+func (store *Storage) GetIMAPAcct(accountName string) (backend.User, error) {
-	accountName, err := prepareUsername(username)
-	if err != nil {
-		return nil, err
-	}
-
 	return store.Back.GetUser(accountName)
 }

internal/storage/imapsql/normalize.go

@@ -0,0 +1,65 @@
+/*
+Maddy Mail Server - Composable all-in-one email server.
+Copyright © 2019-2020 Max Mazurov <fox.cpp@disroot.org>, Maddy Mail Server contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+package imapsql
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/foxcpp/maddy/framework/address"
+	"github.com/foxcpp/maddy/framework/dns"
+	"golang.org/x/text/secure/precis"
+)
+
+var normalizeFuncs = map[string]func(string) (string, error){
+	"precis_email": func(s string) (string, error) {
+		mbox, domain, err := address.Split(s)
+		if err != nil {
+			return "", fmt.Errorf("imapsql: username prepare: %w", err)
+		}
+
+		// PRECIS is not included in the regular address.ForLookup since it reduces
+		// the range of valid addresses to a subset of actually valid values.
+		// PRECIS is a matter of our own local policy, not a general rule for all
+		// email addresses.
+
+		// Side note: For used profiles, there is no practical difference between
+		// CompareKey and String.
+		mbox, err = precis.UsernameCaseMapped.CompareKey(mbox)
+		if err != nil {
+			return "", fmt.Errorf("imapsql: username prepare: %w", err)
+		}
+
+		domain, err = dns.ForLookup(domain)
+		if err != nil {
+			return "", fmt.Errorf("imapsql: username prepare: %w", err)
+		}
+
+		return mbox + "@" + domain, nil
+	},
+	"precis": func(s string) (string, error) {
+		return precis.UsernameCaseMapped.CompareKey(s)
+	},
+	"casefold": func(s string) (string, error) {
+		return strings.ToLower(s), nil
+	},
+	"no": func(s string) (string, error) {
+		return s, nil
+	},
+}

tests/imapsql_test.go

@@ -111,3 +111,74 @@ func TestImapsqlDelivery(tt *testing.T) {
 	imapConn.Expect(")")
 	imapConn.ExpectPattern(`. OK *`)
 }
+
+func TestImapsqlAuthMap(tt *testing.T) {
+	tt.Parallel()
+	t := tests.NewT(tt)
+
+	t.DNS(nil)
+	t.Port("imap")
+	t.Port("smtp")
+	t.Config(`
+		storage.imapsql test_store {
+			auth_map regexp "(.*)" "$1@maddy.test"
+			auth_normalize precis
+
+			driver sqlite3
+			dsn imapsql.db
+		}
+
+		imap tcp://127.0.0.1:{env:TEST_PORT_imap} {
+			tls off
+
+			auth dummy
+			storage &test_store
+		}
+
+		smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} {
+			hostname maddy.test
+			tls off
+
+			deliver_to &test_store
+		}
+	`)
+	t.Run(2)
+	defer t.Close()
+
+	imapConn := t.Conn("imap")
+	defer imapConn.Close()
+	imapConn.ExpectPattern(`\* OK *`)
+	imapConn.Writeln(". LOGIN testusr 1234")
+	imapConn.ExpectPattern(". OK *")
+	imapConn.Writeln(". SELECT INBOX")
+	imapConn.ExpectPattern(`\* *`)
+	imapConn.ExpectPattern(`\* *`)
+	imapConn.ExpectPattern(`\* *`)
+	imapConn.ExpectPattern(`\* *`)
+	imapConn.ExpectPattern(`\* *`)
+	imapConn.ExpectPattern(`\* *`)
+	imapConn.ExpectPattern(`. OK *`)
+
+	smtpConn := t.Conn("smtp")
+	defer smtpConn.Close()
+	smtpConn.SMTPNegotation("localhost", nil, nil)
+	smtpConn.Writeln("MAIL FROM:<sender@maddy.test>")
+	smtpConn.ExpectPattern("2*")
+	smtpConn.Writeln("RCPT TO:<testusr@maddy.test>")
+	smtpConn.ExpectPattern("2*")
+	smtpConn.Writeln("DATA")
+	smtpConn.ExpectPattern("354 *")
+	smtpConn.Writeln("From: <sender@maddy.test>")
+	smtpConn.Writeln("To: <testusr@maddy.test>")
+	smtpConn.Writeln("Subject: Hi!")
+	smtpConn.Writeln("")
+	smtpConn.Writeln("Hi!")
+	smtpConn.Writeln(".")
+	smtpConn.ExpectPattern("2*")
+
+	time.Sleep(500 * time.Millisecond)
+
+	imapConn.Writeln(". NOOP")
+	imapConn.ExpectPattern(`\* 1 EXISTS`)
+	imapConn.ExpectPattern(". OK *")
+}