diff --git a/cmd/maddyctl/imap.go b/cmd/maddyctl/imap.go index e5215b5..05ebf65 100644 --- a/cmd/maddyctl/imap.go +++ b/cmd/maddyctl/imap.go @@ -66,12 +66,7 @@ func mboxesList(be module.Storage, ctx *cli.Context) error { fmt.Fprintln(os.Stderr, "No mailboxes.") } - for _, mbox := range mboxes { - info, err := mbox.Info() - if err != nil { - return err - } - + for _, info := range mboxes { if len(info.Attributes) != 0 { fmt.Print(info.Name, "\t", info.Attributes, "\n") } else { @@ -126,13 +121,8 @@ func mboxesRemove(be module.Storage, ctx *cli.Context) error { return err } - mbox, err := u.GetMailbox(name) - if err != nil { - return err - } - if !ctx.Bool("yes,y") { - status, err := mbox.Status([]imap.StatusItem{imap.StatusMessages}) + status, err := u.Status(name, []imap.StatusItem{imap.StatusMessages}) if err != nil { return err } @@ -190,11 +180,6 @@ func msgsAdd(be module.Storage, ctx *cli.Context) error { return err } - mbox, err := u.GetMailbox(name) - if err != nil { - return err - } - flags := ctx.StringSlice("flag") if flags == nil { flags = []string{} @@ -214,15 +199,16 @@ func msgsAdd(be module.Storage, ctx *cli.Context) error { return errors.New("Error: Empty message, refusing to continue") } - status, err := mbox.Status([]imap.StatusItem{imap.StatusUidNext}) + status, err := u.Status(name, []imap.StatusItem{imap.StatusUidNext}) if err != nil { return err } - if err := mbox.CreateMessage(flags, date, &buf); err != nil { + if err := u.CreateMessage(name, flags, date, &buf, nil); err != nil { return err } + // TODO: Use APPENDUID fmt.Println(status.UidNext) return nil @@ -252,7 +238,7 @@ func msgsRemove(be module.Storage, ctx *cli.Context) error { return err } - mbox, err := u.GetMailbox(name) + _, mbox, err := u.GetMailbox(name, true, nil) if err != nil { return err } @@ -299,7 +285,7 @@ func msgsCopy(be module.Storage, ctx *cli.Context) error { return err } - srcMbox, err := u.GetMailbox(srcName) + _, srcMbox, err := u.GetMailbox(srcName, true, nil) if err != nil { return err } @@ -339,7 +325,7 @@ func msgsMove(be module.Storage, ctx *cli.Context) error { return err } - srcMbox, err := u.GetMailbox(srcName) + _, srcMbox, err := u.GetMailbox(srcName, true, nil) if err != nil { return err } @@ -373,7 +359,7 @@ func msgsList(be module.Storage, ctx *cli.Context) error { return err } - mbox, err := u.GetMailbox(mboxName) + _, mbox, err := u.GetMailbox(mboxName, true, nil) if err != nil { return err } @@ -449,7 +435,7 @@ func msgsDump(be module.Storage, ctx *cli.Context) error { return err } - mbox, err := u.GetMailbox(mboxName) + _, mbox, err := u.GetMailbox(mboxName, true, nil) if err != nil { return err } @@ -493,7 +479,7 @@ func msgsFlags(be module.Storage, ctx *cli.Context) error { return err } - mbox, err := u.GetMailbox(name) + _, mbox, err := u.GetMailbox(name, true, nil) if err != nil { return err } @@ -515,5 +501,5 @@ func msgsFlags(be module.Storage, ctx *cli.Context) error { panic("unknown command: " + ctx.Command.Name) } - return mbox.UpdateMessagesFlags(ctx.IsSet("uid"), seq, op, flags) + return mbox.UpdateMessagesFlags(ctx.IsSet("uid"), seq, op, true, flags) } diff --git a/docs/man/maddy-storage.5.scd b/docs/man/maddy-storage.5.scd index bcdee04..c752895 100644 --- a/docs/man/maddy-storage.5.scd +++ b/docs/man/maddy-storage.5.scd @@ -123,12 +123,13 @@ Enable verbose logging. The folder to put quarantined messages in. Thishis setting is not used if user does have a folder with "Junk" special-use attribute. -*Syntax*: sqlite_exclusive_lock _boolean_ ++ -*Default*: no +*Syntax*: disable_recent _boolean_ ++ +*Default: true -SQLite-specific performance tuning option. Slightly decereases ovehead of -DB locking at cost of making DB inaccessible for other processes (including -maddyctl utility). +Disable RFC 3501-conforming handling of \Recent flag. + +This significantly improves storage performance when SQLite3 or CockroackDB is +used at the cost of confusing clients that use this flag. *Syntax*: sqlite_cache_size _integer_ ++ *Default*: defined by SQLite diff --git a/go.mod b/go.mod index 28db408..32510f0 100644 --- a/go.mod +++ b/go.mod @@ -21,10 +21,11 @@ require ( github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 github.com/emersion/go-smtp v0.15.1-0.20210705155248-26eb4814e227 github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf - github.com/foxcpp/go-imap-backend-tests v0.0.0-20200617132817-958ea5829771 + github.com/foxcpp/go-imap-backend-tests v0.0.0-20200802090154-7e6248c85a0e github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005 - github.com/foxcpp/go-imap-namespace v0.0.0-20200722130255-93092adf35f1 - github.com/foxcpp/go-imap-sql v0.4.1-0.20200823124337-2f57903a7ed0 + github.com/foxcpp/go-imap-mess v0.0.0-20210718073110-d5eb968a0995 + github.com/foxcpp/go-imap-namespace v0.0.0-20200802091432-08496dd8e0ed + github.com/foxcpp/go-imap-sql v0.4.1-0.20210718081250-7f103db60f22 github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15 github.com/foxcpp/go-mtasts v0.0.0-20191219193356-62bc3f1f74b8 github.com/go-ldap/ldap/v3 v3.3.0 @@ -60,3 +61,7 @@ require ( golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/text v0.3.6 ) + +replace github.com/emersion/go-imap => github.com/foxcpp/go-imap v1.0.0-beta.1.0.20201001193006-5a1d05e53e2c + +replace github.com/emersion/go-imap-idle => github.com/foxcpp/go-imap-idle v0.0.0-20200829140055-32dc40172769 diff --git a/go.sum b/go.sum index 06fd49f..f8a7034 100644 --- a/go.sum +++ b/go.sum @@ -111,19 +111,10 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/emersion/go-imap v1.0.0-beta.4.0.20190504114255-4d5af3d05147/go.mod h1:mOPegfAgLVXbhRm1bh2JTX08z2Y3HYmKYpbrKDeAzsQ= -github.com/emersion/go-imap v1.0.0/go.mod h1:MEiDDwwQFcZ+L45Pa68jNGv0qU9kbW+SJzwDpvSfX1s= -github.com/emersion/go-imap v1.0.3/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU= -github.com/emersion/go-imap v1.0.4/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU= -github.com/emersion/go-imap v1.0.5/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU= -github.com/emersion/go-imap v1.0.6 h1:N9+o5laOGuntStBo+BOgfEB5evPsPD+K5+M0T2dctIc= -github.com/emersion/go-imap v1.0.6/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU= github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a h1:bMdSPm6sssuOFpIaveu3XGAijMS3Tq2S3EqFZmZxidc= github.com/emersion/go-imap-appendlimit v0.0.0-20190308131241-25671c986a6a/go.mod h1:ikgISoP7pRAolqsVP64yMteJa2FIpS6ju88eBT6K1yQ= github.com/emersion/go-imap-compress v0.0.0-20201103190257-14809af1d1b9 h1:7dmV11mle4UAQ7lX+Hdzx6akKFg3hVm/UUmQ7t6VgTQ= github.com/emersion/go-imap-compress v0.0.0-20201103190257-14809af1d1b9/go.mod h1:2Ro1PbmiqYiRe5Ct2sGR5hHaKSVHeRpVZwXx8vyYt98= -github.com/emersion/go-imap-idle v0.0.0-20201224103203-6f42b9020098 h1:J+qvrz94n18fVThwhUWwrBwRbcNqi+VgcUJlaph430A= -github.com/emersion/go-imap-idle v0.0.0-20201224103203-6f42b9020098/go.mod h1:N/6S3dRTVt8xT867m+476C16+v/Fq4WZYvh2Chg0nmg= github.com/emersion/go-imap-move v0.0.0-20180601155324-5eb20cb834bf/go.mod h1:QuMaZcKFDVI0yCrnAbPLfbwllz1wtOrZH8/vZ5yzp4w= github.com/emersion/go-imap-move v0.0.0-20190710073258-6e5a51a5b342 h1:5p1t3e1PomYgLWwEwhwEU5kVBwcyAcVrOpexv8AeZx0= github.com/emersion/go-imap-move v0.0.0-20190710073258-6e5a51a5b342/go.mod h1:QuMaZcKFDVI0yCrnAbPLfbwllz1wtOrZH8/vZ5yzp4w= @@ -135,9 +126,7 @@ github.com/emersion/go-imap-specialuse v0.0.0-20201101201809-1ab93d3d150e h1:AwV github.com/emersion/go-imap-specialuse v0.0.0-20201101201809-1ab93d3d150e/go.mod h1:/nybxhI8kXom8Tw6BrHMl42usALvka6meORflnnYwe4= github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26 h1:FiSb8+XBQQSkcX3ubr+1tAtlRJBYaFmRZqOAweZ9Wy8= github.com/emersion/go-imap-unselect v0.0.0-20171113212723-b985794e5f26/go.mod h1:+gnnZx3Mg3MnCzZrv0eZdp5puxXQUgGT/6N6L7ShKfM= -github.com/emersion/go-message v0.9.1/go.mod h1:m3cK90skCWxm5sIMs1sXxly4Tn9Plvcf6eayHZJ1NzM= github.com/emersion/go-message v0.10.3/go.mod h1:3h+HsGTCFHmk4ngJ2IV/YPhdlaOcR6hcgqM3yca9v7c= -github.com/emersion/go-message v0.10.4-0.20190609165112-592ace5bc1ca/go.mod h1:3h+HsGTCFHmk4ngJ2IV/YPhdlaOcR6hcgqM3yca9v7c= github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY= github.com/emersion/go-message v0.11.2/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY= github.com/emersion/go-message v0.14.1 h1:j3rj9F+7VtXE9c8P5UHBq8FTHLW/AjnmvSRre6AHoYI= @@ -146,8 +135,6 @@ github.com/emersion/go-milter v0.3.2 h1:j8hrLXf8PAHFhRHDdBoBKluQveMZYoaK7aRIqvao github.com/emersion/go-milter v0.3.2/go.mod h1:ablHK0pbLB83kMFBznp/Rj8aV+Kc3jw8cxzzmCNLIOY= github.com/emersion/go-msgauth v0.6.5 h1:UaXBtrjYBM3SWw9BBODeSp0uYtScx3CuIF7/RQfkeWo= github.com/emersion/go-msgauth v0.6.5/go.mod h1:/jbQISFJgtT12T8akRs20l+wI4HcyN/kWy7VRdHEAmA= -github.com/emersion/go-sasl v0.0.0-20161116183048-7e096a0a6197/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= -github.com/emersion/go-sasl v0.0.0-20190520160400-47d427600317/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= github.com/emersion/go-sasl v0.0.0-20190817083125-240c8404624e/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ= @@ -168,14 +155,21 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf h1:rmBPY5fryjp9zLQYsUmQqqgsYq7qeVfrjtr96Tf9vD8= github.com/foxcpp/go-dovecot-sasl v0.0.0-20200522223722-c4699d7a24bf/go.mod h1:5yZUmwr851vgjyAfN7OEfnrmKOh/qLA5dbGelXYsu1E= -github.com/foxcpp/go-imap-backend-tests v0.0.0-20200617132817-958ea5829771 h1:xemWCEhBz86Y8v5YgRBnqf6PdZg+ilVgn2jxWVoLOGo= +github.com/foxcpp/go-imap v1.0.0-beta.1.0.20201001193006-5a1d05e53e2c h1:EKhtM7IsuerenjPu3bMdZtUG7MEYBf18HLffVlrHHyc= +github.com/foxcpp/go-imap v1.0.0-beta.1.0.20201001193006-5a1d05e53e2c/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU= github.com/foxcpp/go-imap-backend-tests v0.0.0-20200617132817-958ea5829771/go.mod h1:yUISYv/uXLQ6tQZcds/p/hdcZ5JzrEUifyED2VffWpc= +github.com/foxcpp/go-imap-backend-tests v0.0.0-20200802090154-7e6248c85a0e h1:wUYcbeGDlKqaa02jHuFMBVVfw51v7KaizLRT+i8wftc= +github.com/foxcpp/go-imap-backend-tests v0.0.0-20200802090154-7e6248c85a0e/go.mod h1:dWepYGUClcebWyUy7yl8wm2ioE6NC4m82Xy/hDAng1c= github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005 h1:pfoFtkTTQ473qStSN79jhCFBWqMQt/3DQ3NGuXvT+50= github.com/foxcpp/go-imap-i18nlevel v0.0.0-20200208001533-d6ec88553005/go.mod h1:34FwxnjC2N+EFs2wMtsHevrZLWRKRuVU8wEcHWKq/nE= -github.com/foxcpp/go-imap-namespace v0.0.0-20200722130255-93092adf35f1 h1:B4zNQ2r4qC7FLn8J8+LWt09fFW0tXddypBPS0+HI50s= -github.com/foxcpp/go-imap-namespace v0.0.0-20200722130255-93092adf35f1/go.mod h1:WJYkFIdxyljR/byiqcYMKUF4iFDej4CaIKe2JJrQxu8= -github.com/foxcpp/go-imap-sql v0.4.1-0.20200823124337-2f57903a7ed0 h1:PWETJtfTn94l9zt1nqIZiFO0/hUQeBmptjtktqrau/Y= -github.com/foxcpp/go-imap-sql v0.4.1-0.20200823124337-2f57903a7ed0/go.mod h1:1dHCAq3XRkYRwTDOtL/vCgvvQ13gLqNt2+nLjL1UHyk= +github.com/foxcpp/go-imap-idle v0.0.0-20200829140055-32dc40172769 h1:qMdULYMxKuAgboOllBNLmA7wQ4YEwnhvq8onksrMC3A= +github.com/foxcpp/go-imap-idle v0.0.0-20200829140055-32dc40172769/go.mod h1:PLnHIusEiOdmy63Y7IL2RjShIk4cyFi3a8MTC/WcLkk= +github.com/foxcpp/go-imap-mess v0.0.0-20210718073110-d5eb968a0995 h1:UXksj5CP+1Zg5GCD74JEijuQ2C22vXFgquhdG/YlXuI= +github.com/foxcpp/go-imap-mess v0.0.0-20210718073110-d5eb968a0995/go.mod h1:cps13jIcqI/3FGVJQ8azz3apLjikGoKKcB6xb65VSag= +github.com/foxcpp/go-imap-namespace v0.0.0-20200802091432-08496dd8e0ed h1:1Jo7geyvunrPSjL6F6D9EcXoNApS5v3LQaro7aUNPnE= +github.com/foxcpp/go-imap-namespace v0.0.0-20200802091432-08496dd8e0ed/go.mod h1:Shows1vmkBWO40ChOClaUe6DUnZrsP1UPAuoWzIUdgQ= +github.com/foxcpp/go-imap-sql v0.4.1-0.20210718081250-7f103db60f22 h1:ZXgI+FkKj+iKSrI58CZEkbSSojV84jm4LjEwglyH4Ik= +github.com/foxcpp/go-imap-sql v0.4.1-0.20210718081250-7f103db60f22/go.mod h1:kl+x+noffdBsp1pAR+PTHupLoxHnJZZw6BFcmmCZeUI= github.com/foxcpp/go-mockdns v0.0.0-20191216195825-5eabd8dbfe1f/go.mod h1:tPg4cp4nseejPd+UKxtCVQ2hUxNTZ7qQZJa7CLriIeo= github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15 h1:nLPjjvpUAODOR6vY/7o0hBIk8iTr19Fvmf8aFx/kC7A= github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp4nseejPd+UKxtCVQ2hUxNTZ7qQZJa7CLriIeo= @@ -290,6 +284,7 @@ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -347,6 +342,7 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -533,7 +529,9 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= diff --git a/internal/endpoint/imap/imap.go b/internal/endpoint/imap/imap.go index 3d1eb0b..8cb7169 100644 --- a/internal/endpoint/imap/imap.go +++ b/internal/endpoint/imap/imap.go @@ -57,7 +57,6 @@ type Endpoint struct { listeners []net.Listener Store module.Storage - updater imapbackend.BackendUpdater tlsConfig *tls.Config listenersWg sync.WaitGroup @@ -98,24 +97,12 @@ func (endp *Endpoint) Init(cfg *config.Map) error { return err } - var ok bool - endp.updater, ok = endp.Store.(imapbackend.BackendUpdater) - if !ok { - return fmt.Errorf("imap: storage module %T does not implement imapbackend.BackendUpdater", endp.Store) - } - if updBe, ok := endp.Store.(updatepipe.Backend); ok { if err := updBe.EnableUpdatePipe(updatepipe.ModeReplicate); err != nil { endp.Log.Error("failed to initialize updates pipe", err) } } - // Call Updates once at start, some storage backends initialize update - // channel lazily and may not generate updates at all unless it is called. - if endp.updater.Updates() == nil { - return fmt.Errorf("imap: failed to init backend: nil update channel") - } - addresses := make([]config.Endpoint, 0, len(endp.addrs)) for _, addr := range endp.addrs { saddr, err := config.ParseEndpoint(addr) @@ -198,10 +185,6 @@ func (endp *Endpoint) setupListeners(addresses []config.Endpoint) error { return nil } -func (endp *Endpoint) Updates() <-chan imapbackend.Update { - return endp.updater.Updates() -} - func (endp *Endpoint) Name() string { return "imap" } diff --git a/internal/storage/imapsql/imapsql.go b/internal/storage/imapsql/imapsql.go index ed12bcd..4ebc731 100644 --- a/internal/storage/imapsql/imapsql.go +++ b/internal/storage/imapsql/imapsql.go @@ -39,6 +39,7 @@ import ( "github.com/emersion/go-imap" sortthread "github.com/emersion/go-imap-sortthread" "github.com/emersion/go-imap/backend" + mess "github.com/foxcpp/go-imap-mess" imapsql "github.com/foxcpp/go-imap-sql" "github.com/foxcpp/maddy/framework/config" modconfig "github.com/foxcpp/maddy/framework/config/module" @@ -64,9 +65,9 @@ type Storage struct { resolver dns.Resolver - updates <-chan backend.Update - updPipe updatepipe.P - updPushStop chan struct{} + updPipe updatepipe.P + updPushStop chan struct{} + outboundUpds chan mess.Update filters module.IMAPFilter @@ -111,11 +112,7 @@ func (store *Storage) Init(cfg *config.Map) error { blobStore module.BlobStore ) - opts := imapsql.Opts{ - // Prevent deadlock if nobody is listening for updates (e.g. no IMAP - // configured). - LazyUpdatesInit: true, - } + opts := imapsql.Opts{} cfg.String("driver", false, false, store.driver, &driver) cfg.StringList("dsn", false, false, store.dsn, &dsn) cfg.Callback("fsstore", func(m *config.Map, node config.Node) error { @@ -139,7 +136,7 @@ func (store *Storage) Init(cfg *config.Map) error { cfg.Bool("debug", true, false, &store.Log.Debug) cfg.Int("sqlite3_cache_size", false, false, 0, &opts.CacheSize) cfg.Int("sqlite3_busy_timeout", false, false, 5000, &opts.BusyTimeout) - cfg.Bool("sqlite3_exclusive_lock", false, false, &opts.ExclusiveLock) + cfg.Bool("disable_recent", false, true, &opts.DisableRecent) cfg.String("junk_mailbox", false, false, "Junk", &store.junkMbox) cfg.Custom("imap_filter", false, false, func() (interface{}, error) { return nil, nil @@ -245,29 +242,28 @@ func (store *Storage) EnableUpdatePipe(mode updatepipe.BackendMode) error { if store.updPipe != nil { return nil } - if store.updates != nil { - panic("imapsql: EnableUpdatePipe called after Updates") - } - - upds := store.Back.Updates() switch store.driver { case "sqlite3": dbId := sha1.Sum([]byte(strings.Join(store.dsn, " "))) + sockPath := filepath.Join( + config.RuntimeDirectory, + fmt.Sprintf("sql-%s.sock", hex.EncodeToString(dbId[:]))) + store.Log.DebugMsg("using unix socket for external updates", "path", sockPath) store.updPipe = &updatepipe.UnixSockPipe{ - SockPath: filepath.Join( - config.RuntimeDirectory, - fmt.Sprintf("sql-%s.sock", hex.EncodeToString(dbId[:]))), + SockPath: sockPath, Log: log.Logger{Name: "sql/updpipe", Debug: store.Log.Debug}, } default: return errors.New("imapsql: driver does not have an update pipe implementation") } - wrapped := make(chan backend.Update, cap(upds)*2) + inbound := make(chan mess.Update, 32) + outbound := make(chan mess.Update, 10) + store.outboundUpds = outbound if mode == updatepipe.ModeReplicate { - if err := store.updPipe.Listen(wrapped); err != nil { + if err := store.updPipe.Listen(inbound); err != nil { store.updPipe = nil return err } @@ -278,11 +274,18 @@ func (store *Storage) EnableUpdatePipe(mode updatepipe.BackendMode) error { return err } - store.updPushStop = make(chan struct{}) + store.Back.UpdateManager().SetExternalSink(outbound) + + store.updPushStop = make(chan struct{}, 1) go func() { defer func() { + // Ensure we sent all outbound updates. + for upd := range outbound { + if err := store.updPipe.Push(upd); err != nil { + store.Log.Error("IMAP update pipe push failed", err) + } + } store.updPushStop <- struct{}{} - close(wrapped) if err := recover(); err != nil { stack := debug.Stack() @@ -292,27 +295,21 @@ func (store *Storage) EnableUpdatePipe(mode updatepipe.BackendMode) error { for { select { - case <-store.updPushStop: - return - case u := <-upds: - if u == nil { - // The channel is closed. We must be stopping now. - <-store.updPushStop + case u := <-inbound: + store.Log.DebugMsg("external update received", "type", u.Type, "key", u.Key) + store.Back.UpdateManager().ExternalUpdate(u) + case u, ok := <-outbound: + if !ok { return } - + store.Log.DebugMsg("sending external update", "type", u.Type, "key", u.Key) if err := store.updPipe.Push(u); err != nil { store.Log.Error("IMAP update pipe push failed", err) } - - if mode != updatepipe.ModePush { - wrapped <- u - } } } }() - store.updates = wrapped return nil } @@ -328,15 +325,6 @@ func (store *Storage) CreateMessageLimit() *uint32 { return store.Back.CreateMessageLimit() } -func (store *Storage) Updates() <-chan backend.Update { - if store.updates != nil { - return store.updates - } - - store.updates = store.Back.Updates() - return store.updates -} - func (store *Storage) EnableChildrenExt() bool { return store.Back.EnableChildrenExt() } @@ -378,7 +366,7 @@ func (store *Storage) Close() error { // all updates before shuting down (this is especially important for // maddyctl). if store.updPipe != nil { - store.updPushStop <- struct{}{} + close(store.outboundUpds) <-store.updPushStop store.updPipe.Close() diff --git a/internal/updatepipe/serialize.go b/internal/updatepipe/serialize.go index 8e7b63b..d5a941f 100644 --- a/internal/updatepipe/serialize.go +++ b/internal/updatepipe/serialize.go @@ -22,10 +22,10 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "strings" - "github.com/emersion/go-imap" - "github.com/emersion/go-imap/backend" + mess "github.com/foxcpp/go-imap-mess" ) func unescapeName(s string) string { @@ -36,95 +36,34 @@ func escapeName(s string) string { return strings.ReplaceAll(s, ";", "\x10") } -type message struct { - SeqNum uint32 - Flags []string -} - -func parseUpdate(s string) (id string, upd backend.Update, err error) { - parts := strings.SplitN(s, ";", 5) - if len(parts) != 5 { +func parseUpdate(s string) (id string, upd *mess.Update, err error) { + parts := strings.SplitN(s, ";", 2) + if len(parts) != 2 { return "", nil, errors.New("updatepipe: mismatched parts count") } - updBase := backend.NewUpdate(unescapeName(parts[2]), unescapeName(parts[3])) - switch parts[1] { - case "ExpungeUpdate": - exUpd := &backend.ExpungeUpdate{Update: updBase} - if err := json.Unmarshal([]byte(parts[4]), &exUpd.SeqNum); err != nil { - return "", nil, err - } - upd = exUpd - case "MailboxUpdate": - mboxUpd := &backend.MailboxUpdate{Update: updBase} - if err := json.Unmarshal([]byte(parts[4]), &mboxUpd.MailboxStatus); err != nil { - return "", nil, err - } - upd = mboxUpd - case "MessageUpdate": - // imap.Message is not JSON-serializable because it contains maps with - // complex keys. - // In practice, however, MessageUpdate is used only for FLAGS, so we - // serialize them only with a SeqNum. + upd = &mess.Update{} + dec := json.NewDecoder(strings.NewReader(unescapeName(parts[1]))) + dec.UseNumber() + err = dec.Decode(upd) + if err != nil { + return "", nil, fmt.Errorf("parseUpdate: %w", err) + } - msg := message{} - if err := json.Unmarshal([]byte(parts[4]), &msg); err != nil { - return "", nil, err - } - - msgUpd := &backend.MessageUpdate{ - Update: updBase, - Message: imap.NewMessage(msg.SeqNum, []imap.FetchItem{imap.FetchFlags}), - } - msgUpd.Message.Flags = msg.Flags - upd = msgUpd + if val, ok := upd.Key.(json.Number); ok { + upd.Key, _ = strconv.ParseUint(val.String(), 10, 64) } return parts[0], upd, nil } -func formatUpdate(myID string, upd backend.Update) (string, error) { - var ( - objType string - objStr []byte - err error - ) - switch v := upd.(type) { - case *backend.ExpungeUpdate: - objType = "ExpungeUpdate" - objStr, err = json.Marshal(v.SeqNum) - if err != nil { - return "", err - } - case *backend.MessageUpdate: - // imap.Message is not JSON-serializable because it contains maps with - // complex keys. - // In practice, however, MessageUpdate is used only for FLAGS, so we - // serialize them only with a seqnum. - - objType = "MessageUpdate" - objStr, err = json.Marshal(message{ - SeqNum: v.Message.SeqNum, - Flags: v.Message.Flags, - }) - if err != nil { - return "", err - } - case *backend.MailboxUpdate: - objType = "MailboxUpdate" - objStr, err = json.Marshal(v.MailboxStatus) - if err != nil { - return "", err - } - default: - return "", fmt.Errorf("updatepipe: unknown update type: %T", upd) +func formatUpdate(myID string, upd mess.Update) (string, error) { + updBlob, err := json.Marshal(upd) + if err != nil { + return "", fmt.Errorf("formatUpdate: %w", err) } - return strings.Join([]string{ myID, - objType, - escapeName(upd.Username()), - escapeName(upd.Mailbox()), - string(objStr), + escapeName(string(updBlob)), }, ";") + "\n", nil } diff --git a/internal/updatepipe/unix_pipe.go b/internal/updatepipe/unix_pipe.go index 9e6ff66..d2e2ce8 100644 --- a/internal/updatepipe/unix_pipe.go +++ b/internal/updatepipe/unix_pipe.go @@ -25,7 +25,7 @@ import ( "net" "os" - "github.com/emersion/go-imap/backend" + mess "github.com/foxcpp/go-imap-mess" "github.com/foxcpp/maddy/framework/log" ) @@ -34,11 +34,9 @@ import ( // Listen goroutine can be running. // // The socket is stream-oriented and consists of the following messages: -// OBJ_ID;TYPE_NAME;USER;MAILBOX;JSON_SERIALIZED_INTERNAL_OBJECT\n +// SENDER_ID;JSON_SERIALIZED_INTERNAL_OBJECT\n // -// Where TYPE_NAME is one of the folow: ExpungeUpdate, MailboxUpdate, -// MessageUpdate. -// And OBJ_ID is Process ID and UnixSockPipe address concated as a string. +// And SENDER_ID is Process ID and UnixSockPipe address concated as a string. // It is used to deduplicate updates sent to Push and recevied via Listen. // // The SockPath field specifies the socket path to use. The actual socket @@ -57,7 +55,7 @@ func (usp *UnixSockPipe) myID() string { return fmt.Sprintf("%d-%p", os.Getpid(), usp) } -func (usp *UnixSockPipe) readUpdates(conn net.Conn, updCh chan<- backend.Update) { +func (usp *UnixSockPipe) readUpdates(conn net.Conn, updCh chan<- mess.Update) { scnr := bufio.NewScanner(conn) for scnr.Scan() { id, upd, err := parseUpdate(scnr.Text()) @@ -70,17 +68,11 @@ func (usp *UnixSockPipe) readUpdates(conn net.Conn, updCh chan<- backend.Update) continue } - updCh <- upd + updCh <- *upd } } -func (usp *UnixSockPipe) Wrap(upd <-chan backend.Update) chan backend.Update { - ourUpds := make(chan backend.Update, cap(upd)) - - return ourUpds -} - -func (usp *UnixSockPipe) Listen(upd chan<- backend.Update) error { +func (usp *UnixSockPipe) Listen(upd chan<- mess.Update) error { l, err := net.Listen("unix", usp.SockPath) if err != nil { return err @@ -108,7 +100,7 @@ func (usp *UnixSockPipe) InitPush() error { return nil } -func (usp *UnixSockPipe) Push(upd backend.Update) error { +func (usp *UnixSockPipe) Push(upd mess.Update) error { if usp.sender == nil { if err := usp.InitPush(); err != nil { return err diff --git a/internal/updatepipe/update_pipe.go b/internal/updatepipe/update_pipe.go index 57735ed..5320177 100644 --- a/internal/updatepipe/update_pipe.go +++ b/internal/updatepipe/update_pipe.go @@ -29,7 +29,7 @@ along with this program. If not, see . package updatepipe import ( - "github.com/emersion/go-imap/backend" + mess "github.com/foxcpp/go-imap-mess" ) // The P interface represents the handle for a transport medium used for IMAP @@ -43,7 +43,7 @@ type P interface { // // Updates sent using the same UpdatePipe object using Push are not // duplicates to the channel passed to Listen. - Listen(upds chan<- backend.Update) error + Listen(upds chan<- mess.Update) error // InitPush prepares the UpdatePipe to be used as updates source (Push // method). @@ -56,7 +56,7 @@ type P interface { // // The update will not be duplicated if the UpdatePipe is also listening // for updates. - Push(upd backend.Update) error + Push(upd mess.Update) error Close() error }