Implement the integration testing library

.gitignore

@@ -36,3 +36,6 @@ cmd/maddy/*mtasts-cache
 cmd/maddy/*queue
 
 build/
+
+tests/maddy.cover
+tests/maddy

tests/README.md

@@ -0,0 +1,17 @@
+# maddy integration testing
+
+## Tests structure
+
+The test library creates a temporary state and runtime directory, starts the
+server with the specified configuration file and lets you interact with it
+using a couple of convenient wrappers.
+
+## Running
+
+To run tests, use `go test -tags integration` in this directory. Make sure to
+have a maddy executable in the current working directory.
+Use `-integration.executable` if the executable is named different or is placed
+somewhere else.
+Use `-integration.coverprofile` to pass `-test.coverprofile
+your_value.RANDOM` to test executable. See `./build_cover.sh` to build a
+server executable instrumented with coverage counters.

tests/basic_test.go

@@ -0,0 +1,40 @@
+//+build integration
+
+package tests_test
+
+import (
+	"testing"
+
+	"github.com/foxcpp/maddy/tests"
+)
+
+func TestBasic(tt *testing.T) {
+	// This test is mostly intended to test whether the integration testing
+	// library is working as expected.
+
+	t := tests.NewT(tt)
+	t.DNS(nil)
+	t.Port("smtp")
+	t.Config(`
+		smtp tcp://127.0.0.1:{env:TEST_PORT_smtp} {
+			hostname mx.maddy.test
+			tls off
+
+			deliver_to dummy
+		}`)
+	t.Run(1)
+	defer t.Close()
+
+	conn := t.Conn("smtp")
+	defer conn.Close()
+	conn.ExpectPattern("220 mx.maddy.test *")
+	conn.Writeln("EHLO localhost")
+	conn.ExpectPattern("250-*")
+	conn.ExpectPattern("250-PIPELINING")
+	conn.ExpectPattern("250-8BITMIME")
+	conn.ExpectPattern("250-ENHANCEDSTATUSCODES")
+	conn.ExpectPattern("250-SMTPUTF8")
+	conn.ExpectPattern("250 SIZE *")
+	conn.Writeln("QUIT")
+	conn.ExpectPattern("221 *")
+}

tests/build_cover.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec go test -tags 'cover_main debugflags' -coverpkg 'github.com/foxcpp/maddy,github.com/foxcpp/maddy/pkg/...,github.com/foxcpp/maddy/internal/...' -cover -covermode atomic -c cover_test.go -o maddy.cover

tests/conn.go

@@ -0,0 +1,182 @@
+package tests
+
+import (
+	"bufio"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Conn is a helper that simplifies testing of text protocol interactions.
+type Conn struct {
+	T *T
+
+	WriteTimeout time.Duration
+	ReadTimeout  time.Duration
+
+	allowIOErr bool
+
+	Conn    net.Conn
+	Scanner *bufio.Scanner
+}
+
+// AllowIOErr toggles whether I/O errors should be returned to the caller of
+// Conn method or should immedately fail the test.
+//
+// By default (ok = false), the latter happens.
+func (c *Conn) AllowIOErr(ok bool) {
+	c.allowIOErr = ok
+}
+
+// Write writes the string to the connection socket.
+func (c *Conn) Write(s string) error {
+	c.T.Helper()
+
+	// Make sure the test will not accidentally hang waiting for I/O forever if
+	// the server breaks.
+	if err := c.Conn.SetWriteDeadline(time.Now().Add(c.WriteTimeout)); err != nil {
+		c.fatal("Cannot set write deadline: %v", err)
+	}
+	defer func() {
+		if err := c.Conn.SetWriteDeadline(time.Time{}); err != nil {
+			c.log('-', "Failed to reset connection deadline: %v", err)
+		}
+	}()
+
+	c.log('>', "%s", s)
+	if _, err := io.WriteString(c.Conn, s); err != nil {
+		if c.allowIOErr {
+			return err
+		}
+		c.fatal("Unexpected I/O error: %v", err)
+	}
+
+	return nil
+}
+
+func (c *Conn) Writeln(s string) error {
+	c.T.Helper()
+
+	return c.Write(s + "\r\n")
+}
+
+func (c *Conn) consumeLine() (string, error) {
+	c.T.Helper()
+
+	// Make sure the test will not accidentally hang waiting for I/O forever if
+	// the server breaks.
+	if err := c.Conn.SetReadDeadline(time.Now().Add(c.ReadTimeout)); err != nil {
+		c.fatal("Cannot set write deadline: %v", err)
+	}
+	defer func() {
+		if err := c.Conn.SetReadDeadline(time.Time{}); err != nil {
+			c.log('-', "Failed to reset connection deadline: %v", err)
+		}
+	}()
+
+	if !c.Scanner.Scan() {
+		if err := c.Scanner.Err(); err != nil {
+			if c.allowIOErr {
+				return "", err
+			}
+			c.fatal("Unexpected I/O error: %v", err)
+		}
+		if c.allowIOErr {
+			return "", io.EOF
+		}
+		c.fatal("Unexpected EOF")
+	}
+
+	c.log('<', "%v", c.Scanner.Text())
+
+	return c.Scanner.Text(), nil
+}
+
+// ExpectPattern reads a line from the connection socket and checks whether is
+// matches the supplied shell pattern (as defined by path.Match). The original
+// line is returned.
+func (c *Conn) ExpectPattern(pat string) (string, error) {
+	c.T.Helper()
+
+	line, err := c.consumeLine()
+	if err != nil {
+		return line, err
+	}
+
+	match, err := path.Match(pat, line)
+	if err != nil {
+		c.T.Fatal("Malformed pattern:", err)
+	}
+	if !match {
+		c.T.Fatal("Response line not matching the expected pattern, want", pat)
+	}
+
+	return line, nil
+}
+
+func (c *Conn) fatal(f string, args ...interface{}) {
+	c.log('-', f, args...)
+	c.T.FailNow()
+}
+
+func (c *Conn) error(f string, args ...interface{}) {
+	c.log('-', f, args...)
+	c.T.Fail()
+}
+
+func (c *Conn) log(direction rune, f string, args ...interface{}) {
+	local, remote := c.Conn.LocalAddr().(*net.TCPAddr), c.Conn.RemoteAddr().(*net.TCPAddr)
+	msg := strings.Builder{}
+	if local.IP.IsLoopback() {
+		msg.WriteString(strconv.Itoa(local.Port))
+	} else {
+		msg.WriteString(local.String())
+	}
+
+	msg.WriteRune(' ')
+	msg.WriteRune(direction)
+	msg.WriteRune(' ')
+
+	if remote.IP.IsLoopback() {
+		textPort := c.T.portsRev[uint16(remote.Port)]
+		if textPort != "" {
+			msg.WriteString(textPort)
+		} else {
+			msg.WriteString(strconv.Itoa(remote.Port))
+		}
+
+	} else {
+		msg.WriteString(local.String())
+	}
+
+	if _, ok := c.Conn.(*tls.Conn); ok {
+		msg.WriteString(" [tls]")
+	}
+	msg.WriteString(": ")
+	fmt.Fprintf(&msg, f, args...)
+	c.T.Log(strings.TrimRight(msg.String(), "\r\n "))
+}
+
+func (c *Conn) TLS() {
+	c.T.Helper()
+
+	tlsC := tls.Client(c.Conn, &tls.Config{
+		ServerName:         "maddy.test",
+		InsecureSkipVerify: true,
+	})
+	if err := tlsC.Handshake(); err != nil {
+		c.fatal("TLS handshake fail: %v", err)
+	}
+
+	c.Conn = tlsC
+	c.Scanner = bufio.NewScanner(c.Conn)
+}
+
+func (c *Conn) Close() error {
+	return c.Conn.Close()
+}

tests/cover_test.go

@@ -0,0 +1,58 @@
+//+build cover_main
+
+package tests
+
+/*
+Go toolchain lacks the ability to instrument arbitrary executables with
+coverage counters.
+
+This file wraps the maddy executable into a minimal layer of "test" logic to
+make 'go test' work for it and produce the coverage report.
+
+Use ./build_cover.sh to compile it into ./maddy.cover.
+
+References:
+https://stackoverflow.com/questions/43381335/how-to-capture-code-coverage-from-a-go-binary
+https://blog.cloudflare.com/go-coverage-with-external-tests/
+https://github.com/albertito/chasquid/blob/master/coverage_test.go
+*/
+
+import (
+	"os"
+	"testing"
+
+	"github.com/foxcpp/maddy"
+)
+
+func TestMain(m *testing.M) {
+	// -test.* flags are registered somewhere in init() in "testing" (?)
+	// so calling flag.Parse() in maddy.Run() catches them up.
+
+	// maddy.Run changes the working directory, we need to change it back so
+	// -test.coverprofile writes out profile in the right location.
+	wd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+
+	code := maddy.Run()
+
+	if err := os.Chdir(wd); err != nil {
+		panic(err)
+	}
+
+	// Silence output produced by "testing" runtime.
+	_, w, err := os.Pipe()
+	if err == nil {
+		os.Stderr = w
+		os.Stdout = w
+	}
+
+	// Even though we do not have any tests to run, we need to call out into
+	// "testing" to make it process flags and produce the coverage report.
+	m.Run()
+
+	// TestMain doc says we have to exit with a sensible status code on our
+	// own.
+	os.Exit(code)
+}

tests/gocovcat.go

@@ -0,0 +1,91 @@
+//usr/bin/env go run "$0" "$@"; exit $?
+//
+// From: https://git.lukeshu.com/go/cmd/gocovcat/
+//
+// +build ignore
+
+// Copyright 2017 Luke Shumaker <lukeshu@parabola.nu>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Command gocovcat combines multiple go cover runs, and prints the
+// result on stdout.
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+func handleErr(err error) {
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func main() {
+	modeBool := false
+	blocks := map[string]int{}
+	for _, filename := range os.Args[1:] {
+		file, err := os.Open(filename)
+		handleErr(err)
+		buf := bufio.NewScanner(file)
+		for buf.Scan() {
+			line := buf.Text()
+
+			if strings.HasPrefix(line, "mode: ") {
+				m := strings.TrimPrefix(line, "mode: ")
+				switch m {
+				case "set":
+					modeBool = true
+				case "count", "atomic":
+					// do nothing
+				default:
+					fmt.Fprintf(os.Stderr, "Unrecognized mode: %s\n", m)
+					os.Exit(1)
+				}
+			} else {
+				sp := strings.LastIndexByte(line, ' ')
+				block := line[:sp]
+				cntStr := line[sp+1:]
+				cnt, err := strconv.Atoi(cntStr)
+				handleErr(err)
+				blocks[block] += cnt
+			}
+		}
+		handleErr(buf.Err())
+	}
+	keys := make([]string, 0, len(blocks))
+	for key := range blocks {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	modeStr := "count"
+	if modeBool {
+		modeStr = "set"
+	}
+	fmt.Printf("mode: %s\n", modeStr)
+	for _, block := range keys {
+		cnt := blocks[block]
+		if modeBool && cnt > 1 {
+			cnt = 1
+		}
+		fmt.Printf("%s %d\n", block, cnt)
+	}
+}

tests/run.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+./build_cover.sh
+
+clean() {
+    rm -f /tmp/maddy-coverage-report*
+}
+trap clean EXIT
+
+go test -tags integration -integration.executable ./maddy.cover -integration.coverprofile /tmp/maddy-coverage-report "$@"
+go run gocovcat.go /tmp/maddy-coverage-report* > coverage.out

tests/t.go

@@ -0,0 +1,285 @@
+// Package tests provides the framework for integration testing of maddy.
+//
+// The packages core object is tests.T object that encapsulates all test
+// state. It runs the server using test-provided configuration file and acts as
+// a proxy for all interactions with the server.
+package tests
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/foxcpp/go-mockdns"
+)
+
+var (
+	TestBinary  = "./maddy"
+	CoverageOut string
+)
+
+type T struct {
+	*testing.T
+
+	testDir string
+	cfg     string
+
+	dnsServ  *mockdns.Server
+	ports    map[string]uint16
+	portsRev map[uint16]string
+
+	servProc *exec.Cmd
+}
+
+func NewT(t *testing.T) *T {
+	return &T{
+		T:        t,
+		ports:    map[string]uint16{},
+		portsRev: map[uint16]string{},
+	}
+}
+
+// Config sets the configuration to use for the server. It must be called
+// before Run.
+func (t *T) Config(cfg string) {
+	t.Helper()
+
+	if t.servProc != nil {
+		panic("tests: Config called after Run")
+	}
+
+	t.cfg = cfg
+}
+
+// DNS sets the DNS zones to emulate for the tested server instance.
+//
+// If it is not called before Run, DNS(nil) call is assumed which makes the
+// mockdns server respond with NXDOMAIN to all queries.
+func (t *T) DNS(zones map[string]mockdns.Zone) {
+	t.Helper()
+
+	if t.dnsServ != nil {
+		t.Log("NOTE: Multiple DNS calls, replacing the server instance...")
+		t.dnsServ.Close()
+	}
+
+	dnsServ, err := mockdns.NewServer(zones)
+	if err != nil {
+		t.Fatal("Test configuration failed:", err)
+	}
+	t.dnsServ = dnsServ
+}
+
+// Port allocates the random TCP port for use by test. It will made accessible
+// in the configuration via environment variables with name in the form
+// TEST_PORT_name.
+//
+// If there is a port with name remote_smtp, it will be passed as the value for
+// the -debug.smtpport parameter.
+func (t *T) Port(name string) uint16 {
+	if port := t.ports[name]; port != 0 {
+		return port
+	}
+
+	// TODO: Try to bind on port to test its usability.
+	port := rand.Int31n(45536) + 20000
+	t.ports[name] = uint16(port)
+	t.portsRev[uint16(port)] = name
+	return uint16(port)
+}
+
+// Run completes the configuration of test environment and starts the test server.
+//
+// T.Close should be called by the end of test to release any resources and
+// shutdown the server.
+//
+// The parameter waitListeners specifies the amount of listeners the server is
+// supposed to configure. Run() will block before all of them are up.
+func (t *T) Run(waitListeners int) {
+	if t.cfg == "" {
+		panic("tests: Run called without configuration set")
+	}
+	if t.dnsServ == nil {
+		// If there is no DNS zones set in test - start a server that will
+		// respond with NXDOMAIN to all queries to avoid accidentally leaking
+		// any DNS queries to the real world.
+		dnsServ, err := mockdns.NewServer(nil)
+		if err != nil {
+			t.Fatal("Test configuration failed:", err)
+		}
+		t.dnsServ = dnsServ
+	}
+
+	// Setup file system, create statedir, runtimedir, write out config.
+	testDir, err := ioutil.TempDir("", "maddy-tests-")
+	if err != nil {
+		t.Fatal("Test configuration failed:", err)
+	}
+	t.testDir = testDir
+
+	defer func() {
+		if !t.Failed() {
+			return
+		}
+
+		// Clean-up on test failure (if Run failed somewhere)
+
+		t.dnsServ.Close()
+		t.dnsServ = nil
+
+		os.RemoveAll(t.testDir)
+		t.testDir = ""
+	}()
+
+	if err := os.MkdirAll(filepath.Join(t.testDir, "statedir"), os.ModePerm); err != nil {
+		t.Fatal("Test configuration failed:", err)
+	}
+	if err := os.MkdirAll(filepath.Join(t.testDir, "runtimedir"), os.ModePerm); err != nil {
+		t.Fatal("Test configuration failed:", err)
+	}
+
+	configPreable := "state_dir " + filepath.Join(t.testDir, "statedir") + "\n" +
+		"runtime_dir " + filepath.Join(t.testDir, "runtime") + "\n\n"
+
+	ioutil.WriteFile(filepath.Join(t.testDir, "maddy.conf"), []byte(configPreable+t.cfg), os.ModePerm)
+	if err != nil {
+		t.Fatal("Test configuration failed:", err)
+	}
+
+	// Assigning 0 by default will make outbound SMTP unusable.
+	remoteSmtp := "0"
+	if port := t.ports["remote_smtp"]; port != 0 {
+		remoteSmtp = strconv.Itoa(int(port))
+	}
+
+	cmd := exec.Command(TestBinary,
+		"-config", filepath.Join(t.testDir, "maddy.conf"),
+		"-debug.smtpport", remoteSmtp,
+		"-debug.dnsoverride", t.dnsServ.LocalAddr().String(),
+		"-log", "stderr")
+
+	if CoverageOut != "" {
+		cmd.Args = append(cmd.Args, "-test.coverprofile", CoverageOut+"."+strconv.FormatInt(time.Now().UnixNano(), 16))
+	}
+
+	t.Logf("launching %v", cmd.Args)
+
+	// Set environment variables.
+	cmd.Env = []string{
+		"TEST_STATE_DIR=" + filepath.Join(t.testDir, "statedir"),
+		"TEST_RUNTIME_DIR=" + filepath.Join(t.testDir, "statedir"),
+	}
+	for name, port := range t.ports {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("TEST_PORT_%s=%d", name, port))
+	}
+
+	// Capture maddy log and redirect it.
+	logOut, err := cmd.StderrPipe()
+	if err != nil {
+		t.Fatal("Test configuration failed:", err)
+	}
+
+	if err := cmd.Start(); err != nil {
+		t.Fatal("Test configuration failed:", err)
+	}
+
+	// TODO: Mock the systemd notify socket and use it to detect start-up errors
+	// and other problems (?). Though, it would be Linux-specific.
+
+	// Log scanning goroutine checks for the "listening" messages and sends 'true'
+	// on the channel each time.
+	listeningMsg := make(chan bool)
+
+	go func() {
+		defer logOut.Close()
+		defer close(listeningMsg)
+		scnr := bufio.NewScanner(logOut)
+		for scnr.Scan() {
+			line := scnr.Text()
+
+			if strings.Contains(line, "listening on") {
+				listeningMsg <- true
+				line += " (test runner>listener wait trigger<)"
+			}
+
+			t.Log("maddy:", line)
+		}
+		if err := scnr.Err(); err != nil {
+			t.Log("stderr I/O error:", err)
+		}
+	}()
+
+	for i := 0; i < waitListeners; i++ {
+		if !<-listeningMsg {
+			t.Fatal("Log ended before all expected listeners are up. Start-up error?")
+		}
+	}
+
+	t.servProc = cmd
+}
+
+func (t *T) Close() {
+	if err := t.dnsServ.Close(); err != nil {
+		t.Log("Unable to stop the DNS server:", err)
+	}
+	t.dnsServ = nil
+
+	if err := t.servProc.Process.Signal(os.Interrupt); err != nil {
+		t.Log("Unable to kill the server process:", err)
+		os.RemoveAll(t.testDir)
+		return // Return, as now it is pointless to wait for it.
+	}
+
+	go func() {
+		time.Sleep(5 * time.Second)
+		if t.servProc != nil {
+			t.servProc.Process.Kill()
+		}
+	}()
+
+	if err := t.servProc.Wait(); err != nil {
+		t.Error("The server did not stop cleanly, deadlock?")
+	}
+
+	t.servProc = nil
+
+	if err := os.RemoveAll(t.testDir); err != nil {
+		t.Log("Failed to remove test directory:", err)
+	}
+	t.testDir = ""
+}
+
+func (t *T) Conn(portName string) Conn {
+	port := t.ports[portName]
+	if port == 0 {
+		panic("tests: connection for the unused port name is requested")
+	}
+
+	conn, err := net.Dial("tcp4", "127.0.0.1:"+strconv.Itoa(int(port)))
+	if err != nil {
+		t.Fatal("Could not connect, is server listening?", err)
+	}
+
+	return Conn{
+		T:            t,
+		WriteTimeout: 1 * time.Second,
+		ReadTimeout:  15 * time.Second,
+		Conn:         conn,
+		Scanner:      bufio.NewScanner(conn),
+	}
+}
+
+func init() {
+	flag.StringVar(&TestBinary, "integration.executable", "./maddy", "executable to test")
+	flag.StringVar(&CoverageOut, "integration.coverprofile", "", "write coverage stats to file (requires special maddy executable)")
+}