From 74be4d2755eec6f9af63f63d7fbeb1c3c68a275c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 8 May 2023 14:19:57 +0300 Subject: [PATCH] add a function to set the UDP send buffer size This function is the equivalent to the function used to set the UDP receive buffer size. It's so similar that code generation is used to make a copy of the setReceiveBuffer function. --- .github/workflows/go-generate.sh | 4 ++ internal/protocol/params.go | 3 ++ sys_conn_buffers.go | 1 + sys_conn_buffers_write.go | 70 ++++++++++++++++++++++++++++++++ sys_conn_helper_linux.go | 10 +++++ sys_conn_helper_linux_test.go | 42 ++++++++++++++++--- sys_conn_helper_nonlinux.go | 3 +- sys_conn_no_oob.go | 3 +- sys_conn_oob.go | 11 +++++ sys_conn_windows.go | 11 +++++ 10 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 sys_conn_buffers_write.go diff --git a/.github/workflows/go-generate.sh b/.github/workflows/go-generate.sh index cd84098e..2d1b9447 100755 --- a/.github/workflows/go-generate.sh +++ b/.github/workflows/go-generate.sh @@ -11,6 +11,10 @@ cp -r "$DIR" generated cd generated # delete all go-generated files generated (that adhere to the comment convention) grep --include \*.go -lrIZ "^// Code generated .* DO NOT EDIT\.$" . | xargs --null rm + +# First regenerate sys_conn_buffers_write.go. +# If it doesn't exist, the following mockgen calls will fail. +go generate -run "sys_conn_buffers_write.go" # now generate everything go generate ./... cd .. diff --git a/internal/protocol/params.go b/internal/protocol/params.go index 60c86779..fe3a7562 100644 --- a/internal/protocol/params.go +++ b/internal/protocol/params.go @@ -5,6 +5,9 @@ import "time" // DesiredReceiveBufferSize is the kernel UDP receive buffer size that we'd like to use. const DesiredReceiveBufferSize = (1 << 20) * 2 // 2 MB +// DesiredSendBufferSize is the kernel UDP send buffer size that we'd like to use. +const DesiredSendBufferSize = (1 << 20) * 2 // 2 MB + // InitialPacketSizeIPv4 is the maximum packet size that we use for sending IPv4 packets. const InitialPacketSizeIPv4 = 1252 diff --git a/sys_conn_buffers.go b/sys_conn_buffers.go index 5d3b10b1..0c4e024a 100644 --- a/sys_conn_buffers.go +++ b/sys_conn_buffers.go @@ -10,6 +10,7 @@ import ( "github.com/quic-go/quic-go/internal/utils" ) +//go:generate sh -c "echo '// Code generated by go generate. DO NOT EDIT.\n// Source: sys_conn_buffers.go\n' > sys_conn_buffers_write.go && sed -e 's/SetReadBuffer/SetWriteBuffer/g' -e 's/setReceiveBuffer/setSendBuffer/g' -e 's/inspectReadBuffer/inspectWriteBuffer/g' -e 's/protocol\\.DesiredReceiveBufferSize/protocol\\.DesiredSendBufferSize/g' -e 's/forceSetReceiveBuffer/forceSetSendBuffer/g' -e 's/receive buffer/send buffer/g' sys_conn_buffers.go | sed '/^\\/\\/go:generate/d' >> sys_conn_buffers_write.go" func setReceiveBuffer(c net.PacketConn, logger utils.Logger) error { conn, ok := c.(interface{ SetReadBuffer(int) error }) if !ok { diff --git a/sys_conn_buffers_write.go b/sys_conn_buffers_write.go new file mode 100644 index 00000000..86fead96 --- /dev/null +++ b/sys_conn_buffers_write.go @@ -0,0 +1,70 @@ +// Code generated by go generate. DO NOT EDIT. +// Source: sys_conn_buffers.go + +package quic + +import ( + "errors" + "fmt" + "net" + "syscall" + + "github.com/quic-go/quic-go/internal/protocol" + "github.com/quic-go/quic-go/internal/utils" +) + +func setSendBuffer(c net.PacketConn, logger utils.Logger) error { + conn, ok := c.(interface{ SetWriteBuffer(int) error }) + if !ok { + return errors.New("connection doesn't allow setting of send buffer size. Not a *net.UDPConn?") + } + + var syscallConn syscall.RawConn + if sc, ok := c.(interface { + SyscallConn() (syscall.RawConn, error) + }); ok { + var err error + syscallConn, err = sc.SyscallConn() + if err != nil { + syscallConn = nil + } + } + // The connection has a SetWriteBuffer method, but we couldn't obtain a syscall.RawConn. + // This shouldn't happen for a net.UDPConn, but is possible if the connection just implements the + // net.PacketConn interface and the SetWriteBuffer method. + // We have no way of checking if increasing the buffer size actually worked. + if syscallConn == nil { + return conn.SetWriteBuffer(protocol.DesiredSendBufferSize) + } + + size, err := inspectWriteBuffer(syscallConn) + if err != nil { + return fmt.Errorf("failed to determine send buffer size: %w", err) + } + if size >= protocol.DesiredSendBufferSize { + logger.Debugf("Conn has send buffer of %d kiB (wanted: at least %d kiB)", size/1024, protocol.DesiredSendBufferSize/1024) + return nil + } + // Ignore the error. We check if we succeeded by querying the buffer size afterward. + _ = conn.SetWriteBuffer(protocol.DesiredSendBufferSize) + newSize, err := inspectWriteBuffer(syscallConn) + if newSize < protocol.DesiredSendBufferSize { + // Try again with RCVBUFFORCE on Linux + _ = forceSetSendBuffer(syscallConn, protocol.DesiredSendBufferSize) + newSize, err = inspectWriteBuffer(syscallConn) + if err != nil { + return fmt.Errorf("failed to determine send buffer size: %w", err) + } + } + if err != nil { + return fmt.Errorf("failed to determine send buffer size: %w", err) + } + if newSize == size { + return fmt.Errorf("failed to increase send buffer size (wanted: %d kiB, got %d kiB)", protocol.DesiredSendBufferSize/1024, newSize/1024) + } + if newSize < protocol.DesiredSendBufferSize { + return fmt.Errorf("failed to sufficiently increase send buffer size (was: %d kiB, wanted: %d kiB, got: %d kiB)", size/1024, protocol.DesiredSendBufferSize/1024, newSize/1024) + } + logger.Debugf("Increased send buffer size to %d kiB", newSize/1024) + return nil +} diff --git a/sys_conn_helper_linux.go b/sys_conn_helper_linux.go index 69e1a2d5..2c12233c 100644 --- a/sys_conn_helper_linux.go +++ b/sys_conn_helper_linux.go @@ -31,3 +31,13 @@ func forceSetReceiveBuffer(c syscall.RawConn, bytes int) error { } return serr } + +func forceSetSendBuffer(c syscall.RawConn, bytes int) error { + var serr error + if err := c.Control(func(fd uintptr) { + serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, bytes) + }); err != nil { + return err + } + return serr +} diff --git a/sys_conn_helper_linux_test.go b/sys_conn_helper_linux_test.go index dd095f95..fa39c523 100644 --- a/sys_conn_helper_linux_test.go +++ b/sys_conn_helper_linux_test.go @@ -13,8 +13,8 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("Can change the receive buffer size", func() { - It("Force a change (if we have CAP_NET_ADMIN)", func() { +var _ = Describe("forcing a change of send and receive buffer sizes", func() { + It("forces a change of the receive buffer size", func() { if os.Getuid() != 0 { Fail("Must be root to force change the receive buffer size") } @@ -24,17 +24,47 @@ var _ = Describe("Can change the receive buffer size", func() { defer c.Close() syscallConn, err := c.(*net.UDPConn).SyscallConn() Expect(err).ToNot(HaveOccurred()) - forceSetReceiveBuffer(syscallConn, 256<<10) + + const small = 256 << 10 // 256 KB + Expect(forceSetReceiveBuffer(syscallConn, small)).To(Succeed()) size, err := inspectReadBuffer(syscallConn) Expect(err).ToNot(HaveOccurred()) // The kernel doubles this value (to allow space for bookkeeping overhead) - Expect(size).To(Equal(512 << 10)) + Expect(size).To(Equal(2 * small)) - forceSetReceiveBuffer(syscallConn, 512<<10) + const large = 32 << 20 // 32 MB + Expect(forceSetReceiveBuffer(syscallConn, large)).To(Succeed()) size, err = inspectReadBuffer(syscallConn) Expect(err).ToNot(HaveOccurred()) // The kernel doubles this value (to allow space for bookkeeping overhead) - Expect(size).To(Equal(1024 << 10)) + Expect(size).To(Equal(2 * large)) + }) + + It("forces a change of the send buffer size", func() { + if os.Getuid() != 0 { + Fail("Must be root to force change the send buffer size") + } + + c, err := net.ListenPacket("udp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + defer c.Close() + syscallConn, err := c.(*net.UDPConn).SyscallConn() + Expect(err).ToNot(HaveOccurred()) + + const small = 256 << 10 // 256 KB + Expect(forceSetSendBuffer(syscallConn, small)).To(Succeed()) + + size, err := inspectWriteBuffer(syscallConn) + Expect(err).ToNot(HaveOccurred()) + // The kernel doubles this value (to allow space for bookkeeping overhead) + Expect(size).To(Equal(2 * small)) + + const large = 32 << 20 // 32 MB + Expect(forceSetSendBuffer(syscallConn, large)).To(Succeed()) + size, err = inspectWriteBuffer(syscallConn) + Expect(err).ToNot(HaveOccurred()) + // The kernel doubles this value (to allow space for bookkeeping overhead) + Expect(size).To(Equal(2 * large)) }) }) diff --git a/sys_conn_helper_nonlinux.go b/sys_conn_helper_nonlinux.go index 318cbd09..80b795c3 100644 --- a/sys_conn_helper_nonlinux.go +++ b/sys_conn_helper_nonlinux.go @@ -2,4 +2,5 @@ package quic -func forceSetReceiveBuffer(c interface{}, bytes int) error { return nil } +func forceSetReceiveBuffer(c any, bytes int) error { return nil } +func forceSetSendBuffer(c any, bytes int) error { return nil } diff --git a/sys_conn_no_oob.go b/sys_conn_no_oob.go index e5189b18..c5c7f486 100644 --- a/sys_conn_no_oob.go +++ b/sys_conn_no_oob.go @@ -8,6 +8,7 @@ func newConn(c net.PacketConn) (rawConn, error) { return &basicConn{PacketConn: c}, nil } -func inspectReadBuffer(any) (int, error) { return 0, nil } +func inspectReadBuffer(any) (int, error) { return 0, nil } +func inspectWriteBuffer(any) (int, error) { return 0, nil } func (i *packetInfo) OOB() []byte { return nil } diff --git a/sys_conn_oob.go b/sys_conn_oob.go index bc833efa..42e5f53b 100644 --- a/sys_conn_oob.go +++ b/sys_conn_oob.go @@ -42,6 +42,17 @@ func inspectReadBuffer(c syscall.RawConn) (int, error) { return size, serr } +func inspectWriteBuffer(c syscall.RawConn) (int, error) { + var size int + var serr error + if err := c.Control(func(fd uintptr) { + size, serr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_SNDBUF) + }); err != nil { + return 0, err + } + return size, serr +} + type oobConn struct { OOBCapablePacketConn batchConn batchConn diff --git a/sys_conn_windows.go b/sys_conn_windows.go index 205abd14..580d18a5 100644 --- a/sys_conn_windows.go +++ b/sys_conn_windows.go @@ -23,4 +23,15 @@ func inspectReadBuffer(c syscall.RawConn) (int, error) { return size, serr } +func inspectWriteBuffer(c syscall.RawConn) (int, error) { + var size int + var serr error + if err := c.Control(func(fd uintptr) { + size, serr = windows.GetsockoptInt(windows.Handle(fd), windows.SOL_SOCKET, windows.SO_SNDBUF) + }); err != nil { + return 0, err + } + return size, serr +} + func (i *packetInfo) OOB() []byte { return nil }