diff --git a/server.go b/server.go
index 1df61d17..a6fa2713 100644
--- a/server.go
+++ b/server.go
@@ -436,6 +436,7 @@ func (s *baseServer) handleInitialImpl(p *receivedPacket, hdr *wire.Header) erro
 	if queueLen := atomic.LoadInt32(&s.sessionQueueLen); queueLen >= protocol.MaxAcceptQueueSize {
 		s.logger.Debugf("Rejecting new connection. Server currently busy. Accept queue length: %d (max %d)", queueLen, protocol.MaxAcceptQueueSize)
 		go func() {
+			defer p.buffer.Release()
 			if err := s.sendConnectionRefused(p.remoteAddr, hdr); err != nil {
 				s.logger.Debugf("Error rejecting connection: %s", err)
 			}