implement ECN path validation logic, send ECN-marked 1-RTT packets

This commit is contained in:
Marten Seemann 2023-09-01 14:43:51 +07:00
parent ffe6546833
commit f9cfa248da
3 changed files with 479 additions and 8 deletions

258
internal/ackhandler/ecn.go Normal file
View file

@ -0,0 +1,258 @@
package ackhandler
import (
"fmt"
"github.com/quic-go/quic-go/internal/protocol"
"github.com/quic-go/quic-go/internal/utils"
"github.com/quic-go/quic-go/logging"
)
type ecnState uint8
const (
ecnStateInitial ecnState = iota
ecnStateTesting
ecnStateUnknown
ecnStateCapable
ecnStateFailed
)
// must fit into an uint8, otherwise numSentTesting and numLostTesting must have a larger type
const numECNTestingPackets = 10
// The ecnTracker performs ECN validation of a path.
// Once failed, it doesn't do any re-validation of the path.
// It is designed only work for 1-RTT packets, it doesn't handle multiple packet number spaces.
// In order to avoid revealing any internal state to on-path observers,
// callers should make sure to start using ECN (i.e. calling Mode) for the very first 1-RTT packet sent.
// The validation logic implemented here strictly follows the algorithm described in RFC 9000 section 13.4.2 and A.4.
type ecnTracker struct {
state ecnState
numSentTesting, numLostTesting uint8
firstTestingPacket protocol.PacketNumber
lastTestingPacket protocol.PacketNumber
firstCapablePacket protocol.PacketNumber
numSentECT0, numSentECT1 int64
numAckedECT0, numAckedECT1, numAckedECNCE int64
tracer logging.ConnectionTracer
logger utils.Logger
}
func newECNTracker(logger utils.Logger, tracer logging.ConnectionTracer) *ecnTracker {
return &ecnTracker{
firstTestingPacket: protocol.InvalidPacketNumber,
lastTestingPacket: protocol.InvalidPacketNumber,
firstCapablePacket: protocol.InvalidPacketNumber,
state: ecnStateInitial,
logger: logger,
tracer: tracer,
}
}
func (e *ecnTracker) SentPacket(pn protocol.PacketNumber, ecn protocol.ECN) {
//nolint:exhaustive // These are the only ones we need to take care of.
switch ecn {
case protocol.ECNNon:
return
case protocol.ECT0:
e.numSentECT0++
case protocol.ECT1:
e.numSentECT1++
case protocol.ECNUnsupported:
if e.state != ecnStateFailed {
panic("didn't expect ECN to be unsupported")
}
default:
panic(fmt.Sprintf("sent packet with unexpected ECN marking: %s", ecn))
}
if e.state == ecnStateCapable && e.firstCapablePacket == protocol.InvalidPacketNumber {
e.firstCapablePacket = pn
}
if e.state != ecnStateTesting {
return
}
e.numSentTesting++
if e.firstTestingPacket == protocol.InvalidPacketNumber {
e.firstTestingPacket = pn
}
if e.numSentECT0+e.numSentECT1 >= numECNTestingPackets {
if e.tracer != nil {
e.tracer.ECNStateUpdated(logging.ECNStateUnknown, logging.ECNTriggerNoTrigger)
}
e.state = ecnStateUnknown
e.lastTestingPacket = pn
}
}
func (e *ecnTracker) Mode() protocol.ECN {
switch e.state {
case ecnStateInitial:
if e.tracer != nil {
e.tracer.ECNStateUpdated(logging.ECNStateTesting, logging.ECNTriggerNoTrigger)
}
e.state = ecnStateTesting
return e.Mode()
case ecnStateTesting, ecnStateCapable:
return protocol.ECT0
case ecnStateUnknown, ecnStateFailed:
return protocol.ECNNon
default:
panic(fmt.Sprintf("unknown ECN state: %d", e.state))
}
}
func (e *ecnTracker) LostPacket(pn protocol.PacketNumber) {
if e.state != ecnStateTesting && e.state != ecnStateUnknown {
return
}
if !e.isTestingPacket(pn) {
return
}
e.numLostTesting++
if e.numLostTesting >= e.numSentTesting {
e.logger.Debugf("Disabling ECN. All testing packets were lost.")
if e.tracer != nil {
e.tracer.ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedLostAllTestingPackets)
}
e.state = ecnStateFailed
}
}
// HandleNewlyAcked handles the ECN counts on an ACK frame.
// It must only be called for ACK frames that increase the largest acknowledged packet number,
// see section 13.4.2.1 of RFC 9000.
func (e *ecnTracker) HandleNewlyAcked(packets []*packet, ect0, ect1, ecnce int64) (congested bool) {
if e.state == ecnStateFailed {
return false
}
// ECN validation can fail if the received total count for either ECT(0) or ECT(1) exceeds
// the total number of packets sent with each corresponding ECT codepoint.
if ect0 > e.numSentECT0 || ect1 > e.numSentECT1 {
e.logger.Debugf("Disabling ECN. Received more ECT(0) / ECT(1) acknowledgements than packets sent.")
if e.tracer != nil {
e.tracer.ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedMoreECNCountsThanSent)
}
e.state = ecnStateFailed
return false
}
// Count ECT0 and ECT1 marks that we used when sending the packets that are now being acknowledged.
var ackedECT0, ackedECT1 int64
for _, p := range packets {
//nolint:exhaustive // We only ever send ECT(0) and ECT(1).
switch e.ecnMarking(p.PacketNumber) {
case protocol.ECT0:
ackedECT0++
case protocol.ECT1:
ackedECT1++
}
}
// If an ACK frame newly acknowledges a packet that the endpoint sent with either the ECT(0) or ECT(1)
// codepoint set, ECN validation fails if the corresponding ECN counts are not present in the ACK frame.
// This check detects:
// * paths that bleach all ECN marks, and
// * peers that don't report any ECN counts
if (ackedECT0 > 0 || ackedECT1 > 0) && ect0 == 0 && ect1 == 0 && ecnce == 0 {
e.logger.Debugf("Disabling ECN. ECN-marked packet acknowledged, but no ECN counts on ACK frame.")
if e.tracer != nil {
e.tracer.ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedNoECNCounts)
}
e.state = ecnStateFailed
return false
}
// Determine the increase in ECT0, ECT1 and ECNCE marks
newECT0 := ect0 - e.numAckedECT0
newECT1 := ect1 - e.numAckedECT1
newECNCE := ecnce - e.numAckedECNCE
// We're only processing ACKs that increase the Largest Acked.
// Therefore, the ECN counters should only ever increase.
// Any decrease means that the peer's counting logic is broken.
if newECT0 < 0 || newECT1 < 0 || newECNCE < 0 {
e.logger.Debugf("Disabling ECN. ECN counts decreased unexpectedly.")
if e.tracer != nil {
e.tracer.ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedDecreasedECNCounts)
}
e.state = ecnStateFailed
return false
}
// ECN validation also fails if the sum of the increase in ECT(0) and ECN-CE counts is less than the number
// of newly acknowledged packets that were originally sent with an ECT(0) marking.
// This could be the result of (partial) bleaching.
if newECT0+newECNCE < ackedECT0 {
e.logger.Debugf("Disabling ECN. Received less ECT(0) + ECN-CE than packets sent with ECT(0).")
if e.tracer != nil {
e.tracer.ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedTooFewECNCounts)
}
e.state = ecnStateFailed
return false
}
// Similarly, ECN validation fails if the sum of the increases to ECT(1) and ECN-CE counts is less than
// the number of newly acknowledged packets sent with an ECT(1) marking.
if newECT1+newECNCE < ackedECT1 {
e.logger.Debugf("Disabling ECN. Received less ECT(1) + ECN-CE than packets sent with ECT(1).")
if e.tracer != nil {
e.tracer.ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedTooFewECNCounts)
}
e.state = ecnStateFailed
return false
}
if e.state == ecnStateTesting || e.state == ecnStateUnknown {
var ackedTestingPacket bool
for _, p := range packets {
if e.isTestingPacket(p.PacketNumber) {
ackedTestingPacket = true
break
}
}
// This check won't succeed if the path is mangling ECN-marks (i.e. rewrites all ECN-marked packets to CE).
if ackedTestingPacket && (newECT0 > 0 || newECT1 > 0) {
e.logger.Debugf("ECN capability confirmed.")
if e.tracer != nil {
e.tracer.ECNStateUpdated(logging.ECNStateCapable, logging.ECNTriggerNoTrigger)
}
e.state = ecnStateCapable
}
}
// update our counters
e.numAckedECT0 = ect0
e.numAckedECT1 = ect1
e.numAckedECNCE = ecnce
return newECNCE > 0
}
func (e *ecnTracker) ecnMarking(pn protocol.PacketNumber) protocol.ECN {
if pn < e.firstTestingPacket || e.firstTestingPacket == protocol.InvalidPacketNumber {
return protocol.ECNNon
}
if pn < e.lastTestingPacket || e.lastTestingPacket == protocol.InvalidPacketNumber {
return protocol.ECT0
}
if pn < e.firstCapablePacket || e.firstCapablePacket == protocol.InvalidPacketNumber {
return protocol.ECNNon
}
// We don't need to deal with the case when ECN validation fails,
// since we're ignoring any ECN counts reported in ACK frames in that case.
return protocol.ECT0
}
func (e *ecnTracker) isTestingPacket(pn protocol.PacketNumber) bool {
if e.firstTestingPacket == protocol.InvalidPacketNumber {
return false
}
return pn >= e.firstTestingPacket && (pn <= e.lastTestingPacket || e.lastTestingPacket == protocol.InvalidPacketNumber)
}

View file

@ -0,0 +1,192 @@
package ackhandler
import (
mocklogging "github.com/quic-go/quic-go/internal/mocks/logging"
"github.com/quic-go/quic-go/internal/protocol"
"github.com/quic-go/quic-go/internal/utils"
"github.com/quic-go/quic-go/logging"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("ECN tracker", func() {
var ecnTracker *ecnTracker
var tracer *mocklogging.MockConnectionTracer
getAckedPackets := func(pns ...protocol.PacketNumber) []*packet {
var packets []*packet
for _, p := range pns {
packets = append(packets, &packet{PacketNumber: p})
}
return packets
}
BeforeEach(func() {
tracer = mocklogging.NewMockConnectionTracer(mockCtrl)
ecnTracker = newECNTracker(utils.DefaultLogger, tracer)
})
It("sends exactly 10 testing packets", func() {
tracer.EXPECT().ECNStateUpdated(logging.ECNStateTesting, logging.ECNTriggerNoTrigger)
for i := 0; i < 9; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECT0))
// Do this twice to make sure only sent packets are counted
Expect(ecnTracker.Mode()).To(Equal(protocol.ECT0))
ecnTracker.SentPacket(protocol.PacketNumber(10+i), protocol.ECT0)
}
Expect(ecnTracker.Mode()).To(Equal(protocol.ECT0))
tracer.EXPECT().ECNStateUpdated(logging.ECNStateUnknown, logging.ECNTriggerNoTrigger)
ecnTracker.SentPacket(20, protocol.ECT0)
// In unknown state, packets shouldn't be ECN-marked.
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
})
sendAllTestingPackets := func() {
tracer.EXPECT().ECNStateUpdated(logging.ECNStateTesting, logging.ECNTriggerNoTrigger)
tracer.EXPECT().ECNStateUpdated(logging.ECNStateUnknown, logging.ECNTriggerNoTrigger)
for i := 0; i < 10; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECT0))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECT0)
}
}
It("fails ECN validation if all ECN testing packets are lost", func() {
sendAllTestingPackets()
for i := 10; i < 20; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECNNon)
}
for i := 0; i < 9; i++ {
ecnTracker.LostPacket(protocol.PacketNumber(i))
}
// We don't care about the loss of non-testing packets
ecnTracker.LostPacket(15)
// Now lose the last testing packet.
tracer.EXPECT().ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedLostAllTestingPackets)
ecnTracker.LostPacket(9)
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
// We still don't care about more non-testing packets being lost
ecnTracker.LostPacket(16)
})
It("passes ECN validation when a testing packet is acknowledged, while still in testing state", func() {
tracer.EXPECT().ECNStateUpdated(logging.ECNStateTesting, logging.ECNTriggerNoTrigger)
for i := 0; i < 5; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECT0))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECT0)
}
tracer.EXPECT().ECNStateUpdated(logging.ECNStateCapable, logging.ECNTriggerNoTrigger)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(3), 1, 0, 0)).To(BeFalse())
// make sure we continue sending ECT(0) packets
for i := 5; i < 100; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECT0))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECT0)
}
})
It("passes ECN validation when a testing packet is acknowledged, while in unknown state", func() {
sendAllTestingPackets()
for i := 10; i < 20; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECNNon)
}
// Lose some packets to make sure this doesn't influence the outcome.
for i := 0; i < 5; i++ {
ecnTracker.LostPacket(protocol.PacketNumber(i))
}
tracer.EXPECT().ECNStateUpdated(logging.ECNStateCapable, logging.ECNTriggerNoTrigger)
Expect(ecnTracker.HandleNewlyAcked([]*packet{{PacketNumber: 7}}, 1, 0, 0)).To(BeFalse())
})
It("fails ECN validation when the ACK contains more ECN counts than we sent packets", func() {
sendAllTestingPackets()
for i := 10; i < 20; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECNNon)
}
// only 10 ECT(0) packets were sent, but the ACK claims to have received 12 of them
tracer.EXPECT().ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedMoreECNCountsThanSent)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), 12, 0, 0)).To(BeFalse())
})
It("fails ECN validation when the ACK contains ECN counts for the wrong code point", func() {
sendAllTestingPackets()
for i := 10; i < 20; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECNNon)
}
// We sent ECT(0), but this ACK acknowledges ECT(1).
tracer.EXPECT().ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedMoreECNCountsThanSent)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), 0, 1, 0)).To(BeFalse())
})
It("fails ECN validation when the ACK doesn't contain ECN counts", func() {
sendAllTestingPackets()
for i := 10; i < 20; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECNNon)
}
// First only acknowledge packets sent without ECN marks.
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(12, 13, 14), 0, 0, 0)).To(BeFalse())
// Now acknowledge some packets sent with ECN marks.
tracer.EXPECT().ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedNoECNCounts)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(1, 2, 3, 15), 0, 0, 0)).To(BeFalse())
})
It("fails ECN validation when an ACK decreases ECN counts", func() {
sendAllTestingPackets()
for i := 10; i < 20; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECNNon)
}
tracer.EXPECT().ECNStateUpdated(logging.ECNStateCapable, logging.ECNTriggerNoTrigger)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(1, 2, 3, 12), 3, 0, 0)).To(BeFalse())
// Now acknowledge some more packets, but decrease the ECN counts. Obviously, this doesn't make any sense.
tracer.EXPECT().ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedDecreasedECNCounts)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(4, 5, 6, 13), 2, 0, 0)).To(BeFalse())
// make sure that new ACKs are ignored
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(7, 8, 9, 14), 5, 0, 0)).To(BeFalse())
})
// This can happen if ACK are lost / reordered.
It("doesn't fail validation if the ACK contains more ECN counts than it acknowledges packets", func() {
sendAllTestingPackets()
for i := 10; i < 20; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECNNon)
}
tracer.EXPECT().ECNStateUpdated(logging.ECNStateCapable, logging.ECNTriggerNoTrigger)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(1, 2, 3, 12), 8, 0, 0)).To(BeFalse())
})
It("fails ECN validation when the ACK doesn't contain enough ECN counts", func() {
sendAllTestingPackets()
for i := 10; i < 20; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECNNon)
}
// First only acknowledge some packets sent with ECN marks.
tracer.EXPECT().ECNStateUpdated(logging.ECNStateCapable, logging.ECNTriggerNoTrigger)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(1, 2, 3, 12), 2, 0, 1)).To(BeTrue())
// Now acknowledge some more packets sent with ECN marks, but don't increase the counters enough.
// This ACK acknowledges 3 more ECN-marked packets, but the counters only increase by 2.
tracer.EXPECT().ECNStateUpdated(logging.ECNStateFailed, logging.ECNFailedTooFewECNCounts)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(4, 5, 6, 15), 3, 0, 2)).To(BeFalse())
})
It("declares congestion", func() {
sendAllTestingPackets()
for i := 10; i < 20; i++ {
Expect(ecnTracker.Mode()).To(Equal(protocol.ECNNon))
ecnTracker.SentPacket(protocol.PacketNumber(i), protocol.ECNNon)
}
// Receive one CE count.
tracer.EXPECT().ECNStateUpdated(logging.ECNStateCapable, logging.ECNTriggerNoTrigger)
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(1, 2, 3, 12), 2, 0, 1)).To(BeTrue())
// No increase in CE. No congestion.
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(4, 5, 6, 13), 5, 0, 1)).To(BeFalse())
// Increase in CE. More congestion.
Expect(ecnTracker.HandleNewlyAcked(getAckedPackets(7, 8, 9, 14), 7, 0, 2)).To(BeTrue())
})
})

View file

@ -92,7 +92,8 @@ type sentPacketHandler struct {
// The alarm timeout
alarm time.Time
enableECN bool
enableECN bool
ecnTracker *ecnTracker
perspective protocol.Perspective
@ -125,7 +126,7 @@ func newSentPacketHandler(
tracer,
)
return &sentPacketHandler{
h := &sentPacketHandler{
peerCompletedAddressValidation: pers == protocol.PerspectiveServer,
peerAddressValidated: pers == protocol.PerspectiveClient || clientAddressValidated,
initialPackets: newPacketNumberSpace(initialPN, false),
@ -133,11 +134,15 @@ func newSentPacketHandler(
appDataPackets: newPacketNumberSpace(0, true),
rttStats: rttStats,
congestion: congestion,
enableECN: enableECN,
perspective: pers,
tracer: tracer,
logger: logger,
}
if enableECN {
h.enableECN = true
h.ecnTracker = newECNTracker(logger, tracer)
}
return h
}
func (h *sentPacketHandler) removeFromBytesInFlight(p *packet) {
@ -232,7 +237,7 @@ func (h *sentPacketHandler) SentPacket(
streamFrames []StreamFrame,
frames []Frame,
encLevel protocol.EncryptionLevel,
_ protocol.ECN,
ecn protocol.ECN,
size protocol.ByteCount,
isPathMTUProbePacket bool,
) {
@ -257,6 +262,10 @@ func (h *sentPacketHandler) SentPacket(
}
h.congestion.OnPacketSent(t, h.bytesInFlight, pn, size, isAckEliciting)
if encLevel == protocol.Encryption1RTT && h.ecnTracker != nil {
h.ecnTracker.SentPacket(pn, ecn)
}
if !isAckEliciting {
pnSpace.history.SentNonAckElicitingPacket(pn)
if !h.peerCompletedAddressValidation {
@ -307,8 +316,6 @@ func (h *sentPacketHandler) ReceivedAck(ack *wire.AckFrame, encLevel protocol.En
}
}
pnSpace.largestAcked = utils.Max(pnSpace.largestAcked, largestAcked)
// Servers complete address validation when a protected packet is received.
if h.perspective == protocol.PerspectiveClient && !h.peerCompletedAddressValidation &&
(encLevel == protocol.EncryptionHandshake || encLevel == protocol.Encryption1RTT) {
@ -338,6 +345,18 @@ func (h *sentPacketHandler) ReceivedAck(ack *wire.AckFrame, encLevel protocol.En
h.congestion.MaybeExitSlowStart()
}
}
var ecnCongestionDetected bool
// Only inform the ECN tracker about new 1-RTT ACKs if the ACK increases the largest acked.
if encLevel == protocol.Encryption1RTT && h.ecnTracker != nil && largestAcked > pnSpace.largestAcked {
ecnCongestionDetected = h.ecnTracker.HandleNewlyAcked(ackedPackets, int64(ack.ECT0), int64(ack.ECT1), int64(ack.ECNCE))
}
pnSpace.largestAcked = utils.Max(pnSpace.largestAcked, largestAcked)
// TODO: inform the congestion controller
_ = ecnCongestionDetected
if err := h.detectLostPackets(rcvTime, encLevel); err != nil {
return false, err
}
@ -642,6 +661,9 @@ func (h *sentPacketHandler) detectLostPackets(now time.Time, encLevel protocol.E
if !p.IsPathMTUProbePacket {
h.congestion.OnPacketLost(p.PacketNumber, p.Length, priorInFlight)
}
if encLevel == protocol.Encryption1RTT && h.ecnTracker != nil {
h.ecnTracker.LostPacket(p.PacketNumber)
}
}
}
return true, nil
@ -724,8 +746,7 @@ func (h *sentPacketHandler) ECNMode(isShortHeaderPacket bool) protocol.ECN {
if !isShortHeaderPacket {
return protocol.ECNNon
}
// TODO: implement ECN logic
return protocol.ECNNon
return h.ecnTracker.Mode()
}
func (h *sentPacketHandler) PeekPacketNumber(encLevel protocol.EncryptionLevel) (protocol.PacketNumber, protocol.PacketNumberLen) {