diff --git a/internal/ackhandler/received_packet_handler.go b/internal/ackhandler/received_packet_handler.go
index a8318e5c..bf60de37 100644
--- a/internal/ackhandler/received_packet_handler.go
+++ b/internal/ackhandler/received_packet_handler.go
@@ -159,13 +159,14 @@ func (h *receivedPacketHandler) maybeQueueAck(packetNumber protocol.PacketNumber
 }
 
 func (h *receivedPacketHandler) GetAckFrame() *wire.AckFrame {
-	if !h.ackQueued && (h.ackAlarm.IsZero() || h.ackAlarm.After(time.Now())) {
+	now := time.Now()
+	if !h.ackQueued && (h.ackAlarm.IsZero() || h.ackAlarm.After(now)) {
 		return nil
 	}
 
 	ack := &wire.AckFrame{
-		AckRanges:          h.packetHistory.GetAckRanges(),
-		PacketReceivedTime: h.largestObservedReceivedTime,
+		AckRanges: h.packetHistory.GetAckRanges(),
+		DelayTime: now.Sub(h.largestObservedReceivedTime),
 	}
 
 	h.lastAck = ack
diff --git a/internal/ackhandler/received_packet_handler_test.go b/internal/ackhandler/received_packet_handler_test.go
index c953de7b..e7cbc9e8 100644
--- a/internal/ackhandler/received_packet_handler_test.go
+++ b/internal/ackhandler/received_packet_handler_test.go
@@ -227,6 +227,16 @@ var _ = Describe("receivedPacketHandler", func() {
 				Expect(ack.HasMissingRanges()).To(BeFalse())
 			})
 
+			It("sets the delay time", func() {
+				err := handler.ReceivedPacket(1, time.Time{}, true)
+				Expect(err).ToNot(HaveOccurred())
+				err = handler.ReceivedPacket(2, time.Now().Add(-1337*time.Millisecond), true)
+				Expect(err).ToNot(HaveOccurred())
+				ack := handler.GetAckFrame()
+				Expect(ack).ToNot(BeNil())
+				Expect(ack.DelayTime).To(BeNumerically("~", 1337*time.Millisecond, 50*time.Millisecond))
+			})
+
 			It("saves the last sent ACK", func() {
 				err := handler.ReceivedPacket(1, time.Time{}, true)
 				Expect(err).ToNot(HaveOccurred())
diff --git a/internal/wire/ack_frame.go b/internal/wire/ack_frame.go
index 7b99ef40..021cef53 100644
--- a/internal/wire/ack_frame.go
+++ b/internal/wire/ack_frame.go
@@ -16,11 +16,7 @@ const ackDelayExponent = 3
 // An AckFrame is an ACK frame
 type AckFrame struct {
 	AckRanges []AckRange // has to be ordered. The highest ACK range goes first, the lowest ACK range goes last
-
-	// time when the LargestAcked was receiveid
-	// this field will not be set for received ACKs frames
-	PacketReceivedTime time.Time
-	DelayTime          time.Duration
+	DelayTime time.Duration
 }
 
 // parseAckFrame reads an ACK frame
diff --git a/internal/wire/ack_frame_legacy.go b/internal/wire/ack_frame_legacy.go
index 5b4ce855..c2a71e0b 100644
--- a/internal/wire/ack_frame_legacy.go
+++ b/internal/wire/ack_frame_legacy.go
@@ -191,7 +191,6 @@ func (f *AckFrame) writeLegacy(b *bytes.Buffer, _ protocol.VersionNumber) error
 		utils.BigEndian.WriteUint48(b, uint64(largestAcked)&(1<<48-1))
 	}
 
-	f.DelayTime = time.Since(f.PacketReceivedTime)
 	utils.BigEndian.WriteUfloat16(b, uint64(f.DelayTime/time.Microsecond))
 
 	var numRanges uint64
diff --git a/internal/wire/ack_frame_legacy_test.go b/internal/wire/ack_frame_legacy_test.go
index 6188e7d6..3be341a2 100644
--- a/internal/wire/ack_frame_legacy_test.go
+++ b/internal/wire/ack_frame_legacy_test.go
@@ -498,6 +498,7 @@ var _ = Describe("ACK Frame (for gQUIC)", func() {
 			It("writes a simple ACK frame", func() {
 				frameOrig := &AckFrame{
 					AckRanges: []AckRange{{Smallest: 1, Largest: 1}},
+					DelayTime: 876 * time.Microsecond,
 				}
 				err := frameOrig.Write(b, versionBigEndian)
 				Expect(err).ToNot(HaveOccurred())
@@ -506,6 +507,7 @@ var _ = Describe("ACK Frame (for gQUIC)", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(frame.LargestAcked()).To(Equal(frameOrig.LargestAcked()))
 				Expect(frame.HasMissingRanges()).To(BeFalse())
+				Expect(frame.DelayTime).To(Equal(frameOrig.DelayTime))
 				Expect(r.Len()).To(BeZero())
 			})
 
diff --git a/internal/wire/ack_frame_test.go b/internal/wire/ack_frame_test.go
index 5c382117..a4df8b24 100644
--- a/internal/wire/ack_frame_test.go
+++ b/internal/wire/ack_frame_test.go
@@ -150,7 +150,7 @@ var _ = Describe("ACK Frame (for IETF QUIC)", func() {
 			buf := &bytes.Buffer{}
 			f := &AckFrame{
 				AckRanges: []AckRange{{Smallest: 0xdeadbeef, Largest: 0xdeadbeef}},
-				DelayTime: 18 * time.Second,
+				DelayTime: 18 * time.Millisecond,
 			}
 			err := f.Write(buf, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
@@ -160,6 +160,7 @@ var _ = Describe("ACK Frame (for IETF QUIC)", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(frame).To(Equal(f))
 			Expect(frame.HasMissingRanges()).To(BeFalse())
+			Expect(frame.DelayTime).To(Equal(f.DelayTime))
 			Expect(b.Len()).To(BeZero())
 		})