diff --git a/internal/wire/extended_header.go b/internal/wire/extended_header.go
index b95c9cc6..19a0b064 100644
--- a/internal/wire/extended_header.go
+++ b/internal/wire/extended_header.go
@@ -15,7 +15,6 @@ type ExtendedHeader struct {
 	Header
 
 	typeByte byte
-	Raw      []byte
 
 	PacketNumberLen protocol.PacketNumberLen
 	PacketNumber    protocol.PacketNumber
diff --git a/packet_unpacker.go b/packet_unpacker.go
index a999f0a7..67f15503 100644
--- a/packet_unpacker.go
+++ b/packet_unpacker.go
@@ -69,18 +69,16 @@ func (u *packetUnpacker) Unpack(hdr *wire.Header, data []byte) (*unpackedPacket,
 		&data[0],
 		data[hdrLen:hdrLen+4],
 	)
-
 	// 3. parse the header (and learn the actual length of the packet number)
 	extHdr, err := hdr.ParseExtended(r, u.version)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing extended header: %s", err)
 	}
-	extHdr.Raw = data[:hdrLen+int(extHdr.PacketNumberLen)]
+	extHdrLen := hdrLen + int(extHdr.PacketNumberLen)
 	// 4. if the packet number is shorter than 4 bytes, replace the remaining bytes with the copy we saved earlier
 	if extHdr.PacketNumberLen != protocol.PacketNumberLen4 {
-		copy(data[hdrLen+int(extHdr.PacketNumberLen):hdrLen+4], origPNBytes[int(extHdr.PacketNumberLen):])
+		copy(data[extHdrLen:hdrLen+4], origPNBytes[int(extHdr.PacketNumberLen):])
 	}
-	data = data[hdrLen+int(extHdr.PacketNumberLen):]
 
 	pn := protocol.DecodePacketNumber(
 		extHdr.PacketNumberLen,
@@ -88,7 +86,7 @@ func (u *packetUnpacker) Unpack(hdr *wire.Header, data []byte) (*unpackedPacket,
 		extHdr.PacketNumber,
 	)
 
-	decrypted, err := opener.Open(data[:0], data, pn, extHdr.Raw)
+	decrypted, err := opener.Open(data[extHdrLen:extHdrLen], data[extHdrLen:], pn, data[:extHdrLen])
 	if err != nil {
 		return nil, err
 	}
diff --git a/session_test.go b/session_test.go
index e6d9f102..4b1270f6 100644
--- a/session_test.go
+++ b/session_test.go
@@ -492,7 +492,6 @@ var _ = Describe("Session", func() {
 
 		It("informs the ReceivedPacketHandler", func() {
 			hdr := &wire.ExtendedHeader{
-				Raw:             []byte("raw header"),
 				PacketNumber:    0x37,
 				PacketNumberLen: protocol.PacketNumberLen1,
 			}