diff --git a/internal/wire/header_parser.go b/internal/wire/header_parser.go
index ce1ec0d9..15e1643f 100644
--- a/internal/wire/header_parser.go
+++ b/internal/wire/header_parser.go
@@ -62,12 +62,12 @@ func ParseInvariantHeader(b *bytes.Reader, shortHeaderConnIDLen int) (*Invariant
 }
 
 // Parse parses the version dependent part of the header
-func (iv *InvariantHeader) Parse(b *bytes.Reader, sentBy protocol.Perspective, ver protocol.VersionNumber) (*Header, error) {
+func (iv *InvariantHeader) Parse(b *bytes.Reader, ver protocol.VersionNumber) (*Header, error) {
 	if iv.IsLongHeader {
 		if iv.Version == 0 { // Version Negotiation Packet
 			return iv.parseVersionNegotiationPacket(b)
 		}
-		return iv.parseLongHeader(b, sentBy, ver)
+		return iv.parseLongHeader(b, ver)
 	}
 	return iv.parseShortHeader(b, ver)
 }
@@ -98,7 +98,7 @@ func (iv *InvariantHeader) parseVersionNegotiationPacket(b *bytes.Reader) (*Head
 	return h, nil
 }
 
-func (iv *InvariantHeader) parseLongHeader(b *bytes.Reader, sentBy protocol.Perspective, v protocol.VersionNumber) (*Header, error) {
+func (iv *InvariantHeader) parseLongHeader(b *bytes.Reader, v protocol.VersionNumber) (*Header, error) {
 	h := iv.toHeader()
 	h.Type = protocol.PacketType(iv.typeByte & 0x7f)
 
diff --git a/internal/wire/header_parser_test.go b/internal/wire/header_parser_test.go
index 9477723a..6915d6f5 100644
--- a/internal/wire/header_parser_test.go
+++ b/internal/wire/header_parser_test.go
@@ -31,7 +31,7 @@ var _ = Describe("Header Parsing", func() {
 			Expect(iHdr.DestConnectionID).To(Equal(destConnID))
 			Expect(iHdr.SrcConnectionID).To(Equal(srcConnID))
 			Expect(iHdr.IsLongHeader).To(BeTrue())
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+			hdr, err := iHdr.Parse(b, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(hdr.IsVersionNegotiation).To(BeTrue())
 			Expect(hdr.Version).To(BeZero())
@@ -50,7 +50,7 @@ var _ = Describe("Header Parsing", func() {
 			b := bytes.NewReader(data[:len(data)-2])
 			iHdr, err := ParseInvariantHeader(b, 0)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+			_, err = iHdr.Parse(b, versionIETFFrames)
 			Expect(err).To(MatchError(qerr.InvalidVersionNegotiationPacket))
 		})
 
@@ -63,7 +63,7 @@ var _ = Describe("Header Parsing", func() {
 			b := bytes.NewReader(data[:len(data)-8])
 			iHdr, err := ParseInvariantHeader(b, 0)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+			_, err = iHdr.Parse(b, versionIETFFrames)
 			Expect(err).To(MatchError("InvalidVersionNegotiationPacket: empty version list"))
 		})
 	})
@@ -91,7 +91,7 @@ var _ = Describe("Header Parsing", func() {
 			Expect(iHdr.IsLongHeader).To(BeTrue())
 			Expect(iHdr.DestConnectionID).To(Equal(destConnID))
 			Expect(iHdr.SrcConnectionID).To(Equal(srcConnID))
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+			hdr, err := iHdr.Parse(b, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(hdr.Type).To(Equal(protocol.PacketTypeInitial))
 			Expect(hdr.IsLongHeader).To(BeTrue())
@@ -151,7 +151,7 @@ var _ = Describe("Header Parsing", func() {
 			b := bytes.NewReader(data)
 			iHdr, err := ParseInvariantHeader(b, 0)
 			Expect(err).ToNot(HaveOccurred())
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+			hdr, err := iHdr.Parse(b, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(hdr.PacketNumber).To(Equal(protocol.PacketNumber(0x123)))
 			Expect(hdr.PacketNumberLen).To(Equal(protocol.PacketNumberLen2))
@@ -169,7 +169,7 @@ var _ = Describe("Header Parsing", func() {
 			b := bytes.NewReader(data)
 			iHdr, err := ParseInvariantHeader(b, 0)
 			Expect(err).ToNot(HaveOccurred())
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+			hdr, err := iHdr.Parse(b, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(hdr.Type).To(Equal(protocol.PacketTypeRetry))
 			Expect(hdr.OrigDestConnectionID).To(Equal(protocol.ConnectionID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}))
@@ -191,7 +191,7 @@ var _ = Describe("Header Parsing", func() {
 			b := bytes.NewReader(buf.Bytes())
 			iHdr, err := ParseInvariantHeader(b, 0)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = iHdr.Parse(b, protocol.PerspectiveClient, versionIETFFrames)
+			_, err = iHdr.Parse(b, versionIETFFrames)
 			Expect(err).To(MatchError("InvalidPacketHeader: Received packet with invalid packet type: 42"))
 		})
 
@@ -208,7 +208,7 @@ var _ = Describe("Header Parsing", func() {
 			b := bytes.NewReader(data)
 			iHdr, err := ParseInvariantHeader(b, 0)
 			Expect(err).ToNot(HaveOccurred())
-			_, err = iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+			_, err = iHdr.Parse(b, versionIETFFrames)
 			Expect(err).To(MatchError(io.EOF))
 		})
 
@@ -239,7 +239,7 @@ var _ = Describe("Header Parsing", func() {
 				b := bytes.NewReader(data[:i])
 				iHdr, err := ParseInvariantHeader(b, 0)
 				Expect(err).ToNot(HaveOccurred())
-				_, err = iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+				_, err = iHdr.Parse(b, versionIETFFrames)
 				Expect(err).To(Equal(io.EOF))
 			}
 		})
@@ -259,7 +259,7 @@ var _ = Describe("Header Parsing", func() {
 				b := bytes.NewReader(data[:i])
 				iHdr, err := ParseInvariantHeader(b, 0)
 				Expect(err).ToNot(HaveOccurred())
-				_, err = iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+				_, err = iHdr.Parse(b, versionIETFFrames)
 				Expect(err).To(Equal(io.EOF))
 			}
 		})
@@ -275,7 +275,7 @@ var _ = Describe("Header Parsing", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iHdr.IsLongHeader).To(BeFalse())
 			Expect(iHdr.DestConnectionID).To(Equal(connID))
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveClient, versionIETFFrames)
+			hdr, err := iHdr.Parse(b, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(hdr.KeyPhase).To(Equal(0))
 			Expect(hdr.DestConnectionID).To(Equal(connID))
@@ -294,7 +294,7 @@ var _ = Describe("Header Parsing", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iHdr.IsLongHeader).To(BeFalse())
 			Expect(iHdr.DestConnectionID).To(Equal(connID))
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveClient, versionIETFFrames)
+			hdr, err := iHdr.Parse(b, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(hdr.KeyPhase).To(Equal(0))
 			Expect(hdr.DestConnectionID).To(Equal(connID))
@@ -311,7 +311,7 @@ var _ = Describe("Header Parsing", func() {
 			b := bytes.NewReader(data)
 			iHdr, err := ParseInvariantHeader(b, 6)
 			Expect(err).ToNot(HaveOccurred())
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+			hdr, err := iHdr.Parse(b, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(hdr.IsLongHeader).To(BeFalse())
 			Expect(hdr.KeyPhase).To(Equal(1))
@@ -327,7 +327,7 @@ var _ = Describe("Header Parsing", func() {
 			b := bytes.NewReader(data)
 			iHdr, err := ParseInvariantHeader(b, 4)
 			Expect(err).ToNot(HaveOccurred())
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveClient, versionIETFFrames)
+			hdr, err := iHdr.Parse(b, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(hdr.IsLongHeader).To(BeFalse())
 			Expect(hdr.PacketNumber).To(Equal(protocol.PacketNumber(0x1337)))
@@ -344,7 +344,7 @@ var _ = Describe("Header Parsing", func() {
 			b := bytes.NewReader(data)
 			iHdr, err := ParseInvariantHeader(b, 10)
 			Expect(err).ToNot(HaveOccurred())
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+			hdr, err := iHdr.Parse(b, versionIETFFrames)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(hdr.IsLongHeader).To(BeFalse())
 			Expect(hdr.PacketNumber).To(Equal(protocol.PacketNumber(0x99beef)))
@@ -374,7 +374,7 @@ var _ = Describe("Header Parsing", func() {
 				b := bytes.NewReader(data[:i])
 				iHdr, err := ParseInvariantHeader(b, 6)
 				Expect(err).ToNot(HaveOccurred())
-				_, err = iHdr.Parse(b, protocol.PerspectiveClient, versionIETFFrames)
+				_, err = iHdr.Parse(b, versionIETFFrames)
 				Expect(err).To(Equal(io.EOF))
 			}
 		})
diff --git a/internal/wire/header_test.go b/internal/wire/header_test.go
index b6c8c34e..a10aab1d 100644
--- a/internal/wire/header_test.go
+++ b/internal/wire/header_test.go
@@ -344,7 +344,7 @@ var _ = Describe("Header", func() {
 			b := bytes.NewReader(data)
 			iHdr, err := ParseInvariantHeader(b, 4)
 			Expect(err).ToNot(HaveOccurred())
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveServer, versionIETFHeader)
+			hdr, err := iHdr.Parse(b, versionIETFHeader)
 			Expect(err).ToNot(HaveOccurred())
 			hdr.Log(logger)
 			Expect(buf.String()).To(ContainSubstring("VersionNegotiationPacket{DestConnectionID: 0xdeadbeefcafe1337, SrcConnectionID: 0xdecafbad13371337"))
diff --git a/internal/wire/version_negotiation_test.go b/internal/wire/version_negotiation_test.go
index 2439f582..6c4f82f9 100644
--- a/internal/wire/version_negotiation_test.go
+++ b/internal/wire/version_negotiation_test.go
@@ -19,7 +19,7 @@ var _ = Describe("Version Negotiation Packets", func() {
 		b := bytes.NewReader(data)
 		iHdr, err := ParseInvariantHeader(b, 4)
 		Expect(err).ToNot(HaveOccurred())
-		hdr, err := iHdr.Parse(b, protocol.PerspectiveServer, versionIETFFrames)
+		hdr, err := iHdr.Parse(b, versionIETFFrames)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(hdr.IsVersionNegotiation).To(BeTrue())
 		Expect(hdr.DestConnectionID).To(Equal(destConnID))
diff --git a/packet_handler_map.go b/packet_handler_map.go
index a4acb6fd..deec1e15 100644
--- a/packet_handler_map.go
+++ b/packet_handler_map.go
@@ -175,12 +175,10 @@ func (h *packetHandlerMap) handlePacket(addr net.Addr, data []byte) error {
 	handlerEntry, handlerFound := h.handlers[string(iHdr.DestConnectionID)]
 	server := h.server
 
-	var sentBy protocol.Perspective
 	var version protocol.VersionNumber
 	var handlePacket func(*receivedPacket)
 	if handlerFound { // existing session
 		handler := handlerEntry.handler
-		sentBy = handler.GetPerspective().Opposite()
 		version = handler.GetVersion()
 		handlePacket = handler.handlePacket
 	} else { // no session found
@@ -203,12 +201,11 @@ func (h *packetHandlerMap) handlePacket(addr net.Addr, data []byte) error {
 			return fmt.Errorf("received a packet with an unexpected connection ID %s", iHdr.DestConnectionID)
 		}
 		handlePacket = server.handlePacket
-		sentBy = protocol.PerspectiveClient
 		version = iHdr.Version
 	}
 	h.mutex.RUnlock()
 
-	hdr, err := iHdr.Parse(r, sentBy, version)
+	hdr, err := iHdr.Parse(r, version)
 	if err != nil {
 		return fmt.Errorf("error parsing header: %s", err)
 	}
diff --git a/packet_handler_map_test.go b/packet_handler_map_test.go
index 1ab43de8..075d71be 100644
--- a/packet_handler_map_test.go
+++ b/packet_handler_map_test.go
@@ -62,13 +62,11 @@ var _ = Describe("Packet Handler Map", func() {
 				close(handledPacket1)
 			})
 			packetHandler1.EXPECT().GetVersion()
-			packetHandler1.EXPECT().GetPerspective().Return(protocol.PerspectiveClient)
 			packetHandler2.EXPECT().handlePacket(gomock.Any()).Do(func(p *receivedPacket) {
 				Expect(p.header.DestConnectionID).To(Equal(connID2))
 				close(handledPacket2)
 			})
 			packetHandler2.EXPECT().GetVersion()
-			packetHandler2.EXPECT().GetPerspective().Return(protocol.PerspectiveClient)
 			handler.Add(connID1, packetHandler1)
 			handler.Add(connID2, packetHandler2)
 
@@ -111,7 +109,6 @@ var _ = Describe("Packet Handler Map", func() {
 			connID := protocol.ConnectionID{1, 2, 3, 4, 5, 6, 7, 8}
 			packetHandler := NewMockPacketHandler(mockCtrl)
 			packetHandler.EXPECT().GetVersion().Return(protocol.VersionWhatever)
-			packetHandler.EXPECT().GetPerspective().Return(protocol.PerspectiveClient)
 			packetHandler.EXPECT().handlePacket(gomock.Any())
 			handler.Add(connID, packetHandler)
 			handler.Retire(connID)
@@ -129,7 +126,6 @@ var _ = Describe("Packet Handler Map", func() {
 			connID := protocol.ConnectionID{1, 2, 3, 4, 5, 6, 7, 8}
 			packetHandler := NewMockPacketHandler(mockCtrl)
 			packetHandler.EXPECT().GetVersion().Return(protocol.VersionWhatever)
-			packetHandler.EXPECT().GetPerspective().Return(protocol.PerspectiveClient)
 			handler.Add(connID, packetHandler)
 			hdr := &wire.Header{
 				IsLongHeader:     true,
@@ -151,7 +147,6 @@ var _ = Describe("Packet Handler Map", func() {
 			connID := protocol.ConnectionID{1, 2, 3, 4, 5, 6, 7, 8}
 			packetHandler := NewMockPacketHandler(mockCtrl)
 			packetHandler.EXPECT().GetVersion().Return(protocol.VersionWhatever)
-			packetHandler.EXPECT().GetPerspective().Return(protocol.PerspectiveClient)
 			handler.Add(connID, packetHandler)
 			hdr := &wire.Header{
 				IsLongHeader:     true,
@@ -170,7 +165,6 @@ var _ = Describe("Packet Handler Map", func() {
 			connID := protocol.ConnectionID{1, 2, 3, 4, 5, 6, 7, 8}
 			packetHandler := NewMockPacketHandler(mockCtrl)
 			packetHandler.EXPECT().GetVersion().Return(protocol.VersionWhatever)
-			packetHandler.EXPECT().GetPerspective().Return(protocol.PerspectiveClient)
 			handler.Add(connID, packetHandler)
 			packetHandler.EXPECT().handlePacket(gomock.Any()).Do(func(p *receivedPacket) {
 				Expect(p.data).To(HaveLen(456 - int(p.header.PacketNumberLen)))
@@ -211,7 +205,6 @@ var _ = Describe("Packet Handler Map", func() {
 			handler.AddWithResetToken(connID, packetHandler, token)
 			// first send a normal packet
 			handledPacket := make(chan struct{})
-			packetHandler.EXPECT().GetPerspective()
 			packetHandler.EXPECT().GetVersion()
 			packetHandler.EXPECT().handlePacket(gomock.Any()).Do(func(p *receivedPacket) {
 				Expect(p.header.DestConnectionID).To(Equal(connID))
diff --git a/packet_packer_test.go b/packet_packer_test.go
index 2135a8b2..2287e376 100644
--- a/packet_packer_test.go
+++ b/packet_packer_test.go
@@ -34,7 +34,7 @@ var _ = Describe("Packet packer", func() {
 		r := bytes.NewReader(data)
 		iHdr, err := wire.ParseInvariantHeader(r, 0)
 		Expect(err).ToNot(HaveOccurred())
-		hdr, err := iHdr.Parse(r, protocol.PerspectiveServer, protocol.VersionWhatever)
+		hdr, err := iHdr.Parse(r, protocol.VersionWhatever)
 		Expect(err).ToNot(HaveOccurred())
 		ExpectWithOffset(0, hdr.Length).To(BeEquivalentTo(r.Len() + int(hdr.PacketNumberLen)))
 	}
diff --git a/server_test.go b/server_test.go
index 6418289b..865760b0 100644
--- a/server_test.go
+++ b/server_test.go
@@ -102,7 +102,7 @@ var _ = Describe("Server", func() {
 			b := bytes.NewReader(data)
 			iHdr, err := wire.ParseInvariantHeader(b, 0)
 			Expect(err).ToNot(HaveOccurred())
-			hdr, err := iHdr.Parse(b, protocol.PerspectiveServer, protocol.VersionTLS)
+			hdr, err := iHdr.Parse(b, protocol.VersionTLS)
 			Expect(err).ToNot(HaveOccurred())
 			return hdr
 		}