diff --git a/server.go b/server.go
index 503d9e37..cb035e86 100644
--- a/server.go
+++ b/server.go
@@ -322,7 +322,7 @@ func (s *baseServer) handlePacketImpl(p *receivedPacket) bool /* should the buff
 	}
 	// Short header packets should never end up here in the first place
 	if !hdr.IsLongHeader {
-		return false
+		panic(fmt.Sprintf("misrouted packet: %#v", hdr))
 	}
 	if hdr.Type == protocol.PacketTypeInitial && len(p.data) < protocol.MinInitialPacketSize {
 		s.logger.Debugf("Dropping a packet that is too small to be a valid Initial (%d bytes)", len(p.data))
diff --git a/server_test.go b/server_test.go
index 675f3256..ff7756a7 100644
--- a/server_test.go
+++ b/server_test.go
@@ -234,8 +234,9 @@ var _ = Describe("Server", func() {
 			It("drops non-Initial packets", func() {
 				serv.handlePacket(getPacket(
 					&wire.Header{
-						Type:    protocol.PacketTypeHandshake,
-						Version: serv.config.Versions[0],
+						IsLongHeader: true,
+						Type:         protocol.PacketTypeHandshake,
+						Version:      serv.config.Versions[0],
 					},
 					[]byte("invalid"),
 				))