diff --git a/example/main.go b/example/main.go
index b84b8d34..b1ba613f 100644
--- a/example/main.go
+++ b/example/main.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/lucas-clemente/quic-go"
 	"github.com/lucas-clemente/quic-go/crypto"
+	"github.com/lucas-clemente/quic-go/handshake"
 	"github.com/lucas-clemente/quic-go/protocol"
 	"github.com/lucas-clemente/quic-go/utils"
 )
@@ -27,7 +28,7 @@ func main() {
 		panic(err)
 	}
 
-	serverConfig := quic.NewServerConfig(crypto.NewCurve25519KEX(), keyData)
+	serverConfig := handshake.NewServerConfig(crypto.NewCurve25519KEX(), keyData)
 
 	// TODO: When should a session be created?
 	sessions := map[protocol.ConnectionID]*quic.Session{}
diff --git a/handshake/handshake.go b/handshake/handshake.go
new file mode 100644
index 00000000..853053b9
--- /dev/null
+++ b/handshake/handshake.go
@@ -0,0 +1,80 @@
+package handshake
+
+import (
+	"bytes"
+	"errors"
+	"io"
+
+	"github.com/lucas-clemente/quic-go/crypto"
+	"github.com/lucas-clemente/quic-go/protocol"
+)
+
+// The Handshake handles all things crypto for the Session
+type Handshake struct {
+	connID  protocol.ConnectionID
+	version protocol.VersionNumber
+	aead    crypto.AEAD
+	scfg    *ServerConfig
+}
+
+// NewHandshake creates a new Handshake instance
+func NewHandshake(connID protocol.ConnectionID, version protocol.VersionNumber, scfg *ServerConfig) *Handshake {
+	return &Handshake{
+		connID:  connID,
+		version: version,
+		aead:    &crypto.NullAEAD{},
+		scfg:    scfg,
+	}
+}
+
+// Open a message
+func (h *Handshake) Open(packetNumber protocol.PacketNumber, associatedData []byte, ciphertext io.Reader) (*bytes.Reader, error) {
+	return h.aead.Open(packetNumber, associatedData, ciphertext)
+}
+
+// Seal a messageTag
+func (h *Handshake) Seal(packetNumber protocol.PacketNumber, b *bytes.Buffer, associatedData []byte, plaintext []byte) {
+	h.aead.Seal(packetNumber, b, associatedData, plaintext)
+}
+
+// HandleCryptoMessage handles the crypto handshake and returns the answer
+func (h *Handshake) HandleCryptoMessage(data []byte) ([]byte, error) {
+	messageTag, cryptoData, err := ParseHandshakeMessage(data)
+	if err != nil {
+		return nil, err
+	}
+	if messageTag != TagCHLO {
+		return nil, errors.New("Session: expected CHLO")
+	}
+
+	if _, ok := cryptoData[TagSCID]; ok {
+		var sharedSecret []byte
+		sharedSecret, err = h.scfg.kex.CalculateSharedKey(cryptoData[TagPUBS])
+		if err != nil {
+			return nil, err
+		}
+		h.aead, err = crypto.DeriveKeysChacha20(sharedSecret, cryptoData[TagNONC], h.connID, data, h.scfg.Get(), h.scfg.kd.GetCertUncompressed())
+		if err != nil {
+			return nil, err
+		}
+		// TODO: Send SHLO
+		return nil, nil
+	}
+
+	var chloOrNil []byte
+	if h.version > protocol.VersionNumber(30) {
+		chloOrNil = data
+	}
+	proof, err := h.scfg.Sign(chloOrNil)
+	if err != nil {
+		return nil, err
+	}
+	var serverReply bytes.Buffer
+	WriteHandshakeMessage(&serverReply, TagREJ, map[Tag][]byte{
+		TagSCFG: h.scfg.Get(),
+		TagCERT: h.scfg.GetCertCompressed(),
+		TagPROF: proof,
+	})
+
+	return serverReply.Bytes(), nil
+}
diff --git a/handshake/handshake_test.go b/handshake/handshake_test.go
new file mode 100644
index 00000000..7e164db3
--- /dev/null
+++ b/handshake/handshake_test.go
@@ -0,0 +1,9 @@
+package handshake
+
+import (
+	. "github.com/onsi/ginkgo"
+	// . "github.com/onsi/gomega"
+)
+
+var _ = Describe("Handshake", func() {
+})
diff --git a/server_config.go b/handshake/server_config.go
similarity index 57%
rename from server_config.go
rename to handshake/server_config.go
index 0d69d812..3fe86831 100644
--- a/server_config.go
+++ b/handshake/server_config.go
@@ -1,10 +1,9 @@
-package quic
+package handshake
 
 import (
 	"bytes"
 
 	"github.com/lucas-clemente/quic-go/crypto"
-	"github.com/lucas-clemente/quic-go/handshake"
 )
 
 // ServerConfig is a server config
@@ -24,14 +23,14 @@ func NewServerConfig(kex crypto.KeyExchange, kd *crypto.KeyData) *ServerConfig {
 // Get the server config binary representation
 func (s *ServerConfig) Get() []byte {
 	var serverConfig bytes.Buffer
-	handshake.WriteHandshakeMessage(&serverConfig, handshake.TagSCFG, map[handshake.Tag][]byte{
-		handshake.TagSCID: []byte{0xC5, 0x1C, 0x73, 0x6B, 0x8F, 0x48, 0x49, 0xAE, 0xB3, 0x00, 0xA2, 0xD4, 0x4B, 0xA0, 0xCF, 0xDF},
-		handshake.TagKEXS: []byte("C255"),
-		handshake.TagAEAD: []byte("CC20"),
-		handshake.TagPUBS: append([]byte{0x20, 0x00, 0x00}, s.kex.PublicKey()...),
-		handshake.TagOBIT: []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7},
-		handshake.TagEXPY: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
-		handshake.TagVER:  []byte("Q032"),
+	WriteHandshakeMessage(&serverConfig, TagSCFG, map[Tag][]byte{
+		TagSCID: []byte{0xC5, 0x1C, 0x73, 0x6B, 0x8F, 0x48, 0x49, 0xAE, 0xB3, 0x00, 0xA2, 0xD4, 0x4B, 0xA0, 0xCF, 0xDF},
+		TagKEXS: []byte("C255"),
+		TagAEAD: []byte("CC20"),
+		TagPUBS: append([]byte{0x20, 0x00, 0x00}, s.kex.PublicKey()...),
+		TagOBIT: []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7},
+		TagEXPY: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
+		TagVER:  []byte("Q032"),
 	})
 	return serverConfig.Bytes()
 }
diff --git a/server_config_test.go b/handshake/server_config_test.go
similarity index 98%
rename from server_config_test.go
rename to handshake/server_config_test.go
index 3710529b..3d3122d3 100644
--- a/server_config_test.go
+++ b/handshake/server_config_test.go
@@ -1,4 +1,4 @@
-package quic
+package handshake
 
 import (
 	"bytes"
diff --git a/session.go b/session.go
index 1cc5ec83..dc0b1f72 100644
--- a/session.go
+++ b/session.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net"
 
-	"github.com/lucas-clemente/quic-go/crypto"
 	"github.com/lucas-clemente/quic-go/handshake"
 	"github.com/lucas-clemente/quic-go/protocol"
 )
@@ -18,12 +17,12 @@ type StreamCallback func(*StreamFrame) []Frame
 type Session struct {
 	VersionNumber protocol.VersionNumber
 	ConnectionID  protocol.ConnectionID
-	ServerConfig  *ServerConfig
 
 	Connection        *net.UDPConn
 	CurrentRemoteAddr *net.UDPAddr
 
-	aead crypto.AEAD
+	ServerConfig *handshake.ServerConfig
+	hshk         *handshake.Handshake
 
 	Entropy EntropyAccumulator
 
@@ -33,13 +32,13 @@ type Session struct {
 }
 
 // NewSession makes a new session
-func NewSession(conn *net.UDPConn, v protocol.VersionNumber, connectionID protocol.ConnectionID, sCfg *ServerConfig, streamCallback StreamCallback) *Session {
+func NewSession(conn *net.UDPConn, v protocol.VersionNumber, connectionID protocol.ConnectionID, sCfg *handshake.ServerConfig, streamCallback StreamCallback) *Session {
 	return &Session{
 		Connection:     conn,
 		VersionNumber:  v,
 		ConnectionID:   connectionID,
 		ServerConfig:   sCfg,
-		aead:           &crypto.NullAEAD{},
+		hshk:           handshake.NewHandshake(connectionID, v, sCfg),
 		streamCallback: streamCallback,
 	}
 }
@@ -51,7 +50,7 @@ func (s *Session) HandlePacket(addr *net.UDPAddr, publicHeaderBinary []byte, pub
 		s.CurrentRemoteAddr = addr
 	}
 
-	r, err := s.aead.Open(publicHeader.PacketNumber, publicHeaderBinary, r)
+	r, err := s.hshk.Open(publicHeader.PacketNumber, publicHeaderBinary, r)
 	if err != nil {
 		return err
 	}
@@ -62,6 +61,11 @@ func (s *Session) HandlePacket(addr *net.UDPAddr, publicHeaderBinary []byte, pub
 	}
 	s.Entropy.Add(publicHeader.PacketNumber, privateFlag&0x01 > 0)
 
+	s.SendFrames([]Frame{&AckFrame{
+		LargestObserved: uint64(publicHeader.PacketNumber),
+		Entropy:         s.Entropy.Get(),
+	}})
+
 	frameCounter := 0
 
 	// read all frames in the packet
@@ -89,10 +93,19 @@ func (s *Session) HandlePacket(addr *net.UDPAddr, publicHeaderBinary []byte, pub
 			}
 
 			if frame.StreamID == 1 {
-				s.HandleCryptoHandshake(frame)
+				reply, err := s.hshk.HandleCryptoMessage(frame.Data)
+				if err != nil {
+					return err
+				}
+				if reply != nil {
+					s.SendFrames([]Frame{&StreamFrame{StreamID: 1, Data: reply}})
+				}
+				// TODO: Send reply
 			} else {
 				replyFrames := s.streamCallback(frame)
-				s.SendFrames(append([]Frame{&AckFrame{Entropy: s.Entropy.Get(), LargestObserved: 3}}, replyFrames...))
+				if replyFrames != nil {
+					s.SendFrames(replyFrames)
+				}
 			}
 			continue
 		} else if typeByte&0xC0 == 0x40 { // ACK
@@ -150,65 +163,9 @@ func (s *Session) SendFrames(frames []Frame) error {
 		return err
 	}
 
-	s.aead.Seal(s.lastSentPacketNumber, &fullReply, fullReply.Bytes(), framesData.Bytes())
+	s.hshk.Seal(s.lastSentPacketNumber, &fullReply, fullReply.Bytes(), framesData.Bytes())
 
 	fmt.Printf("Sending %d bytes to %v\n", len(fullReply.Bytes()), s.CurrentRemoteAddr)
 	_, err := s.Connection.WriteToUDP(fullReply.Bytes(), s.CurrentRemoteAddr)
 	return err
 }
-
-// HandleCryptoHandshake handles the crypto handshake
-func (s *Session) HandleCryptoHandshake(frame *StreamFrame) error {
-	messageTag, cryptoData, err := handshake.ParseHandshakeMessage(frame.Data)
-	if err != nil {
-		panic(err)
-	}
-
-	// TODO: Switch client messages here
-	if messageTag != handshake.TagCHLO {
-		return errors.New("Session: expected CHLO")
-	}
-
-	if _, ok := cryptoData[handshake.TagSCID]; ok {
-		var sharedSecret []byte
-		sharedSecret, err = s.ServerConfig.kex.CalculateSharedKey(cryptoData[handshake.TagPUBS])
-		if err != nil {
-			return err
-		}
-		s.aead, err = crypto.DeriveKeysChacha20(sharedSecret, cryptoData[handshake.TagNONC], s.ConnectionID, frame.Data, s.ServerConfig.Get(), s.ServerConfig.kd.GetCertUncompressed())
-		if err != nil {
-			return err
-		}
-		s.SendFrames([]Frame{&AckFrame{
-			Entropy:         s.Entropy.Get(),
-			LargestObserved: 2,
-		}})
-		return nil
-	}
-
-	var chloOrNil []byte
-	if s.VersionNumber > protocol.VersionNumber(30) {
-		chloOrNil = frame.Data
-	}
-	proof, err := s.ServerConfig.Sign(chloOrNil)
-	if err != nil {
-		return err
-	}
-	var serverReply bytes.Buffer
-	handshake.WriteHandshakeMessage(&serverReply, handshake.TagREJ, map[handshake.Tag][]byte{
-		handshake.TagSCFG: s.ServerConfig.Get(),
-		handshake.TagCERT: s.ServerConfig.GetCertCompressed(),
-		handshake.TagPROF: proof,
-	})
-
-	return s.SendFrames([]Frame{
-		&AckFrame{
-			Entropy:         s.Entropy.Get(),
-			LargestObserved: 1,
-		},
-		&StreamFrame{
-			StreamID: 1,
-			Data:     serverReply.Bytes(),
-		},
-	})
-}