diff --git a/go.mod b/go.mod
index 59da8f9a..a7075e53 100644
--- a/go.mod
+++ b/go.mod
@@ -8,12 +8,11 @@ require (
 	github.com/francoispqt/gojay v1.2.13
 	github.com/golang/mock v1.4.0
 	github.com/golang/protobuf v1.3.1
-	github.com/marten-seemann/chacha20 v0.2.0
 	github.com/marten-seemann/qpack v0.1.0
 	github.com/marten-seemann/qtls v0.7.1
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.8.1
-	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
+	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
 	golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 )
diff --git a/go.sum b/go.sum
index d395e5de..d3858f46 100644
--- a/go.sum
+++ b/go.sum
@@ -63,8 +63,6 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/marten-seemann/chacha20 v0.2.0 h1:f40vqzzx+3GdOmzQoItkLX5WLvHgPgyYqFFIO5Gh4hQ=
-github.com/marten-seemann/chacha20 v0.2.0/go.mod h1:HSdjFau7GzYRj+ahFNwsO3ouVJr1HFkWoEwNDb4TMtE=
 github.com/marten-seemann/qpack v0.1.0 h1:/0M7lkda/6mus9B8u34Asqm8ZhHAAt9Ho0vniNuVSVg=
 github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
 github.com/marten-seemann/qtls v0.7.1 h1:+jBCODo2UW86tVlc1HCxAG+ccD51CZquJunNPUsaPQU=
@@ -127,8 +125,9 @@ golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
 golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
+golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
diff --git a/internal/handshake/header_protector.go b/internal/handshake/header_protector.go
index 79530739..9962180e 100644
--- a/internal/handshake/header_protector.go
+++ b/internal/handshake/header_protector.go
@@ -4,9 +4,11 @@ import (
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
+	"encoding/binary"
 	"fmt"
 
-	"github.com/marten-seemann/chacha20"
+	"golang.org/x/crypto/chacha20"
+
 	"github.com/marten-seemann/qtls"
 )
 
@@ -75,7 +77,6 @@ type chachaHeaderProtector struct {
 	mask [5]byte
 
 	key          [32]byte
-	sampleBuf    [16]byte
 	isLongHeader bool
 }
 
@@ -117,14 +118,18 @@ func (p *chachaHeaderProtector) EncryptHeader(sample []byte, firstByte *byte, hd
 }
 
 func (p *chachaHeaderProtector) apply(sample []byte, firstByte *byte, hdrBytes []byte) {
-	if len(sample) < len(p.mask) {
+	if len(sample) != 16 {
 		panic("invalid sample size")
 	}
 	for i := 0; i < 5; i++ {
 		p.mask[i] = 0
 	}
-	copy(p.sampleBuf[:], sample)
-	chacha20.XORKeyStream(p.mask[:], p.mask[:], &p.sampleBuf, &p.key)
+	cipher, err := chacha20.NewUnauthenticatedCipher(p.key[:], sample[4:])
+	if err != nil {
+		panic(err)
+	}
+	cipher.SetCounter(binary.BigEndian.Uint32(sample[4:]))
+	cipher.XORKeyStream(p.mask[:], p.mask[:])
 	p.applyMask(firstByte, hdrBytes)
 }