From 690d770374b8623fa37bb7bf0c41c0f4a3463285 Mon Sep 17 00:00:00 2001 From: Sergey Frolov Date: Thu, 21 Jun 2018 10:23:09 -0400 Subject: [PATCH] Standard internal logic to detect hardware crypto We can't simply link to system internal package, as it is prohibited by Golang. Let's copy-paste instead. --- cpu/cpu.go | 77 +++++++++++++++++++++++++++++++++++++++++++++ cpu/cpu_arm.go | 7 +++++ cpu/cpu_arm64.go | 45 ++++++++++++++++++++++++++ cpu/cpu_mips.go | 7 +++++ cpu/cpu_mips64.go | 7 +++++ cpu/cpu_mips64le.go | 7 +++++ cpu/cpu_mipsle.go | 7 +++++ cpu/cpu_ppc64x.go | 54 +++++++++++++++++++++++++++++++ cpu/cpu_s390x.go | 7 +++++ cpu/cpu_test.go | 50 +++++++++++++++++++++++++++++ cpu/cpu_x86.go | 61 +++++++++++++++++++++++++++++++++++ cpu/cpu_x86.s | 32 +++++++++++++++++++ 12 files changed, 361 insertions(+) create mode 100644 cpu/cpu.go create mode 100644 cpu/cpu_arm.go create mode 100644 cpu/cpu_arm64.go create mode 100644 cpu/cpu_mips.go create mode 100644 cpu/cpu_mips64.go create mode 100644 cpu/cpu_mips64le.go create mode 100644 cpu/cpu_mipsle.go create mode 100644 cpu/cpu_ppc64x.go create mode 100644 cpu/cpu_s390x.go create mode 100644 cpu/cpu_test.go create mode 100644 cpu/cpu_x86.go create mode 100644 cpu/cpu_x86.s diff --git a/cpu/cpu.go b/cpu/cpu.go new file mode 100644 index 0000000..22fc561 --- /dev/null +++ b/cpu/cpu.go @@ -0,0 +1,77 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cpu implements processor feature detection +// used by the Go standard library. +package cpu + +var X86 x86 + +// The booleans in x86 contain the correspondingly named cpuid feature bit. +// HasAVX and HasAVX2 are only set if the OS does support XMM and YMM registers +// in addition to the cpuid feature bit being set. +// The struct is padded to avoid false sharing. +type x86 struct { + _ [CacheLineSize]byte + HasAES bool + HasADX bool + HasAVX bool + HasAVX2 bool + HasBMI1 bool + HasBMI2 bool + HasERMS bool + HasFMA bool + HasOSXSAVE bool + HasPCLMULQDQ bool + HasPOPCNT bool + HasSSE2 bool + HasSSE3 bool + HasSSSE3 bool + HasSSE41 bool + HasSSE42 bool + _ [CacheLineSize]byte +} + +var PPC64 ppc64 + +// For ppc64x, it is safe to check only for ISA level starting on ISA v3.00, +// since there are no optional categories. There are some exceptions that also +// require kernel support to work (darn, scv), so there are capability bits for +// those as well. The minimum processor requirement is POWER8 (ISA 2.07), so we +// maintain some of the old capability checks for optional categories for +// safety. +// The struct is padded to avoid false sharing. +type ppc64 struct { + _ [CacheLineSize]byte + HasVMX bool // Vector unit (Altivec) + HasDFP bool // Decimal Floating Point unit + HasVSX bool // Vector-scalar unit + HasHTM bool // Hardware Transactional Memory + HasISEL bool // Integer select + HasVCRYPTO bool // Vector cryptography + HasHTMNOSC bool // HTM: kernel-aborted transaction in syscalls + HasDARN bool // Hardware random number generator (requires kernel enablement) + HasSCV bool // Syscall vectored (requires kernel enablement) + IsPOWER8 bool // ISA v2.07 (POWER8) + IsPOWER9 bool // ISA v3.00 (POWER9) + _ [CacheLineSize]byte +} + +var ARM64 arm64 + +// The booleans in arm64 contain the correspondingly named cpu feature bit. +// The struct is padded to avoid false sharing. +type arm64 struct { + _ [CacheLineSize]byte + HasFP bool + HasASIMD bool + HasEVTSTRM bool + HasAES bool + HasPMULL bool + HasSHA1 bool + HasSHA2 bool + HasCRC32 bool + HasATOMICS bool + _ [CacheLineSize]byte +} diff --git a/cpu/cpu_arm.go b/cpu/cpu_arm.go new file mode 100644 index 0000000..078a6c3 --- /dev/null +++ b/cpu/cpu_arm.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLineSize = 32 diff --git a/cpu/cpu_arm64.go b/cpu/cpu_arm64.go new file mode 100644 index 0000000..e1278a1 --- /dev/null +++ b/cpu/cpu_arm64.go @@ -0,0 +1,45 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 + +package cpu + +const CacheLineSize = 64 + +// arm64 doesn't have a 'cpuid' equivalent, so we rely on HWCAP/HWCAP2. +// These are linknamed in runtime/os_linux_arm64.go and are initialized by +// archauxv(). +var arm64_hwcap uint +var arm64_hwcap2 uint + +// HWCAP/HWCAP2 bits. These are exposed by Linux. +const ( + _ARM64_FEATURE_HAS_FP = (1 << 0) + _ARM64_FEATURE_HAS_ASIMD = (1 << 1) + _ARM64_FEATURE_HAS_EVTSTRM = (1 << 2) + _ARM64_FEATURE_HAS_AES = (1 << 3) + _ARM64_FEATURE_HAS_PMULL = (1 << 4) + _ARM64_FEATURE_HAS_SHA1 = (1 << 5) + _ARM64_FEATURE_HAS_SHA2 = (1 << 6) + _ARM64_FEATURE_HAS_CRC32 = (1 << 7) + _ARM64_FEATURE_HAS_ATOMICS = (1 << 8) +) + +func init() { + // HWCAP feature bits + ARM64.HasFP = isSet(arm64_hwcap, _ARM64_FEATURE_HAS_FP) + ARM64.HasASIMD = isSet(arm64_hwcap, _ARM64_FEATURE_HAS_ASIMD) + ARM64.HasEVTSTRM = isSet(arm64_hwcap, _ARM64_FEATURE_HAS_EVTSTRM) + ARM64.HasAES = isSet(arm64_hwcap, _ARM64_FEATURE_HAS_AES) + ARM64.HasPMULL = isSet(arm64_hwcap, _ARM64_FEATURE_HAS_PMULL) + ARM64.HasSHA1 = isSet(arm64_hwcap, _ARM64_FEATURE_HAS_SHA1) + ARM64.HasSHA2 = isSet(arm64_hwcap, _ARM64_FEATURE_HAS_SHA2) + ARM64.HasCRC32 = isSet(arm64_hwcap, _ARM64_FEATURE_HAS_CRC32) + ARM64.HasATOMICS = isSet(arm64_hwcap, _ARM64_FEATURE_HAS_ATOMICS) +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/cpu/cpu_mips.go b/cpu/cpu_mips.go new file mode 100644 index 0000000..078a6c3 --- /dev/null +++ b/cpu/cpu_mips.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLineSize = 32 diff --git a/cpu/cpu_mips64.go b/cpu/cpu_mips64.go new file mode 100644 index 0000000..078a6c3 --- /dev/null +++ b/cpu/cpu_mips64.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLineSize = 32 diff --git a/cpu/cpu_mips64le.go b/cpu/cpu_mips64le.go new file mode 100644 index 0000000..078a6c3 --- /dev/null +++ b/cpu/cpu_mips64le.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLineSize = 32 diff --git a/cpu/cpu_mipsle.go b/cpu/cpu_mipsle.go new file mode 100644 index 0000000..078a6c3 --- /dev/null +++ b/cpu/cpu_mipsle.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLineSize = 32 diff --git a/cpu/cpu_ppc64x.go b/cpu/cpu_ppc64x.go new file mode 100644 index 0000000..7f09372 --- /dev/null +++ b/cpu/cpu_ppc64x.go @@ -0,0 +1,54 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +package cpu + +const CacheLineSize = 128 + +// ppc64x doesn't have a 'cpuid' equivalent, so we rely on HWCAP/HWCAP2. +// These are linknamed in runtime/os_linux_ppc64x.go and are initialized by +// archauxv(). +var ppc64x_hwcap uint +var ppc64x_hwcap2 uint + +// HWCAP/HWCAP2 bits. These are exposed by the kernel. +const ( + // ISA Level + _PPC_FEATURE2_ARCH_2_07 = 0x80000000 + _PPC_FEATURE2_ARCH_3_00 = 0x00800000 + + // CPU features + _PPC_FEATURE_HAS_ALTIVEC = 0x10000000 + _PPC_FEATURE_HAS_DFP = 0x00000400 + _PPC_FEATURE_HAS_VSX = 0x00000080 + _PPC_FEATURE2_HAS_HTM = 0x40000000 + _PPC_FEATURE2_HAS_ISEL = 0x08000000 + _PPC_FEATURE2_HAS_VEC_CRYPTO = 0x02000000 + _PPC_FEATURE2_HTM_NOSC = 0x01000000 + _PPC_FEATURE2_DARN = 0x00200000 + _PPC_FEATURE2_SCV = 0x00100000 +) + +func init() { + // HWCAP feature bits + PPC64.HasVMX = isSet(ppc64x_hwcap, _PPC_FEATURE_HAS_ALTIVEC) + PPC64.HasDFP = isSet(ppc64x_hwcap, _PPC_FEATURE_HAS_DFP) + PPC64.HasVSX = isSet(ppc64x_hwcap, _PPC_FEATURE_HAS_VSX) + + // HWCAP2 feature bits + PPC64.IsPOWER8 = isSet(ppc64x_hwcap2, _PPC_FEATURE2_ARCH_2_07) + PPC64.HasHTM = isSet(ppc64x_hwcap2, _PPC_FEATURE2_HAS_HTM) + PPC64.HasISEL = isSet(ppc64x_hwcap2, _PPC_FEATURE2_HAS_ISEL) + PPC64.HasVCRYPTO = isSet(ppc64x_hwcap2, _PPC_FEATURE2_HAS_VEC_CRYPTO) + PPC64.HasHTMNOSC = isSet(ppc64x_hwcap2, _PPC_FEATURE2_HTM_NOSC) + PPC64.IsPOWER9 = isSet(ppc64x_hwcap2, _PPC_FEATURE2_ARCH_3_00) + PPC64.HasDARN = isSet(ppc64x_hwcap2, _PPC_FEATURE2_DARN) + PPC64.HasSCV = isSet(ppc64x_hwcap2, _PPC_FEATURE2_SCV) +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/cpu/cpu_s390x.go b/cpu/cpu_s390x.go new file mode 100644 index 0000000..4455809 --- /dev/null +++ b/cpu/cpu_s390x.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLineSize = 256 diff --git a/cpu/cpu_test.go b/cpu/cpu_test.go new file mode 100644 index 0000000..94214b5 --- /dev/null +++ b/cpu/cpu_test.go @@ -0,0 +1,50 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu_test + +import ( + "github.com/refraction-networking/utls/cpu" + "runtime" + "testing" +) + +func TestAMD64minimalFeatures(t *testing.T) { + if runtime.GOARCH == "amd64" { + if !cpu.X86.HasSSE2 { + t.Fatalf("HasSSE2 expected true, got false") + } + } +} + +func TestAVX2hasAVX(t *testing.T) { + if runtime.GOARCH == "amd64" { + if cpu.X86.HasAVX2 && !cpu.X86.HasAVX { + t.Fatalf("HasAVX expected true, got false") + } + } +} + +func TestPPC64minimalFeatures(t *testing.T) { + if runtime.GOARCH == "ppc64" || runtime.GOARCH == "ppc64le" { + if !cpu.PPC64.IsPOWER8 { + t.Fatalf("IsPOWER8 expected true, got false") + } + if !cpu.PPC64.HasVMX { + t.Fatalf("HasVMX expected true, got false") + } + if !cpu.PPC64.HasDFP { + t.Fatalf("HasDFP expected true, got false") + } + if !cpu.PPC64.HasVSX { + t.Fatalf("HasVSX expected true, got false") + } + if !cpu.PPC64.HasISEL { + t.Fatalf("HasISEL expected true, got false") + } + if !cpu.PPC64.HasVCRYPTO { + t.Fatalf("HasVCRYPTO expected true, got false") + } + } +} diff --git a/cpu/cpu_x86.go b/cpu/cpu_x86.go new file mode 100644 index 0000000..34c632f --- /dev/null +++ b/cpu/cpu_x86.go @@ -0,0 +1,61 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 + +package cpu + +const CacheLineSize = 64 + +// cpuid is implemented in cpu_x86.s. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv with ecx = 0 is implemented in cpu_x86.s. +func xgetbv() (eax, edx uint32) + +func init() { + maxID, _, _, _ := cpuid(0, 0) + + if maxID < 1 { + return + } + + _, _, ecx1, edx1 := cpuid(1, 0) + X86.HasSSE2 = isSet(26, edx1) + + X86.HasSSE3 = isSet(0, ecx1) + X86.HasPCLMULQDQ = isSet(1, ecx1) + X86.HasSSSE3 = isSet(9, ecx1) + X86.HasFMA = isSet(12, ecx1) + X86.HasSSE41 = isSet(19, ecx1) + X86.HasSSE42 = isSet(20, ecx1) + X86.HasPOPCNT = isSet(23, ecx1) + X86.HasAES = isSet(25, ecx1) + X86.HasOSXSAVE = isSet(27, ecx1) + + osSupportsAVX := false + // For XGETBV, OSXSAVE bit is required and sufficient. + if X86.HasOSXSAVE { + eax, _ := xgetbv() + // Check if XMM and YMM registers have OS support. + osSupportsAVX = isSet(1, eax) && isSet(2, eax) + } + + X86.HasAVX = isSet(28, ecx1) && osSupportsAVX + + if maxID < 7 { + return + } + + _, ebx7, _, _ := cpuid(7, 0) + X86.HasBMI1 = isSet(3, ebx7) + X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX + X86.HasBMI2 = isSet(8, ebx7) + X86.HasERMS = isSet(9, ebx7) + X86.HasADX = isSet(19, ebx7) +} + +func isSet(bitpos uint, value uint32) bool { + return value&(1<