crypto/tls: FIPS 140-3 mode

Consolidates handling of FIPS 140-3 considerations for the tls package.
Considerations specific to certificates are now handled in tls instead
of x509 to limit the area-of-effect of FIPS as much as possible.
Boringcrypto specific prefixes are renamed as appropriate.

For #69536

Co-authored-by: Filippo Valsorda <filippo@golang.org>
Change-Id: I1b1fef83c3599e4c9b98ad81db582ac93253030b
Reviewed-on: https://go-review.googlesource.com/c/go/+/629675
Reviewed-by: Filippo Valsorda <filippo@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Russ Cox <rsc@golang.org>
Auto-Submit: Filippo Valsorda <filippo@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

common.go

@@ -15,6 +15,7 @@ import (
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/sha512"
+	"crypto/tls/internal/fips140tls"
 	"crypto/x509"
 	"errors"
 	"fmt"
@@ -1061,12 +1062,12 @@ func (c *Config) time() time.Time {
 
 func (c *Config) cipherSuites() []uint16 {
 	if c.CipherSuites == nil {
-		if needFIPS() {
+		if fips140tls.Required() {
 			return defaultCipherSuitesFIPS
 		}
 		return defaultCipherSuites()
 	}
-	if needFIPS() {
+	if fips140tls.Required() {
 		cipherSuites := slices.Clone(c.CipherSuites)
 		return slices.DeleteFunc(cipherSuites, func(id uint16) bool {
 			return !slices.Contains(defaultCipherSuitesFIPS, id)
@@ -1092,7 +1093,7 @@ var tls10server = godebug.New("tls10server")
 func (c *Config) supportedVersions(isClient bool) []uint16 {
 	versions := make([]uint16, 0, len(supportedVersions))
 	for _, v := range supportedVersions {
-		if needFIPS() && !slices.Contains(defaultSupportedVersionsFIPS, v) {
+		if fips140tls.Required() && !slices.Contains(defaultSupportedVersionsFIPS, v) {
 			continue
 		}
 		if (c == nil || c.MinVersion == 0) && v < VersionTLS12 {
@@ -1140,12 +1141,12 @@ func (c *Config) curvePreferences(version uint16) []CurveID {
 	var curvePreferences []CurveID
 	if c != nil && len(c.CurvePreferences) != 0 {
 		curvePreferences = slices.Clone(c.CurvePreferences)
-		if needFIPS() {
+		if fips140tls.Required() {
 			return slices.DeleteFunc(curvePreferences, func(c CurveID) bool {
 				return !slices.Contains(defaultCurvePreferencesFIPS, c)
 			})
 		}
-	} else if needFIPS() {
+	} else if fips140tls.Required() {
 		curvePreferences = slices.Clone(defaultCurvePreferencesFIPS)
 	} else {
 		curvePreferences = defaultCurvePreferences()
@@ -1617,7 +1618,7 @@ func unexpectedMessageError(wanted, got any) error {
 
 // supportedSignatureAlgorithms returns the supported signature algorithms.
 func supportedSignatureAlgorithms() []SignatureScheme {
-	if !needFIPS() {
+	if !fips140tls.Required() {
 		return defaultSupportedSignatureAlgorithms
 	}
 	return defaultSupportedSignatureAlgorithmsFIPS
@@ -1646,3 +1647,56 @@ func (e *CertificateVerificationError) Error() string {
 func (e *CertificateVerificationError) Unwrap() error {
 	return e.Err
 }
+
+// fipsAllowedChains returns chains that are allowed to be used in a TLS connection
+// based on the current fips140tls enforcement setting.
+//
+// If fips140tls is not required, the chains are returned as-is with no processing.
+// Otherwise, the returned chains are filtered to only those allowed by FIPS 140-3.
+// If this results in no chains it returns an error.
+func fipsAllowedChains(chains [][]*x509.Certificate) ([][]*x509.Certificate, error) {
+	if !fips140tls.Required() {
+		return chains, nil
+	}
+
+	permittedChains := make([][]*x509.Certificate, 0, len(chains))
+	for _, chain := range chains {
+		if fipsAllowChain(chain) {
+			permittedChains = append(permittedChains, chain)
+		}
+	}
+
+	if len(permittedChains) == 0 {
+		return nil, errors.New("tls: no FIPS compatible certificate chains found")
+	}
+
+	return permittedChains, nil
+}
+
+func fipsAllowChain(chain []*x509.Certificate) bool {
+	if len(chain) == 0 {
+		return false
+	}
+
+	for _, cert := range chain {
+		if !fipsAllowCert(cert) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func fipsAllowCert(c *x509.Certificate) bool {
+	// The key must be RSA 2048, RSA 3072, RSA 4096,
+	// or ECDSA P-256, P-384, P-521.
+	switch k := c.PublicKey.(type) {
+	case *rsa.PublicKey:
+		size := k.N.BitLen()
+		return size == 2048 || size == 3072 || size == 4096
+	case *ecdsa.PublicKey:
+		return k.Curve == elliptic.P256() || k.Curve == elliptic.P384() || k.Curve == elliptic.P521()
+	}
+
+	return false
+}