Implement ClientHelloSpec JSON Unmarshaler (#176)

* wip: staging work

* wip: staging work

* feat: ClientHello JSON Unmarshaler

Allowing unmarshalling a JSON object into a ClientHelloSpec.

* feat: ClientHello JSON Unmarshaler rev

- Revised JSON ClientHello format
- Implemented `TLSExtensionJSON` interface for some more extensions

go.mod

@@ -4,6 +4,7 @@ go 1.19
 
 require (
 	github.com/andybalholm/brotli v1.0.4
+	github.com/gaukas/godicttls v0.0.3
 	github.com/klauspost/compress v1.15.15
 	golang.org/x/crypto v0.5.0
 	golang.org/x/net v0.7.0

go.sum

@@ -1,5 +1,7 @@
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/gaukas/godicttls v0.0.3 h1:YNDIf0d9adcxOijiLrEzpfZGAkNwLRzPaG6OjU7EITk=
+github.com/gaukas/godicttls v0.0.3/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
 github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
 github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
 golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=

testdata/ClientHello-JSON-Chrome102.json

@@ -0,0 +1,75 @@
+{
+	"cipher_suites": [
+        "GREASE",
+		"TLS_AES_128_GCM_SHA256",
+		"TLS_AES_256_GCM_SHA384",
+        "TLS_CHACHA20_POLY1305_SHA256",
+        "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+        "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+        "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+        "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+        "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
+        "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
+        "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
+        "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
+        "TLS_RSA_WITH_AES_128_GCM_SHA256",
+        "TLS_RSA_WITH_AES_256_GCM_SHA384",
+        "TLS_RSA_WITH_AES_128_CBC_SHA",
+        "TLS_RSA_WITH_AES_256_CBC_SHA"
+	],
+	"compression_methods": [
+		"NULL"
+	],
+	"extensions": [
+		{"name": "GREASE"},
+		{"name": "server_name"},
+		{"name": "extended_master_secret"},
+		{"name": "renegotiation_info"},
+		{"name": "supported_groups", "named_group_list": [
+			"GREASE",
+			"x25519",
+			"secp256r1",
+			"secp384r1"
+		]},
+		{"name": "ec_point_formats", "ec_point_format_list": [
+			"uncompressed"
+		]},
+		{"name": "session_ticket"},
+		{"name": "application_layer_protocol_negotiation", "protocol_name_list": [
+			"h2",
+			"http/1.1"
+		]},
+		{"name": "status_request"},
+		{"name": "signature_algorithms", "supported_signature_algorithms": [
+			"ecdsa_secp256r1_sha256",
+			"rsa_pss_rsae_sha256",
+			"rsa_pkcs1_sha256",
+			"ecdsa_secp384r1_sha384",
+			"rsa_pss_rsae_sha384",
+			"rsa_pkcs1_sha384",
+			"rsa_pss_rsae_sha512",
+			"rsa_pkcs1_sha512"
+		]},
+		{"name": "signed_certificate_timestamp"},
+		{"name": "key_share", "client_shares": [
+			{"group": "GREASE", "key_exchange": [0]},
+			{"group": "x25519"}
+		]},
+		{"name": "psk_key_exchange_modes", "ke_modes": [
+			"psk_dhe_ke"
+		]},
+		{"name": "supported_versions", "versions": [
+			"GREASE",
+			"TLS 1.3",
+			"TLS 1.2"
+		]},
+		{"name": "compress_certificate", "algorithms": [
+			"brotli"
+		]},
+		{"name": "application_settings", "supported_protocols": [
+			"h2"
+		]},
+		{"name": "GREASE"},
+		{"name": "padding", "len": 0}
+	]
+}

testdata/ClientHello-JSON-Edge106.json

@@ -0,0 +1,76 @@
+{
+	"cipher_suites": [
+        "GREASE",
+		"TLS_AES_128_GCM_SHA256",
+		"TLS_AES_256_GCM_SHA384",
+		"TLS_AES_256_GCM_SHA384",
+		"TLS_CHACHA20_POLY1305_SHA256",
+		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+		"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+		"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+		"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
+		"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
+		"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
+		"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
+		"TLS_RSA_WITH_AES_128_GCM_SHA256",
+		"TLS_RSA_WITH_AES_256_GCM_SHA384",
+		"TLS_RSA_WITH_AES_128_CBC_SHA",
+		"TLS_RSA_WITH_AES_256_CBC_SHA"
+	],
+	"compression_methods": [
+		"NULL"
+	],
+	"extensions": [
+		{"name": "GREASE"},
+		{"name": "server_name"},
+		{"name": "extended_master_secret"},
+		{"name": "renegotiation_info"},
+		{"name": "supported_groups", "named_group_list": [
+			"GREASE",
+			"x25519",
+			"secp256r1",
+			"secp384r1"
+		]},
+		{"name": "ec_point_formats", "ec_point_format_list": [
+			"uncompressed"
+		]},
+		{"name": "session_ticket"},
+		{"name": "application_layer_protocol_negotiation", "protocol_name_list": [
+			"h2",
+			"http/1.1"
+		]},
+		{"name": "status_request"},
+		{"name": "signature_algorithms", "supported_signature_algorithms": [
+			"ecdsa_secp256r1_sha256",
+			"rsa_pss_rsae_sha256",
+			"rsa_pkcs1_sha256",
+			"ecdsa_secp384r1_sha384",
+			"rsa_pss_rsae_sha384",
+			"rsa_pkcs1_sha384",
+			"rsa_pss_rsae_sha512",
+			"rsa_pkcs1_sha512"
+		]},
+		{"name": "signed_certificate_timestamp"},
+		{"name": "key_share", "client_shares": [
+			{"group": "GREASE", "key_exchange": [0]},
+			{"group": "x25519"}
+		]},
+		{"name": "psk_key_exchange_modes", "ke_modes": [
+			"psk_dhe_ke"
+		]},
+		{"name": "supported_versions", "versions": [
+			"GREASE",
+			"TLS 1.3",
+			"TLS 1.2"
+		]},
+		{"name": "compress_certificate", "algorithms": [
+			"brotli"
+		]},
+		{"name": "application_settings", "supported_protocols": [
+			"h2"
+		]},
+		{"name": "GREASE"},
+		{"name": "padding", "len": 0}
+	]
+}

testdata/ClientHello-JSON-Firefox105.json

@@ -0,0 +1,78 @@
+{
+	"cipher_suites": [
+        "TLS_AES_128_GCM_SHA256",
+		"TLS_CHACHA20_POLY1305_SHA256",
+		"TLS_AES_256_GCM_SHA384",
+		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+		"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
+		"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
+		"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+		"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+		"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA",
+		"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA",
+		"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
+		"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
+		"TLS_RSA_WITH_AES_128_GCM_SHA256",
+		"TLS_RSA_WITH_AES_256_GCM_SHA384",
+		"TLS_RSA_WITH_AES_128_CBC_SHA",
+		"TLS_RSA_WITH_AES_256_CBC_SHA"
+	],
+	"compression_methods": [
+		"NULL"
+	],
+	"extensions": [
+		{"name": "server_name"},
+		{"name": "extended_master_secret"},
+		{"name": "renegotiation_info"},
+		{"name": "supported_groups", "named_group_list": [
+			"x25519",
+			"secp256r1",
+			"secp384r1",
+			"secp521r1",
+			"ffdhe2048",
+	   		"ffdhe3072"
+		]},
+		{"name": "ec_point_formats", "ec_point_format_list": [
+			"uncompressed"
+		]},
+		{"name": "session_ticket"},
+		{"name": "application_layer_protocol_negotiation", "protocol_name_list": [
+			"h2",
+			"http/1.1"
+		]},
+		{"name": "status_request"},
+		{"name": "delegated_credentials", "supported_signature_algorithms": [
+			"ecdsa_secp256r1_sha256",
+			"ecdsa_secp384r1_sha384",
+			"ecdsa_secp521r1_sha512",
+			"ecdsa_sha1"
+		]},
+		{"name": "key_share", "client_shares": [
+			{"group": "x25519"},
+			{"group": "secp256r1"}
+		]},
+		{"name": "supported_versions", "versions": [
+			"TLS 1.3",
+			"TLS 1.2"
+		]},
+		{"name": "signature_algorithms", "supported_signature_algorithms": [
+			"ecdsa_secp256r1_sha256",
+			"ecdsa_secp384r1_sha384",
+			"ecdsa_secp521r1_sha512",
+			"rsa_pss_rsae_sha256",
+			"rsa_pss_rsae_sha384",
+			"rsa_pss_rsae_sha512",
+			"rsa_pkcs1_sha256",
+			"rsa_pkcs1_sha384",
+			"rsa_pkcs1_sha512",
+			"ecdsa_sha1",
+			"rsa_pkcs1_sha1"
+		]},
+		{"name": "psk_key_exchange_modes", "ke_modes": [
+			"psk_dhe_ke"
+		]},
+		{"name": "record_size_limit", "record_size_limit": 16385},
+		{"name": "padding", "len": 0}
+	]
+}

testdata/ClientHello-JSON-iOS14.json

@@ -0,0 +1,85 @@
+{
+	"cipher_suites": [
+        "GREASE",
+		"TLS_AES_128_GCM_SHA256",
+		"TLS_AES_256_GCM_SHA384",
+		"TLS_CHACHA20_POLY1305_SHA256",
+		"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+		"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
+		"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+		"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
+		"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384",
+		"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256",
+		"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA",
+		"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA",
+		"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
+		"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
+		"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
+		"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
+		"TLS_RSA_WITH_AES_256_GCM_SHA384",
+		"TLS_RSA_WITH_AES_128_GCM_SHA256",
+		"TLS_RSA_WITH_AES_256_CBC_SHA256",
+		"TLS_RSA_WITH_AES_128_CBC_SHA256",
+		"TLS_RSA_WITH_AES_256_CBC_SHA",
+		"TLS_RSA_WITH_AES_128_CBC_SHA",
+		"TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA",
+		"TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA",
+		"TLS_RSA_WITH_3DES_EDE_CBC_SHA"
+	],
+	"compression_methods": [
+		"NULL"
+	],
+	"extensions": [
+		{"name": "GREASE"},
+		{"name": "server_name"},
+		{"name": "extended_master_secret"},
+		{"name": "renegotiation_info"},
+		{"name": "supported_groups", "named_group_list": [
+			"GREASE",
+			"x25519",
+			"secp256r1",
+			"secp384r1",
+			"secp521r1"
+		]},
+		{"name": "ec_point_formats", "ec_point_format_list": [
+			"uncompressed"
+		]},
+		{"name": "application_layer_protocol_negotiation", "protocol_name_list": [
+			"h2",
+			"http/1.1"
+		]},
+		{"name": "status_request"},
+		{"name": "signature_algorithms", "supported_signature_algorithms": [
+			"ecdsa_secp256r1_sha256",
+			"rsa_pss_rsae_sha256",
+			"rsa_pkcs1_sha256",
+			"ecdsa_secp384r1_sha384",
+			"ecdsa_sha1",
+			"rsa_pss_rsae_sha384",
+			"rsa_pss_rsae_sha384",
+			"rsa_pkcs1_sha384",
+			"rsa_pss_rsae_sha512",
+			"rsa_pkcs1_sha512",
+			"rsa_pkcs1_sha1"
+		]},
+		{"name": "signed_certificate_timestamp"},
+		{"name": "key_share", "client_shares": [
+			{"group": "GREASE", "key_exchange": [0]},
+			{"group": "x25519"}
+		]},
+		{"name": "psk_key_exchange_modes", "ke_modes": [
+			"psk_dhe_ke"
+		]},
+		{"name": "supported_versions", "versions": [
+			"GREASE",
+			"TLS 1.3",
+			"TLS 1.2",
+			"TLS 1.1",
+			"TLS 1.0"
+		]},
+		{"name": "GREASE"},
+		{"name": "padding"}
+	]
+}

u_clienthello_json.go

@@ -0,0 +1,168 @@
+package tls
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/gaukas/godicttls"
+)
+
+var ErrUnknownExtension = errors.New("extension name is unknown to the dictionary")
+
+type ClientHelloSpecJSONUnmarshaler struct {
+	CipherSuites       *CipherSuitesJSONUnmarshaler       `json:"cipher_suites"`
+	CompressionMethods *CompressionMethodsJSONUnmarshaler `json:"compression_methods"`
+	Extensions         *TLSExtensionsJSONUnmarshaler      `json:"extensions"`
+	TLSVersMin         uint16                             `json:"min_vers,omitempty"` // optional
+	TLSVersMax         uint16                             `json:"max_vers,omitempty"` // optional
+}
+
+func (chsju *ClientHelloSpecJSONUnmarshaler) ClientHelloSpec() ClientHelloSpec {
+	return ClientHelloSpec{
+		CipherSuites:       chsju.CipherSuites.CipherSuites(),
+		CompressionMethods: chsju.CompressionMethods.CompressionMethods(),
+		Extensions:         chsju.Extensions.Extensions(),
+		TLSVersMin:         chsju.TLSVersMin,
+		TLSVersMax:         chsju.TLSVersMax,
+	}
+}
+
+type CipherSuitesJSONUnmarshaler struct {
+	cipherSuites []uint16
+}
+
+func (c *CipherSuitesJSONUnmarshaler) UnmarshalJSON(jsonStr []byte) error {
+	var cipherSuiteNames []string
+	if err := json.Unmarshal(jsonStr, &cipherSuiteNames); err != nil {
+		return err
+	}
+
+	for _, name := range cipherSuiteNames {
+		if name == "GREASE" {
+			c.cipherSuites = append(c.cipherSuites, GREASE_PLACEHOLDER)
+			continue
+		}
+
+		if id, ok := godicttls.DictCipherSuiteNameIndexed[name]; ok {
+			c.cipherSuites = append(c.cipherSuites, id)
+		} else {
+			return fmt.Errorf("unknown cipher suite name: %s", name)
+		}
+	}
+
+	return nil
+}
+
+func (c *CipherSuitesJSONUnmarshaler) CipherSuites() []uint16 {
+	return c.cipherSuites
+}
+
+type CompressionMethodsJSONUnmarshaler struct {
+	compressionMethods []uint8
+}
+
+func (c *CompressionMethodsJSONUnmarshaler) UnmarshalJSON(jsonStr []byte) error {
+	var compressionMethodNames []string
+	if err := json.Unmarshal(jsonStr, &compressionMethodNames); err != nil {
+		return err
+	}
+
+	for _, name := range compressionMethodNames {
+		if id, ok := godicttls.DictCompMethNameIndexed[name]; ok {
+			c.compressionMethods = append(c.compressionMethods, id)
+		} else {
+			return fmt.Errorf("unknown compression method name: %s", name)
+		}
+	}
+
+	return nil
+}
+
+func (c *CompressionMethodsJSONUnmarshaler) CompressionMethods() []uint8 {
+	return c.compressionMethods
+}
+
+type TLSExtensionsJSONUnmarshaler struct {
+	extensions []TLSExtensionJSON
+}
+
+func (e *TLSExtensionsJSONUnmarshaler) UnmarshalJSON(jsonStr []byte) error {
+	var accepters []tlsExtensionJSONAccepter
+	if err := json.Unmarshal(jsonStr, &accepters); err != nil {
+		return err
+	}
+
+	var exts []TLSExtensionJSON = make([]TLSExtensionJSON, 0, len(accepters))
+	for _, accepter := range accepters {
+		if accepter.extNameOnly.Name == "GREASE" {
+			exts = append(exts, &UtlsGREASEExtension{})
+			continue
+		}
+
+		if extID, ok := godicttls.DictExtTypeNameIndexed[accepter.extNameOnly.Name]; !ok {
+			return fmt.Errorf("%w: %s", ErrUnknownExtension, accepter.extNameOnly.Name)
+		} else {
+			// get extension type from ID
+			var ext TLSExtension = ExtensionFromID(extID)
+			if ext == nil {
+				// fallback to generic extension
+				ext = genericExtension(extID, accepter.extNameOnly.Name)
+			}
+
+			if extJsonCompatible, ok := ext.(TLSExtensionJSON); ok {
+				exts = append(exts, extJsonCompatible)
+			} else {
+				return fmt.Errorf("extension %d (%s) is not JSON compatible", extID, accepter.extNameOnly.Name)
+			}
+		}
+	}
+
+	// unmashal extensions
+	for idx, ext := range exts {
+		// json.Unmarshal will call the UnmarshalJSON method of the extension
+		if err := json.Unmarshal(accepters[idx].origJsonInput, ext); err != nil {
+			return err
+		}
+	}
+
+	e.extensions = exts
+	return nil
+}
+
+func (e *TLSExtensionsJSONUnmarshaler) Extensions() []TLSExtension {
+	var exts []TLSExtension = make([]TLSExtension, 0, len(e.extensions))
+	for _, ext := range e.extensions {
+		exts = append(exts, ext)
+	}
+	return exts
+}
+
+func genericExtension(id uint16, name string) TLSExtension {
+	var warningMsg string = "WARNING: extension "
+	warningMsg += fmt.Sprintf("%d ", id)
+	if len(name) > 0 {
+		warningMsg += fmt.Sprintf("(%s) ", name)
+	}
+	warningMsg += "is falling back to generic extension"
+	warningMsg += "\n"
+
+	fmt.Fprint(os.Stderr, warningMsg)
+
+	// fallback to generic extension
+	return &GenericExtension{Id: id}
+}
+
+type tlsExtensionJSONAccepter struct {
+	extNameOnly struct {
+		Name string `json:"name"`
+	}
+	origJsonInput []byte
+}
+
+func (t *tlsExtensionJSONAccepter) UnmarshalJSON(jsonStr []byte) error {
+	t.origJsonInput = make([]byte, len(jsonStr))
+	copy(t.origJsonInput, jsonStr)
+	return json.Unmarshal(jsonStr, &t.extNameOnly)
+}

u_clienthello_json_test.go

@@ -0,0 +1,123 @@
+package tls
+
+import (
+	"encoding/json"
+	"os"
+	"reflect"
+	"testing"
+)
+
+func TestClientHelloSpecJSONUnmarshaler(t *testing.T) {
+	testClientHelloSpecJSONUnmarshaler(t, "testdata/ClientHello-JSON-Chrome102.json", HelloChrome_102)
+	testClientHelloSpecJSONUnmarshaler(t, "testdata/ClientHello-JSON-Firefox105.json", HelloFirefox_105)
+	testClientHelloSpecJSONUnmarshaler(t, "testdata/ClientHello-JSON-iOS14.json", HelloIOS_14)
+	testClientHelloSpecJSONUnmarshaler(t, "testdata/ClientHello-JSON-Edge106.json", HelloEdge_106)
+}
+
+func testClientHelloSpecJSONUnmarshaler(
+	t *testing.T,
+	jsonFilepath string,
+	truthClientHelloID ClientHelloID,
+) {
+	jsonCH, err := os.ReadFile(jsonFilepath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var chsju ClientHelloSpecJSONUnmarshaler
+	if err := json.Unmarshal(jsonCH, &chsju); err != nil {
+		t.Fatal(err)
+	}
+
+	truthSpec, _ := utlsIdToSpec(truthClientHelloID)
+	jsonSpec := chsju.ClientHelloSpec()
+
+	// Compare CipherSuites
+	if !reflect.DeepEqual(jsonSpec.CipherSuites, truthSpec.CipherSuites) {
+		t.Errorf("JSONUnmarshaler %s: got %#v, want %#v", clientHelloSpecJSONTestIdentifier(truthClientHelloID), jsonSpec.CipherSuites, truthSpec.CipherSuites)
+	}
+
+	// Compare CompressionMethods
+	if !reflect.DeepEqual(jsonSpec.CompressionMethods, truthSpec.CompressionMethods) {
+		t.Errorf("JSONUnmarshaler %s: got %#v, want %#v", clientHelloSpecJSONTestIdentifier(truthClientHelloID), jsonSpec.CompressionMethods, truthSpec.CompressionMethods)
+	}
+
+	// Compare Extensions
+	if len(jsonSpec.Extensions) != len(truthSpec.Extensions) {
+		t.Errorf("JSONUnmarshaler %s: len(jsonExtensions) = %d != %d = len(truthExtensions)", clientHelloSpecJSONTestIdentifier(truthClientHelloID), len(jsonSpec.Extensions), len(truthSpec.Extensions))
+	}
+
+	for i := range jsonSpec.Extensions {
+		if !reflect.DeepEqual(jsonSpec.Extensions[i], truthSpec.Extensions[i]) {
+			if _, ok := jsonSpec.Extensions[i].(*UtlsPaddingExtension); ok {
+				testedPaddingExt := jsonSpec.Extensions[i].(*UtlsPaddingExtension)
+				savedPaddingExt := truthSpec.Extensions[i].(*UtlsPaddingExtension)
+				if testedPaddingExt.PaddingLen != savedPaddingExt.PaddingLen || testedPaddingExt.WillPad != savedPaddingExt.WillPad {
+					t.Errorf("got %#v, want %#v", testedPaddingExt, savedPaddingExt)
+				} else {
+					continue // UtlsPaddingExtension has non-nil function member
+				}
+			}
+			t.Errorf("JSONUnmarshaler %s: got %#v, want %#v", clientHelloSpecJSONTestIdentifier(truthClientHelloID), jsonSpec.Extensions[i], truthSpec.Extensions[i])
+		}
+	}
+}
+
+func TestClientHelloSpecUnmarshalJSON(t *testing.T) {
+	testClientHelloSpecUnmarshalJSON(t, "testdata/ClientHello-JSON-Chrome102.json", HelloChrome_102)
+	testClientHelloSpecUnmarshalJSON(t, "testdata/ClientHello-JSON-Firefox105.json", HelloFirefox_105)
+	testClientHelloSpecUnmarshalJSON(t, "testdata/ClientHello-JSON-iOS14.json", HelloIOS_14)
+	testClientHelloSpecUnmarshalJSON(t, "testdata/ClientHello-JSON-Edge106.json", HelloEdge_106)
+}
+
+func testClientHelloSpecUnmarshalJSON(
+	t *testing.T,
+	jsonFilepath string,
+	truthClientHelloID ClientHelloID,
+) {
+	var jsonSpec ClientHelloSpec
+	jsonCH, err := os.ReadFile(jsonFilepath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := json.Unmarshal(jsonCH, &jsonSpec); err != nil {
+		t.Fatal(err)
+	}
+
+	truthSpec, _ := utlsIdToSpec(truthClientHelloID)
+
+	// Compare CipherSuites
+	if !reflect.DeepEqual(jsonSpec.CipherSuites, truthSpec.CipherSuites) {
+		t.Errorf("UnmarshalJSON %s: got %#v, want %#v", clientHelloSpecJSONTestIdentifier(truthClientHelloID), jsonSpec.CipherSuites, truthSpec.CipherSuites)
+	}
+
+	// Compare CompressionMethods
+	if !reflect.DeepEqual(jsonSpec.CompressionMethods, truthSpec.CompressionMethods) {
+		t.Errorf("UnmarshalJSON %s: got %#v, want %#v", clientHelloSpecJSONTestIdentifier(truthClientHelloID), jsonSpec.CompressionMethods, truthSpec.CompressionMethods)
+	}
+
+	// Compare Extensions
+	if len(jsonSpec.Extensions) != len(truthSpec.Extensions) {
+		t.Errorf("UnmarshalJSON %s: len(jsonExtensions) = %d != %d = len(truthExtensions)", jsonFilepath, len(jsonSpec.Extensions), len(truthSpec.Extensions))
+	}
+
+	for i := range jsonSpec.Extensions {
+		if !reflect.DeepEqual(jsonSpec.Extensions[i], truthSpec.Extensions[i]) {
+			if _, ok := jsonSpec.Extensions[i].(*UtlsPaddingExtension); ok {
+				testedPaddingExt := jsonSpec.Extensions[i].(*UtlsPaddingExtension)
+				savedPaddingExt := truthSpec.Extensions[i].(*UtlsPaddingExtension)
+				if testedPaddingExt.PaddingLen != savedPaddingExt.PaddingLen || testedPaddingExt.WillPad != savedPaddingExt.WillPad {
+					t.Errorf("got %#v, want %#v", testedPaddingExt, savedPaddingExt)
+				} else {
+					continue // UtlsPaddingExtension has non-nil function member
+				}
+			}
+			t.Errorf("UnmarshalJSON %s: got %#v, want %#v", clientHelloSpecJSONTestIdentifier(truthClientHelloID), jsonSpec.Extensions[i], truthSpec.Extensions[i])
+		}
+	}
+}
+
+func clientHelloSpecJSONTestIdentifier(id ClientHelloID) string {
+	return id.Client + id.Version
+}

u_common.go

@@ -210,8 +210,9 @@ func (chs *ClientHelloSpec) ReadTLSExtensions(b []byte, allowBluntMimicry bool)
 			return fmt.Errorf("unable to read data for extension %x", extension)
 		}
 
-		extWriter := ExtensionIDToExtension(extension)
+		ext := ExtensionFromID(extension)
-		if extWriter != nil {
+		extWriter, ok := ext.(TLSExtensionWriter)
+		if ext != nil && ok { // known extension and implements TLSExtensionWriter properly
 			if extension == extensionSupportedVersions {
 				chs.TLSVersMin = 0
 				chs.TLSVersMax = 0
@@ -293,8 +294,12 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 	}
 
 	for _, extType := range tlsExtensionTypes {
-		extension := ExtensionIDToExtension(extType)
+		extension := ExtensionFromID(extType)
-		if extension == nil {
+		extWriter, ok := extension.(TLSExtensionWriter)
+		if !ok {
+			return fmt.Errorf("unsupported extension %d", extType)
+		}
+		if extension == nil || !ok {
 			log.Printf("[Warning] Unsupported extension %d added as a &GenericExtension without Data", extType)
 			chs.Extensions = append(chs.Extensions, &GenericExtension{extType, []byte{}})
 		} else {
@@ -303,7 +308,7 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 				if data["pt_fmts"] == nil {
 					return errors.New("pt_fmts is required")
 				}
-				_, err = extension.Write(data["pt_fmts"])
+				_, err = extWriter.Write(data["pt_fmts"])
 				if err != nil {
 					return err
 				}
@@ -311,7 +316,7 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 				if data["sig_algs"] == nil {
 					return errors.New("sig_algs is required")
 				}
-				_, err = extension.Write(data["sig_algs"])
+				_, err = extWriter.Write(data["sig_algs"])
 				if err != nil {
 					return err
 				}
@@ -327,7 +332,7 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 				fixedData := make([]byte, len(data["supported_versions"])+1)
 				fixedData[0] = uint8(len(data["supported_versions"]) & 0xff)
 				copy(fixedData[1:], data["supported_versions"])
-				_, err = extension.Write(fixedData)
+				_, err = extWriter.Write(fixedData)
 				if err != nil {
 					return err
 				}
@@ -336,7 +341,7 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 					return errors.New("curves is required")
 				}
 
-				_, err = extension.Write(data["curves"])
+				_, err = extWriter.Write(data["curves"])
 				if err != nil {
 					return err
 				}
@@ -345,7 +350,7 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 					return errors.New("alpn is required")
 				}
 
-				_, err = extension.Write(data["alpn"])
+				_, err = extWriter.Write(data["alpn"])
 				if err != nil {
 					return err
 				}
@@ -365,7 +370,7 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 				// add uint16 length prefix
 				fixedData = append([]byte{uint8(len(fixedData) >> 8), uint8(len(fixedData) & 0xff)}, fixedData...)
 
-				_, err = extension.Write(fixedData)
+				_, err = extWriter.Write(fixedData)
 				if err != nil {
 					return err
 				}
@@ -378,7 +383,7 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 				fixedData := make([]byte, len(data["psk_key_exchange_modes"])+1)
 				fixedData[0] = uint8(len(data["psk_key_exchange_modes"]) & 0xff)
 				copy(fixedData[1:], data["psk_key_exchange_modes"])
-				_, err = extension.Write(fixedData)
+				_, err = extWriter.Write(fixedData)
 				if err != nil {
 					return err
 				}
@@ -391,7 +396,7 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 				fixedData := make([]byte, len(data["cert_compression_algs"])+1)
 				fixedData[0] = uint8(len(data["cert_compression_algs"]) & 0xff)
 				copy(fixedData[1:], data["cert_compression_algs"])
-				_, err = extension.Write(fixedData)
+				_, err = extWriter.Write(fixedData)
 				if err != nil {
 					return err
 				}
@@ -400,13 +405,13 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 					return errors.New("record_size_limit is required")
 				}
 
-				_, err = extension.Write(data["record_size_limit"]) // uint16 as []byte
+				_, err = extWriter.Write(data["record_size_limit"]) // uint16 as []byte
 				if err != nil {
 					return err
 				}
 			case utlsExtensionApplicationSettings:
 				// TODO: tlsfingerprint.io should record/provide application settings data
-				extension.(*ApplicationSettingsExtension).SupportedProtocols = []string{"h2"}
+				extWriter.(*ApplicationSettingsExtension).SupportedProtocols = []string{"h2"}
 			case fakeExtensionPreSharedKey:
 				log.Printf("[Warning] PSK extension added without data")
 			default:
@@ -416,12 +421,15 @@ func (chs *ClientHelloSpec) ImportTLSClientHello(data map[string][]byte) error {
 					log.Printf("[Warning] GREASE extension added but ID/Data discarded. They will be automatically re-GREASEd on ApplyPreset() call.")
 				}*/
 			}
-			chs.Extensions = append(chs.Extensions, extension)
+			chs.Extensions = append(chs.Extensions, extWriter)
 		}
 	}
 	return nil
 }
 
+// ImportTLSClientHelloFromJSON imports ClientHelloSpec from JSON data from client.tlsfingerprint.io format
+//
+// It calls ImportTLSClientHello internally after unmarshaling JSON data into map[string][]byte
 func (chs *ClientHelloSpec) ImportTLSClientHelloFromJSON(jsonB []byte) error {
 	var data map[string][]byte
 	err := json.Unmarshal(jsonB, &data)
@@ -431,6 +439,99 @@ func (chs *ClientHelloSpec) ImportTLSClientHelloFromJSON(jsonB []byte) error {
 	return chs.ImportTLSClientHello(data)
 }
 
+// FromRaw converts a ClientHello message in the form of raw bytes into a ClientHelloSpec.
+func (chs *ClientHelloSpec) FromRaw(raw []byte, allowBluntMimicry ...bool) error {
+	if chs == nil {
+		return errors.New("cannot unmarshal into nil ClientHelloSpec")
+	}
+
+	var bluntMimicry = false
+	if len(allowBluntMimicry) == 1 {
+		bluntMimicry = allowBluntMimicry[0]
+	}
+
+	*chs = ClientHelloSpec{} // reset
+	s := cryptobyte.String(raw)
+
+	var contentType uint8
+	var recordVersion uint16
+	if !s.ReadUint8(&contentType) || // record type
+		!s.ReadUint16(&recordVersion) || !s.Skip(2) { // record version and length
+		return errors.New("unable to read record type, version, and length")
+	}
+
+	if recordType(contentType) != recordTypeHandshake {
+		return errors.New("record is not a handshake")
+	}
+
+	var handshakeVersion uint16
+	var handshakeType uint8
+
+	if !s.ReadUint8(&handshakeType) || !s.Skip(3) || // message type and 3 byte length
+		!s.ReadUint16(&handshakeVersion) || !s.Skip(32) { // 32 byte random
+		return errors.New("unable to read handshake message type, length, and random")
+	}
+
+	if handshakeType != typeClientHello {
+		return errors.New("handshake message is not a ClientHello")
+	}
+
+	chs.TLSVersMin = recordVersion
+	chs.TLSVersMax = handshakeVersion
+
+	var ignoredSessionID cryptobyte.String
+	if !s.ReadUint8LengthPrefixed(&ignoredSessionID) {
+		return errors.New("unable to read session id")
+	}
+
+	// CipherSuites
+	var cipherSuitesBytes cryptobyte.String
+	if !s.ReadUint16LengthPrefixed(&cipherSuitesBytes) {
+		return errors.New("unable to read ciphersuites")
+	}
+
+	if err := chs.ReadCipherSuites(cipherSuitesBytes); err != nil {
+		return err
+	}
+
+	// CompressionMethods
+	var compressionMethods cryptobyte.String
+	if !s.ReadUint8LengthPrefixed(&compressionMethods) {
+		return errors.New("unable to read compression methods")
+	}
+
+	if err := chs.ReadCompressionMethods(compressionMethods); err != nil {
+		return err
+	}
+
+	if s.Empty() {
+		// Extensions are optional
+		return nil
+	}
+
+	var extensions cryptobyte.String
+	if !s.ReadUint16LengthPrefixed(&extensions) {
+		return errors.New("unable to read extensions data")
+	}
+
+	if err := chs.ReadTLSExtensions(extensions, bluntMimicry); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// UnmarshalJSON unmarshals a ClientHello message in the form of JSON into a ClientHelloSpec.
+func (chs *ClientHelloSpec) UnmarshalJSON(jsonB []byte) error {
+	var chsju ClientHelloSpecJSONUnmarshaler
+	if err := json.Unmarshal(jsonB, &chsju); err != nil {
+		return err
+	}
+
+	*chs = chsju.ClientHelloSpec()
+	return nil
+}
+
 var (
 	// HelloGolang will use default "crypto/tls" handshake marshaling codepath, which WILL
 	// overwrite your changes to Hello(Config, Session are fine).

u_fingerprinter.go

@@ -4,12 +4,6 @@
 
 package tls
 
-import (
-	"errors"
-
-	"golang.org/x/crypto/cryptobyte"
-)
-
 // Fingerprinter is a struct largely for holding options for the FingerprintClientHello func
 type Fingerprinter struct {
 	// AllowBluntMimicry will ensure that unknown extensions are
@@ -36,72 +30,36 @@ type Fingerprinter struct {
 // as well as the handshake type/length/version header
 // https://tools.ietf.org/html/rfc5246#section-6.2
 // https://tools.ietf.org/html/rfc5246#section-7.4
+//
+// It calls UnmarshalClientHello internally, and is kept for backwards compatibility
 func (f *Fingerprinter) FingerprintClientHello(data []byte) (clientHelloSpec *ClientHelloSpec, err error) {
+	return f.RawClientHello(data)
+}
+
+// RawClientHello returns a ClientHelloSpec which is based on the
+// ClientHello raw bytes that is passed in as the raw argument.
+//
+// It was renamed from FingerprintClientHello in v1.3.1 and earlier versions
+// as a more precise name for the function
+func (f *Fingerprinter) RawClientHello(raw []byte) (clientHelloSpec *ClientHelloSpec, err error) {
 	clientHelloSpec = &ClientHelloSpec{}
-	s := cryptobyte.String(data)
+	err = clientHelloSpec.FromRaw(raw, f.AllowBluntMimicry)
-
+	if err != nil {
-	var contentType uint8
+		return nil, err
-	var recordVersion uint16
+	}
-	if !s.ReadUint8(&contentType) || // record type
+
-		!s.ReadUint16(&recordVersion) || !s.Skip(2) { // record version and length
+	if f.AlwaysAddPadding {
-		return nil, errors.New("unable to read record type, version, and length")
+		clientHelloSpec.AlwaysAddPadding()
-	}
+	}
-
+
-	if recordType(contentType) != recordTypeHandshake {
+	return clientHelloSpec, nil
-		return nil, errors.New("record is not a handshake")
+}
-	}
+
-
+// UnmarshalJSONClientHello returns a ClientHelloSpec which is based on the
-	var handshakeVersion uint16
+// ClientHello JSON bytes that is passed in as the json argument.
-	var handshakeType uint8
+func (f *Fingerprinter) UnmarshalJSONClientHello(json []byte) (clientHelloSpec *ClientHelloSpec, err error) {
-
+	clientHelloSpec = &ClientHelloSpec{}
-	if !s.ReadUint8(&handshakeType) || !s.Skip(3) || // message type and 3 byte length
+	err = clientHelloSpec.UnmarshalJSON(json)
-		!s.ReadUint16(&handshakeVersion) || !s.Skip(32) { // 32 byte random
-		return nil, errors.New("unable to read handshake message type, length, and random")
-	}
-
-	if handshakeType != typeClientHello {
-		return nil, errors.New("handshake message is not a ClientHello")
-	}
-
-	clientHelloSpec.TLSVersMin = recordVersion
-	clientHelloSpec.TLSVersMax = handshakeVersion
-
-	var ignoredSessionID cryptobyte.String
-	if !s.ReadUint8LengthPrefixed(&ignoredSessionID) {
-		return nil, errors.New("unable to read session id")
-	}
-
-	// CipherSuites
-	var cipherSuitesBytes cryptobyte.String
-	if !s.ReadUint16LengthPrefixed(&cipherSuitesBytes) {
-		return nil, errors.New("unable to read ciphersuites")
-	}
-	err = clientHelloSpec.ReadCipherSuites(cipherSuitesBytes)
-	if err != nil {
-		return nil, err
-	}
-
-	// CompressionMethods
-	var compressionMethods cryptobyte.String
-	if !s.ReadUint8LengthPrefixed(&compressionMethods) {
-		return nil, errors.New("unable to read compression methods")
-	}
-	err = clientHelloSpec.ReadCompressionMethods(compressionMethods)
-	if err != nil {
-		return nil, err
-	}
-
-	if s.Empty() {
-		// ClientHello is optionally followed by extension data
-		return clientHelloSpec, nil
-	}
-
-	var extensions cryptobyte.String
-	if !s.ReadUint16LengthPrefixed(&extensions) {
-		return nil, errors.New("unable to read extensions data")
-	}
-
-	err = clientHelloSpec.ReadTLSExtensions(extensions, f.AllowBluntMimicry)
 	if err != nil {
 		return nil, err
 	}

u_parrots.go

@@ -680,8 +680,8 @@ func utlsIdToSpec(id ClientHelloID) (ClientHelloSpec, error) {
 				&SessionTicketExtension{},
 				&ALPNExtension{AlpnProtocols: []string{"h2", "http/1.1"}}, //application_layer_protocol_negotiation
 				&StatusRequestExtension{},
-				&DelegatedCredentialsExtension{
+				&FakeDelegatedCredentialsExtension{
-					AlgorithmsSignature: []SignatureScheme{ //signature_algorithms
+					SupportedSignatureAlgorithms: []SignatureScheme{ //signature_algorithms
 						ECDSAWithP256AndSHA256,
 						ECDSAWithP384AndSHA384,
 						ECDSAWithP521AndSHA512,
@@ -761,8 +761,8 @@ func utlsIdToSpec(id ClientHelloID) (ClientHelloSpec, error) {
 				&SessionTicketExtension{},
 				&ALPNExtension{AlpnProtocols: []string{"h2"}}, //application_layer_protocol_negotiation
 				&StatusRequestExtension{},
-				&DelegatedCredentialsExtension{
+				&FakeDelegatedCredentialsExtension{
-					AlgorithmsSignature: []SignatureScheme{ //signature_algorithms
+					SupportedSignatureAlgorithms: []SignatureScheme{ //signature_algorithms
 						ECDSAWithP256AndSHA256,
 						ECDSAWithP384AndSHA384,
 						ECDSAWithP521AndSHA512,

u_public.go

@@ -543,8 +543,8 @@ func (fh *finishedHash) getPublicObj() FinishedHash {
 
 // TLS 1.3 Key Share. See RFC 8446, Section 4.2.8.
 type KeyShare struct {
-	Group CurveID
+	Group CurveID `json:"group"`
-	Data  []byte
+	Data  []byte  `json:"key_exchange,omitempty"` // optional
 }
 
 type KeyShares []KeyShare
@@ -568,8 +568,8 @@ func (KSS KeyShares) ToPrivate() []keyShare {
 // TLS 1.3 PSK Identity. Can be a Session Ticket, or a reference to a saved
 // session. See RFC 8446, Section 4.2.11.
 type PskIdentity struct {
-	Label               []byte
+	Label               []byte `json:"identity"`
-	ObfuscatedTicketAge uint32
+	ObfuscatedTicketAge uint32 `json:"obfuscated_ticket_age"`
 }
 
 type PskIdentities []PskIdentity