net.http: change header behavior to keep custom header case (#9602)

vlib/net/http/header.v

@@ -3,6 +3,8 @@
 // that can be found in the LICENSE file.
 module http
 
+import strings
+
 // CommonHeader is an enum of the most common HTTP headers
 pub enum CommonHeader {
 	accept
@@ -323,11 +325,15 @@ const common_header_map = map{
 pub struct Header {
 mut:
 	data map[string][]string
+	// map of lowercase header keys to their original keys
+	// in order of appearance
+	keys map[string][]string
 }
 
 pub fn (mut h Header) free() {
 	unsafe {
 		h.data.free()
+		h.keys.free()
 	}
 }
 
@@ -349,80 +355,136 @@ pub fn new_header(kvs ...HeaderConfig) Header {
 
 // Append a value to the header key.
 pub fn (mut h Header) add(key CommonHeader, value string) {
-	h.data[key.str()] << value
+	k := key.str()
+	h.data[k] << value
+	h.add_key(k)
 }
 
 // Append a value to a custom header key. This function will return an error
 // if the key contains invalid header characters.
-pub fn (mut h Header) add_str(key string, value string) ? {
+pub fn (mut h Header) add_custom(key string, value string) ? {
-	k := canonicalize(key) ?
+	is_valid(key) ?
-	h.data[k] << value
+	h.data[key] << value
+	h.add_key(key)
 }
 
 // Sets the key-value pair. This function will clear any other values
 // that exist for the CommonHeader.
 pub fn (mut h Header) set(key CommonHeader, value string) {
-	h.data[key.str()] = [value]
+	k := key.str()
+	h.data[k] = [value]
+	h.add_key(k)
 }
 
 // Sets the key-value pair for a custom header key. This function will
-// clear any other values that exist for the CommonHeader.
+// clear any other values that exist for the header. This function will
-pub fn (mut h Header) set_str(key string, value string) {
+// return an error if the key contains invalid header characters.
-	k := canonicalize(key) or { return }
+pub fn (mut h Header) set_custom(key string, value string) ? {
-	h.data[k] = [value]
+	is_valid(key) ?
+	h.data[key] = [value]
+	h.add_key(key)
 }
 
 // Delete all values for a key.
 pub fn (mut h Header) delete(key CommonHeader) {
-	h.data.delete(key.str())
+	h.delete_custom(key.str())
 }
 
 // Delete all values for a custom header key.
-pub fn (mut h Header) delete_str(key string) {
+pub fn (mut h Header) delete_custom(key string) {
-	k := canonicalize(key) or { return }
+	h.data.delete(key)
-	h.data.delete(k)
+
+	// remove key from keys metadata
+	kl := key.to_lower()
+	if kl in h.keys {
+		h.keys[kl] = h.keys[kl].filter(it != key)
+	}
+}
+
+pub struct HeaderCoerceConfig {
+	canonicalize bool
+}
+
+// Coerce data by joining keys that match case-insensitively into one entry
+pub fn (mut h Header) coerce(flags ...HeaderCoerceConfig) {
+	canon := flags.any(it.canonicalize)
+
+	for kl, data_keys in h.keys {
+		master_key := if canon { canonicalize(kl) } else { data_keys[0] }
+
+		// save master data
+		master_data := h.data[master_key]
+		h.data.delete(master_key)
+
+		for key in data_keys {
+			if key == master_key {
+				h.data[master_key] << master_data
+				continue
+			}
+			h.data[master_key] << h.data[key]
+			h.data.delete(key)
+		}
+		h.keys[kl] = [master_key]
+	}
 }
 
 // Returns whether the header key exists in the map.
 pub fn (h Header) contains(key CommonHeader) bool {
-	return key.str() in h.data
+	return h.contains_custom(key.str())
+}
+
+pub struct HeaderQueryConfig {
+	exact bool
 }
 
 // Returns whether the custom header key exists in the map.
-pub fn (h Header) contains_str(key string) bool {
+pub fn (h Header) contains_custom(key string, flags ...HeaderQueryConfig) bool {
-	k := canonicalize(key) or { return false }
+	if flags.any(it.exact) {
-	return k in h.data
+		return key in h.data
+	}
+	return key.to_lower() in h.keys
 }
 
 // Gets the first value for the CommonHeader, or none if the key does
 // not exist.
 pub fn (h Header) get(key CommonHeader) ?string {
-	k := key.str()
+	return h.get_custom(key.str())
-	if h.data[k].len == 0 {
-		return none
-	}
-	return h.data[k][0]
 }
 
 // Gets the first value for the custom header, or none if the key does
 // not exist.
-pub fn (h Header) get_str(key string) ?string {
+pub fn (h Header) get_custom(key string, flags ...HeaderQueryConfig) ?string {
-	k := canonicalize(key) or { return none }
+	mut data_key := key
-	if h.data[k].len == 0 {
+	if !flags.any(it.exact) {
+		// get the first key from key metadata
+		k := key.to_lower()
+		if h.keys[k].len == 0 {
+			return none
+		}
+		data_key = h.keys[k][0]
+	}
+	if h.data[data_key].len == 0 {
 		return none
 	}
-	return h.data[k][0]
+	return h.data[data_key][0]
 }
 
 // Gets all values for the CommonHeader.
 pub fn (h Header) values(key CommonHeader) []string {
-	return h.data[key.str()]
+	return h.custom_values(key.str())
 }
 
 // Gets all values for the custom header.
-pub fn (h Header) values_str(key string) []string {
+pub fn (h Header) custom_values(key string, flags ...HeaderQueryConfig) []string {
-	k := canonicalize(key) or { return [] }
+	if flags.any(it.exact) {
-	return h.data[k]
+		return h.data[key]
+	}
+	// case insensitive lookup
+	mut values := []string{cap: 10}
+	for k in h.keys[key.to_lower()] {
+		values << h.data[k]
+	}
+	return values
 }
 
 // Gets all header keys as strings
@@ -430,37 +492,87 @@ pub fn (h Header) keys() []string {
 	return h.data.keys()
 }
 
-// Validate and canonicalize an HTTP header key
+pub struct HeaderRenderConfig {
-// A canonical header is all lowercase except for the first character
+	version      Version
-// and any character after a `-`. Example: `Example-Header-Key`
+	coerce       bool
-// There are some exceptions like `DNT`, `WWW-Authenticate`, etc. For these we
+	canonicalize bool
-// check if the lowercase matches any in the common_header_map and return that.
+}
-fn canonicalize(s string) ?string {
+
-	// check for valid header bytes
+// Renders the Header into a string for use in sending
-	for _, c in s {
+// HTTP requests. All header lines will end in `\n\r`
+[manualfree]
+pub fn (h Header) render(flags HeaderRenderConfig) string {
+	// estimate ~48 bytes per header
+	mut sb := strings.new_builder(h.data.len * 48)
+	if flags.coerce {
+		for kl, data_keys in h.keys {
+			key := if flags.version == .v2_0 {
+				kl
+			} else if flags.canonicalize {
+				canonicalize(kl)
+			} else {
+				data_keys[0]
+			}
+			sb.write_string(key)
+			sb.write_string(': ')
+			for i in 0 .. data_keys.len - 1 {
+				k := data_keys[i]
+				for v in h.data[k] {
+					sb.write_string(v)
+					sb.write_string(',')
+				}
+			}
+			k := data_keys[data_keys.len - 1]
+			sb.write_string(h.data[k].join(','))
+			sb.write_string('\n\r')
+		}
+	} else {
+		for k, v in h.data {
+			key := if flags.version == .v2_0 {
+				k.to_lower()
+			} else if flags.canonicalize {
+				canonicalize(k.to_lower())
+			} else {
+				k
+			}
+			sb.write_string(key)
+			sb.write_string(': ')
+			sb.write_string(v.join(','))
+			sb.write_string('\n\r')
+		}
+	}
+	res := sb.str()
+	unsafe { sb.free() }
+	return res
+}
+
+// Canonicalize an HTTP header key
+// Common headers are determined by the common_header_map
+// Custom headers are capitalized on the first letter and any letter after a '-'
+// NOTE: Assumes sl is lowercase, since the caller usually already has the lowercase key
+fn canonicalize(sl string) string {
+	// check if we have a common header
+	if sl in http.common_header_map {
+		return http.common_header_map[sl].str()
+	}
+	return sl.split('-').map(it.capitalize()).join('-')
+}
+
+// Helper function to add a key to the keys map
+fn (mut h Header) add_key(key string) {
+	kl := key.to_lower()
+	if !h.keys[kl].contains(key) {
+		h.keys[kl] << key
+	}
+}
+
+// Checks if the header token is valid
+fn is_valid(header string) ? {
+	for _, c in header {
 		if int(c) >= 128 || !is_token(c) {
 			return error('Invalid header key')
 		}
 	}
-
-	// check if we have a common header
-	sl := s.to_lower()
-	if sl in http.common_header_map {
-		return http.common_header_map[sl].str()
-	}
-
-	// check for canonicalization; create a new string if not
-	mut upper := true
-	for _, c in s {
-		if upper && `a` <= c && c <= `z` {
-			return s.to_lower().split('-').map(it.capitalize()).join('-')
-		}
-		if !upper && `A` <= c && c <= `Z` {
-			return s.to_lower().split('-').map(it.capitalize()).join('-')
-		}
-		upper = c == `-`
-	}
-	return s
 }
 
 // Checks if the byte is valid for a header token
@@ -470,3 +582,9 @@ fn is_token(b byte) bool {
 		else { false }
 	}
 }
+
+// Returns the headers string as seen in HTTP/1.1 requests
+// Key order is not guaranteed
+pub fn (h Header) str() string {
+	return h.render(version: .v1_1)
+}

vlib/net/http/header_test.v

@@ -1,11 +1,11 @@
-import net.http
+module http
 
 fn test_header_new() {
 	h := http.new_header(
 		{key: .accept, value: 'nothing'},
 		{key: .expires, value: 'yesterday'}
 	)
-	assert h.contains_str('accept')
+	assert h.contains(.accept)
 	assert h.contains(.expires)
 	accept := h.get(.accept) or { '' }
 	expires := h.get(.expires) or { '' }
@@ -15,7 +15,7 @@ fn test_header_new() {
 
 fn test_header_invalid_key() {
 	mut h := http.new_header()
-	h.add_str('space is invalid', ':(') or { return }
+	h.add_custom('space is invalid', ':(') or { return }
 	panic('should have returned')
 }
 
@@ -27,13 +27,22 @@ fn test_header_adds_multiple() {
 	assert h.values(.accept) == ['one' 'two']
 }
 
-fn test_header_set() {
+fn test_header_get() ? {
+	mut h := http.new_header(key: .dnt, value: 'one')
+	h.add_custom('dnt', 'two') ?
+	dnt := h.get_custom('dnt') or { '' }
+	exact := h.get_custom('dnt', exact: true) or { '' }
+	assert dnt == 'one'
+	assert exact == 'two'
+}
+
+fn test_header_set() ? {
 	mut h := http.new_header(
 		{key: .dnt, value: 'one'},
 		{key: .dnt, value: 'two'}
 	)
 	assert h.values(.dnt) == ['one' 'two']
-	h.set_str('dnt', 'three')
+	h.set_custom('DNT', 'three') ?
 	assert h.values(.dnt) == ['three']
 }
 
@@ -43,6 +52,217 @@ fn test_header_delete() {
 		{key: .dnt, value: 'two'}
 	)
 	assert h.values(.dnt) == ['one' 'two']
-	h.delete_str('dnt')
+	h.delete(.dnt)
 	assert h.values(.dnt) == []
 }
+
+fn test_header_delete_not_existing() {
+	mut h := http.new_header()
+	assert h.data.len == 0
+	assert h.keys.len == 0
+	h.delete(.dnt)
+	assert h.data.len == 0
+	assert h.keys.len == 0
+}
+
+fn test_custom_header() ? {
+	mut h := http.new_header()
+	h.add_custom('AbC', 'dEf') ?
+	h.add_custom('aBc', 'GhI') ?
+	assert h.custom_values('AbC', exact: true) == ['dEf']
+	assert h.custom_values('aBc', exact: true) == ['GhI']
+	assert h.custom_values('ABC') == ['dEf', 'GhI']
+	assert h.custom_values('abc') == ['dEf', 'GhI']
+	assert h.keys() == ['AbC', 'aBc']
+	h.delete_custom('AbC')
+	h.delete_custom('aBc')
+
+	h.add_custom('abc', 'def') ?
+	assert h.custom_values('abc') == ['def']
+	assert h.custom_values('ABC') == ['def']
+	assert h.keys() == ['abc']
+	h.delete_custom('abc')
+
+	h.add_custom('accEPT', '*/*') ?
+	assert h.custom_values('ACCept') == ['*/*']
+	assert h.values(.accept) == ['*/*']
+	assert h.keys() == ['accEPT']
+}
+
+fn test_contains_custom() ? {
+	mut h := http.new_header()
+	h.add_custom('Hello', 'world') ?
+	assert h.contains_custom('hello')
+	assert h.contains_custom('HELLO')
+	assert h.contains_custom('Hello', exact: true)
+	assert h.contains_custom('hello', exact: true) == false
+	assert h.contains_custom('HELLO', exact: true) == false
+}
+
+fn test_get_custom() ? {
+	mut h := http.new_header()
+	h.add_custom('Hello', 'world') ?
+	assert h.get_custom('hello') ? == 'world'
+	assert h.get_custom('HELLO') ? == 'world'
+	assert h.get_custom('Hello', exact: true) ? == 'world'
+	if _ := h.get_custom('hello', exact: true) {
+		// should be none
+		assert false
+	}
+	if _ := h.get_custom('HELLO', exact: true) {
+		// should be none
+		assert false
+	}
+}
+
+fn test_custom_values() ? {
+	mut h := http.new_header()
+	h.add_custom('Hello', 'world') ?
+	assert h.custom_values('hello') == ['world']
+	assert h.custom_values('HELLO') == ['world']
+	assert h.custom_values('Hello', exact: true) == ['world']
+	assert h.custom_values('hello', exact: true) == []
+	assert h.custom_values('HELLO', exact: true) == []
+}
+
+fn test_coerce() ? {
+	mut h := http.new_header()
+	h.add_custom('accept', 'foo') ?
+	h.add(.accept, 'bar')
+	assert h.values(.accept) == ['foo', 'bar']
+	assert h.keys().len == 2
+
+	h.coerce()
+	assert h.values(.accept) == ['foo', 'bar']
+	assert h.keys() == ['accept'] // takes the first occurrence
+}
+
+fn test_coerce_canonicalize() ? {
+	mut h := http.new_header()
+	h.add_custom('accept', 'foo') ?
+	h.add(.accept, 'bar')
+	assert h.values(.accept) == ['foo', 'bar']
+	assert h.keys().len == 2
+
+	h.coerce(canonicalize: true)
+	assert h.values(.accept) == ['foo', 'bar']
+	assert h.keys() == ['Accept'] // canonicalize header
+}
+
+fn test_coerce_custom() ? {
+	mut h := http.new_header()
+	h.add_custom('Hello', 'foo') ?
+	h.add_custom('hello', 'bar') ?
+	h.add_custom('HELLO', 'baz') ?
+	assert h.custom_values('hello') == ['foo', 'bar', 'baz']
+	assert h.keys().len == 3
+
+	h.coerce()
+	assert h.custom_values('hello') == ['foo', 'bar', 'baz']
+	assert h.keys() == ['Hello'] // takes the first occurrence
+}
+
+fn test_coerce_canonicalize_custom() ? {
+	mut h := http.new_header()
+	h.add_custom('foo-BAR', 'foo') ?
+	h.add_custom('FOO-bar', 'bar') ?
+	assert h.custom_values('foo-bar') == ['foo', 'bar']
+	assert h.keys().len == 2
+
+	h.coerce(canonicalize: true)
+	assert h.custom_values('foo-bar') == ['foo', 'bar']
+	assert h.keys() == ['Foo-Bar'] // capitalizes the header
+}
+
+fn test_render_version() ? {
+	mut h := http.new_header()
+	h.add_custom('accept', 'foo') ?
+	h.add_custom('Accept', 'bar') ?
+	h.add(.accept, 'baz')
+
+	s1_0 := h.render(version: .v1_0)
+	assert s1_0.contains('accept: foo\n\r')
+	assert s1_0.contains('Accept: bar,baz\n\r')
+
+	s1_1 := h.render(version: .v1_1)
+	assert s1_1.contains('accept: foo\n\r')
+	assert s1_1.contains('Accept: bar,baz\n\r')
+
+	s2_0 := h.render(version: .v2_0)
+	assert s2_0.contains('accept: foo\n\r')
+	assert s2_0.contains('accept: bar,baz\n\r')
+}
+
+fn test_render_coerce() ? {
+	mut h := http.new_header()
+	h.add_custom('accept', 'foo') ?
+	h.add_custom('Accept', 'bar') ?
+	h.add(.accept, 'baz')
+	h.add(.host, 'host')
+
+	s1_0 := h.render(version: .v1_1, coerce: true)
+	assert s1_0.contains('accept: foo,bar,baz\n\r')
+	assert s1_0.contains('Host: host\n\r')
+
+	s1_1 := h.render(version: .v1_1, coerce: true)
+	assert s1_1.contains('accept: foo,bar,baz\n\r')
+	assert s1_1.contains('Host: host\n\r')
+
+	s2_0 := h.render(version: .v2_0, coerce: true)
+	assert s2_0.contains('accept: foo,bar,baz\n\r')
+	assert s2_0.contains('host: host\n\r')
+}
+
+fn test_render_canonicalize() ? {
+	mut h := http.new_header()
+	h.add_custom('accept', 'foo') ?
+	h.add_custom('Accept', 'bar') ?
+	h.add(.accept, 'baz')
+	h.add(.host, 'host')
+
+	s1_0 := h.render(version: .v1_1, canonicalize: true)
+	assert s1_0.contains('Accept: foo\n\r')
+	assert s1_0.contains('Accept: bar,baz\n\r')
+	assert s1_0.contains('Host: host\n\r')
+
+	s1_1 := h.render(version: .v1_1, canonicalize: true)
+	assert s1_1.contains('Accept: foo\n\r')
+	assert s1_1.contains('Accept: bar,baz\n\r')
+	assert s1_1.contains('Host: host\n\r')
+
+	s2_0 := h.render(version: .v2_0, canonicalize: true)
+	assert s2_0.contains('accept: foo\n\r')
+	assert s2_0.contains('accept: bar,baz\n\r')
+	assert s2_0.contains('host: host\n\r')
+}
+
+fn test_render_coerce_canonicalize() ? {
+	mut h := http.new_header()
+	h.add_custom('accept', 'foo') ?
+	h.add_custom('Accept', 'bar') ?
+	h.add(.accept, 'baz')
+	h.add(.host, 'host')
+
+	s1_0 := h.render(version: .v1_1, coerce: true, canonicalize: true)
+	assert s1_0.contains('Accept: foo,bar,baz\n\r')
+	assert s1_0.contains('Host: host\n\r')
+
+	s1_1 := h.render(version: .v1_1, coerce: true, canonicalize: true)
+	assert s1_1.contains('Accept: foo,bar,baz\n\r')
+	assert s1_1.contains('Host: host\n\r')
+
+	s2_0 := h.render(version: .v2_0, coerce: true, canonicalize: true)
+	assert s2_0.contains('accept: foo,bar,baz\n\r')
+	assert s2_0.contains('host: host\n\r')
+}
+
+fn test_str() ? {
+	mut h := http.new_header()
+	h.add(.accept, 'text/html')
+	h.add_custom('Accept', 'image/jpeg') ?
+	h.add_custom('X-custom', 'Hello') ?
+
+	// key order is not guaranteed
+	assert h.str() == 'Accept: text/html,image/jpeg\n\rX-custom: Hello\n\r'
+		|| h.str() == 'X-custom: Hello\n\rAccept:text/html,image/jpeg\n\r'
+}

vlib/vweb/request.v

@@ -15,16 +15,17 @@ fn parse_request(mut reader io.BufferedReader) ?http.Request {
 	line = reader.read_line() ?
 	for line != '' {
 		key, value := parse_header(line) ?
-		h.add_str(key, value) ?
+		h.add_custom(key, value) ?
 		line = reader.read_line() ?
 	}
+	h.coerce(canonicalize: true)
 
 	// create map[string]string from headers
 	// TODO: replace headers and lheaders with http.Header type
 	mut headers := map[string]string{}
 	mut lheaders := map[string]string{}
 	for key in h.keys() {
-		values := h.values_str(key).join('; ')
+		values := h.custom_values(key).join('; ')
 		headers[key] = values
 		lheaders[key.to_lower()] = values
 	}

vlib/x/websocket/handshake.v

@@ -24,7 +24,7 @@ fn (mut ws Client) handshake() ? {
 	sb.write_string(seckey)
 	sb.write_string('\r\nSec-WebSocket-Version: 13')
 	for key in ws.header.keys() {
-		val := ws.header.values_str(key).join(',')
+		val := ws.header.custom_values(key).join(',')
 		sb.write_string('\r\n$key:$val')
 	}
 	sb.write_string('\r\n\r\n')