net.http: change header behavior to keep custom header case (#9602)

pull/9637/head
Miccah 2021-04-07 19:12:46 -05:00 committed by GitHub
parent 790961e73a
commit f809d4052f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 405 additions and 66 deletions

View File

@ -3,6 +3,8 @@
// that can be found in the LICENSE file.
module http
import strings
// CommonHeader is an enum of the most common HTTP headers
pub enum CommonHeader {
accept
@ -323,11 +325,15 @@ const common_header_map = map{
pub struct Header {
mut:
data map[string][]string
// map of lowercase header keys to their original keys
// in order of appearance
keys map[string][]string
}
pub fn (mut h Header) free() {
unsafe {
h.data.free()
h.keys.free()
}
}
@ -349,80 +355,136 @@ pub fn new_header(kvs ...HeaderConfig) Header {
// Append a value to the header key.
pub fn (mut h Header) add(key CommonHeader, value string) {
h.data[key.str()] << value
k := key.str()
h.data[k] << value
h.add_key(k)
}
// Append a value to a custom header key. This function will return an error
// if the key contains invalid header characters.
pub fn (mut h Header) add_str(key string, value string) ? {
k := canonicalize(key) ?
h.data[k] << value
pub fn (mut h Header) add_custom(key string, value string) ? {
is_valid(key) ?
h.data[key] << value
h.add_key(key)
}
// Sets the key-value pair. This function will clear any other values
// that exist for the CommonHeader.
pub fn (mut h Header) set(key CommonHeader, value string) {
h.data[key.str()] = [value]
k := key.str()
h.data[k] = [value]
h.add_key(k)
}
// Sets the key-value pair for a custom header key. This function will
// clear any other values that exist for the CommonHeader.
pub fn (mut h Header) set_str(key string, value string) {
k := canonicalize(key) or { return }
h.data[k] = [value]
// clear any other values that exist for the header. This function will
// return an error if the key contains invalid header characters.
pub fn (mut h Header) set_custom(key string, value string) ? {
is_valid(key) ?
h.data[key] = [value]
h.add_key(key)
}
// Delete all values for a key.
pub fn (mut h Header) delete(key CommonHeader) {
h.data.delete(key.str())
h.delete_custom(key.str())
}
// Delete all values for a custom header key.
pub fn (mut h Header) delete_str(key string) {
k := canonicalize(key) or { return }
h.data.delete(k)
pub fn (mut h Header) delete_custom(key string) {
h.data.delete(key)
// remove key from keys metadata
kl := key.to_lower()
if kl in h.keys {
h.keys[kl] = h.keys[kl].filter(it != key)
}
}
pub struct HeaderCoerceConfig {
canonicalize bool
}
// Coerce data by joining keys that match case-insensitively into one entry
pub fn (mut h Header) coerce(flags ...HeaderCoerceConfig) {
canon := flags.any(it.canonicalize)
for kl, data_keys in h.keys {
master_key := if canon { canonicalize(kl) } else { data_keys[0] }
// save master data
master_data := h.data[master_key]
h.data.delete(master_key)
for key in data_keys {
if key == master_key {
h.data[master_key] << master_data
continue
}
h.data[master_key] << h.data[key]
h.data.delete(key)
}
h.keys[kl] = [master_key]
}
}
// Returns whether the header key exists in the map.
pub fn (h Header) contains(key CommonHeader) bool {
return key.str() in h.data
return h.contains_custom(key.str())
}
pub struct HeaderQueryConfig {
exact bool
}
// Returns whether the custom header key exists in the map.
pub fn (h Header) contains_str(key string) bool {
k := canonicalize(key) or { return false }
return k in h.data
pub fn (h Header) contains_custom(key string, flags ...HeaderQueryConfig) bool {
if flags.any(it.exact) {
return key in h.data
}
return key.to_lower() in h.keys
}
// Gets the first value for the CommonHeader, or none if the key does
// not exist.
pub fn (h Header) get(key CommonHeader) ?string {
k := key.str()
if h.data[k].len == 0 {
return none
}
return h.data[k][0]
return h.get_custom(key.str())
}
// Gets the first value for the custom header, or none if the key does
// not exist.
pub fn (h Header) get_str(key string) ?string {
k := canonicalize(key) or { return none }
if h.data[k].len == 0 {
pub fn (h Header) get_custom(key string, flags ...HeaderQueryConfig) ?string {
mut data_key := key
if !flags.any(it.exact) {
// get the first key from key metadata
k := key.to_lower()
if h.keys[k].len == 0 {
return none
}
data_key = h.keys[k][0]
}
if h.data[data_key].len == 0 {
return none
}
return h.data[k][0]
return h.data[data_key][0]
}
// Gets all values for the CommonHeader.
pub fn (h Header) values(key CommonHeader) []string {
return h.data[key.str()]
return h.custom_values(key.str())
}
// Gets all values for the custom header.
pub fn (h Header) values_str(key string) []string {
k := canonicalize(key) or { return [] }
return h.data[k]
pub fn (h Header) custom_values(key string, flags ...HeaderQueryConfig) []string {
if flags.any(it.exact) {
return h.data[key]
}
// case insensitive lookup
mut values := []string{cap: 10}
for k in h.keys[key.to_lower()] {
values << h.data[k]
}
return values
}
// Gets all header keys as strings
@ -430,37 +492,87 @@ pub fn (h Header) keys() []string {
return h.data.keys()
}
// Validate and canonicalize an HTTP header key
// A canonical header is all lowercase except for the first character
// and any character after a `-`. Example: `Example-Header-Key`
// There are some exceptions like `DNT`, `WWW-Authenticate`, etc. For these we
// check if the lowercase matches any in the common_header_map and return that.
fn canonicalize(s string) ?string {
// check for valid header bytes
for _, c in s {
pub struct HeaderRenderConfig {
version Version
coerce bool
canonicalize bool
}
// Renders the Header into a string for use in sending
// HTTP requests. All header lines will end in `\n\r`
[manualfree]
pub fn (h Header) render(flags HeaderRenderConfig) string {
// estimate ~48 bytes per header
mut sb := strings.new_builder(h.data.len * 48)
if flags.coerce {
for kl, data_keys in h.keys {
key := if flags.version == .v2_0 {
kl
} else if flags.canonicalize {
canonicalize(kl)
} else {
data_keys[0]
}
sb.write_string(key)
sb.write_string(': ')
for i in 0 .. data_keys.len - 1 {
k := data_keys[i]
for v in h.data[k] {
sb.write_string(v)
sb.write_string(',')
}
}
k := data_keys[data_keys.len - 1]
sb.write_string(h.data[k].join(','))
sb.write_string('\n\r')
}
} else {
for k, v in h.data {
key := if flags.version == .v2_0 {
k.to_lower()
} else if flags.canonicalize {
canonicalize(k.to_lower())
} else {
k
}
sb.write_string(key)
sb.write_string(': ')
sb.write_string(v.join(','))
sb.write_string('\n\r')
}
}
res := sb.str()
unsafe { sb.free() }
return res
}
// Canonicalize an HTTP header key
// Common headers are determined by the common_header_map
// Custom headers are capitalized on the first letter and any letter after a '-'
// NOTE: Assumes sl is lowercase, since the caller usually already has the lowercase key
fn canonicalize(sl string) string {
// check if we have a common header
if sl in http.common_header_map {
return http.common_header_map[sl].str()
}
return sl.split('-').map(it.capitalize()).join('-')
}
// Helper function to add a key to the keys map
fn (mut h Header) add_key(key string) {
kl := key.to_lower()
if !h.keys[kl].contains(key) {
h.keys[kl] << key
}
}
// Checks if the header token is valid
fn is_valid(header string) ? {
for _, c in header {
if int(c) >= 128 || !is_token(c) {
return error('Invalid header key')
}
}
// check if we have a common header
sl := s.to_lower()
if sl in http.common_header_map {
return http.common_header_map[sl].str()
}
// check for canonicalization; create a new string if not
mut upper := true
for _, c in s {
if upper && `a` <= c && c <= `z` {
return s.to_lower().split('-').map(it.capitalize()).join('-')
}
if !upper && `A` <= c && c <= `Z` {
return s.to_lower().split('-').map(it.capitalize()).join('-')
}
upper = c == `-`
}
return s
}
// Checks if the byte is valid for a header token
@ -470,3 +582,9 @@ fn is_token(b byte) bool {
else { false }
}
}
// Returns the headers string as seen in HTTP/1.1 requests
// Key order is not guaranteed
pub fn (h Header) str() string {
return h.render(version: .v1_1)
}

View File

@ -1,11 +1,11 @@
import net.http
module http
fn test_header_new() {
h := http.new_header(
{key: .accept, value: 'nothing'},
{key: .expires, value: 'yesterday'}
)
assert h.contains_str('accept')
assert h.contains(.accept)
assert h.contains(.expires)
accept := h.get(.accept) or { '' }
expires := h.get(.expires) or { '' }
@ -15,7 +15,7 @@ fn test_header_new() {
fn test_header_invalid_key() {
mut h := http.new_header()
h.add_str('space is invalid', ':(') or { return }
h.add_custom('space is invalid', ':(') or { return }
panic('should have returned')
}
@ -27,13 +27,22 @@ fn test_header_adds_multiple() {
assert h.values(.accept) == ['one' 'two']
}
fn test_header_set() {
fn test_header_get() ? {
mut h := http.new_header(key: .dnt, value: 'one')
h.add_custom('dnt', 'two') ?
dnt := h.get_custom('dnt') or { '' }
exact := h.get_custom('dnt', exact: true) or { '' }
assert dnt == 'one'
assert exact == 'two'
}
fn test_header_set() ? {
mut h := http.new_header(
{key: .dnt, value: 'one'},
{key: .dnt, value: 'two'}
)
assert h.values(.dnt) == ['one' 'two']
h.set_str('dnt', 'three')
h.set_custom('DNT', 'three') ?
assert h.values(.dnt) == ['three']
}
@ -43,6 +52,217 @@ fn test_header_delete() {
{key: .dnt, value: 'two'}
)
assert h.values(.dnt) == ['one' 'two']
h.delete_str('dnt')
h.delete(.dnt)
assert h.values(.dnt) == []
}
fn test_header_delete_not_existing() {
mut h := http.new_header()
assert h.data.len == 0
assert h.keys.len == 0
h.delete(.dnt)
assert h.data.len == 0
assert h.keys.len == 0
}
fn test_custom_header() ? {
mut h := http.new_header()
h.add_custom('AbC', 'dEf') ?
h.add_custom('aBc', 'GhI') ?
assert h.custom_values('AbC', exact: true) == ['dEf']
assert h.custom_values('aBc', exact: true) == ['GhI']
assert h.custom_values('ABC') == ['dEf', 'GhI']
assert h.custom_values('abc') == ['dEf', 'GhI']
assert h.keys() == ['AbC', 'aBc']
h.delete_custom('AbC')
h.delete_custom('aBc')
h.add_custom('abc', 'def') ?
assert h.custom_values('abc') == ['def']
assert h.custom_values('ABC') == ['def']
assert h.keys() == ['abc']
h.delete_custom('abc')
h.add_custom('accEPT', '*/*') ?
assert h.custom_values('ACCept') == ['*/*']
assert h.values(.accept) == ['*/*']
assert h.keys() == ['accEPT']
}
fn test_contains_custom() ? {
mut h := http.new_header()
h.add_custom('Hello', 'world') ?
assert h.contains_custom('hello')
assert h.contains_custom('HELLO')
assert h.contains_custom('Hello', exact: true)
assert h.contains_custom('hello', exact: true) == false
assert h.contains_custom('HELLO', exact: true) == false
}
fn test_get_custom() ? {
mut h := http.new_header()
h.add_custom('Hello', 'world') ?
assert h.get_custom('hello') ? == 'world'
assert h.get_custom('HELLO') ? == 'world'
assert h.get_custom('Hello', exact: true) ? == 'world'
if _ := h.get_custom('hello', exact: true) {
// should be none
assert false
}
if _ := h.get_custom('HELLO', exact: true) {
// should be none
assert false
}
}
fn test_custom_values() ? {
mut h := http.new_header()
h.add_custom('Hello', 'world') ?
assert h.custom_values('hello') == ['world']
assert h.custom_values('HELLO') == ['world']
assert h.custom_values('Hello', exact: true) == ['world']
assert h.custom_values('hello', exact: true) == []
assert h.custom_values('HELLO', exact: true) == []
}
fn test_coerce() ? {
mut h := http.new_header()
h.add_custom('accept', 'foo') ?
h.add(.accept, 'bar')
assert h.values(.accept) == ['foo', 'bar']
assert h.keys().len == 2
h.coerce()
assert h.values(.accept) == ['foo', 'bar']
assert h.keys() == ['accept'] // takes the first occurrence
}
fn test_coerce_canonicalize() ? {
mut h := http.new_header()
h.add_custom('accept', 'foo') ?
h.add(.accept, 'bar')
assert h.values(.accept) == ['foo', 'bar']
assert h.keys().len == 2
h.coerce(canonicalize: true)
assert h.values(.accept) == ['foo', 'bar']
assert h.keys() == ['Accept'] // canonicalize header
}
fn test_coerce_custom() ? {
mut h := http.new_header()
h.add_custom('Hello', 'foo') ?
h.add_custom('hello', 'bar') ?
h.add_custom('HELLO', 'baz') ?
assert h.custom_values('hello') == ['foo', 'bar', 'baz']
assert h.keys().len == 3
h.coerce()
assert h.custom_values('hello') == ['foo', 'bar', 'baz']
assert h.keys() == ['Hello'] // takes the first occurrence
}
fn test_coerce_canonicalize_custom() ? {
mut h := http.new_header()
h.add_custom('foo-BAR', 'foo') ?
h.add_custom('FOO-bar', 'bar') ?
assert h.custom_values('foo-bar') == ['foo', 'bar']
assert h.keys().len == 2
h.coerce(canonicalize: true)
assert h.custom_values('foo-bar') == ['foo', 'bar']
assert h.keys() == ['Foo-Bar'] // capitalizes the header
}
fn test_render_version() ? {
mut h := http.new_header()
h.add_custom('accept', 'foo') ?
h.add_custom('Accept', 'bar') ?
h.add(.accept, 'baz')
s1_0 := h.render(version: .v1_0)
assert s1_0.contains('accept: foo\n\r')
assert s1_0.contains('Accept: bar,baz\n\r')
s1_1 := h.render(version: .v1_1)
assert s1_1.contains('accept: foo\n\r')
assert s1_1.contains('Accept: bar,baz\n\r')
s2_0 := h.render(version: .v2_0)
assert s2_0.contains('accept: foo\n\r')
assert s2_0.contains('accept: bar,baz\n\r')
}
fn test_render_coerce() ? {
mut h := http.new_header()
h.add_custom('accept', 'foo') ?
h.add_custom('Accept', 'bar') ?
h.add(.accept, 'baz')
h.add(.host, 'host')
s1_0 := h.render(version: .v1_1, coerce: true)
assert s1_0.contains('accept: foo,bar,baz\n\r')
assert s1_0.contains('Host: host\n\r')
s1_1 := h.render(version: .v1_1, coerce: true)
assert s1_1.contains('accept: foo,bar,baz\n\r')
assert s1_1.contains('Host: host\n\r')
s2_0 := h.render(version: .v2_0, coerce: true)
assert s2_0.contains('accept: foo,bar,baz\n\r')
assert s2_0.contains('host: host\n\r')
}
fn test_render_canonicalize() ? {
mut h := http.new_header()
h.add_custom('accept', 'foo') ?
h.add_custom('Accept', 'bar') ?
h.add(.accept, 'baz')
h.add(.host, 'host')
s1_0 := h.render(version: .v1_1, canonicalize: true)
assert s1_0.contains('Accept: foo\n\r')
assert s1_0.contains('Accept: bar,baz\n\r')
assert s1_0.contains('Host: host\n\r')
s1_1 := h.render(version: .v1_1, canonicalize: true)
assert s1_1.contains('Accept: foo\n\r')
assert s1_1.contains('Accept: bar,baz\n\r')
assert s1_1.contains('Host: host\n\r')
s2_0 := h.render(version: .v2_0, canonicalize: true)
assert s2_0.contains('accept: foo\n\r')
assert s2_0.contains('accept: bar,baz\n\r')
assert s2_0.contains('host: host\n\r')
}
fn test_render_coerce_canonicalize() ? {
mut h := http.new_header()
h.add_custom('accept', 'foo') ?
h.add_custom('Accept', 'bar') ?
h.add(.accept, 'baz')
h.add(.host, 'host')
s1_0 := h.render(version: .v1_1, coerce: true, canonicalize: true)
assert s1_0.contains('Accept: foo,bar,baz\n\r')
assert s1_0.contains('Host: host\n\r')
s1_1 := h.render(version: .v1_1, coerce: true, canonicalize: true)
assert s1_1.contains('Accept: foo,bar,baz\n\r')
assert s1_1.contains('Host: host\n\r')
s2_0 := h.render(version: .v2_0, coerce: true, canonicalize: true)
assert s2_0.contains('accept: foo,bar,baz\n\r')
assert s2_0.contains('host: host\n\r')
}
fn test_str() ? {
mut h := http.new_header()
h.add(.accept, 'text/html')
h.add_custom('Accept', 'image/jpeg') ?
h.add_custom('X-custom', 'Hello') ?
// key order is not guaranteed
assert h.str() == 'Accept: text/html,image/jpeg\n\rX-custom: Hello\n\r'
|| h.str() == 'X-custom: Hello\n\rAccept:text/html,image/jpeg\n\r'
}

View File

@ -15,16 +15,17 @@ fn parse_request(mut reader io.BufferedReader) ?http.Request {
line = reader.read_line() ?
for line != '' {
key, value := parse_header(line) ?
h.add_str(key, value) ?
h.add_custom(key, value) ?
line = reader.read_line() ?
}
h.coerce(canonicalize: true)
// create map[string]string from headers
// TODO: replace headers and lheaders with http.Header type
mut headers := map[string]string{}
mut lheaders := map[string]string{}
for key in h.keys() {
values := h.values_str(key).join('; ')
values := h.custom_values(key).join('; ')
headers[key] = values
lheaders[key.to_lower()] = values
}

View File

@ -24,7 +24,7 @@ fn (mut ws Client) handshake() ? {
sb.write_string(seckey)
sb.write_string('\r\nSec-WebSocket-Version: 13')
for key in ws.header.keys() {
val := ws.header.values_str(key).join(',')
val := ws.header.custom_values(key).join(',')
sb.write_string('\r\n$key:$val')
}
sb.write_string('\r\n\r\n')