diff --git a/cmd/tools/vtest-self.v b/cmd/tools/vtest-self.v index 4766bdf4d1..928ef76962 100644 --- a/cmd/tools/vtest-self.v +++ b/cmd/tools/vtest-self.v @@ -40,6 +40,7 @@ const ( 'vlib/v/tests/orm_sub_array_struct_test.v', 'vlib/vweb/tests/vweb_test.v', 'vlib/vweb/request_test.v', + 'vlib/net/http/request_test.v', 'vlib/vweb/route_test.v', 'vlib/x/websocket/websocket_test.v', 'vlib/crypto/rand/crypto_rand_read_test.v', @@ -78,6 +79,7 @@ const ( 'vlib/clipboard/clipboard_test.v', 'vlib/vweb/tests/vweb_test.v', 'vlib/vweb/request_test.v', + 'vlib/net/http/request_test.v', 'vlib/vweb/route_test.v', 'vlib/x/websocket/websocket_test.v', 'vlib/net/http/http_httpbin_test.v', @@ -97,6 +99,7 @@ const ( 'vlib/x/websocket/websocket_test.v', 'vlib/vweb/tests/vweb_test.v', 'vlib/vweb/request_test.v', + 'vlib/net/http/request_test.v', 'vlib/vweb/route_test.v', ] skip_on_non_windows = [ diff --git a/vlib/net/http/header.v b/vlib/net/http/header.v index ca209fd35e..4069d5c434 100644 --- a/vlib/net/http/header.v +++ b/vlib/net/http/header.v @@ -499,7 +499,7 @@ pub struct HeaderRenderConfig { } // Renders the Header into a string for use in sending -// HTTP requests. All header lines will end in `\n\r` +// HTTP requests. All header lines will end in `\r\n` [manualfree] pub fn (h Header) render(flags HeaderRenderConfig) string { // estimate ~48 bytes per header @@ -524,7 +524,7 @@ pub fn (h Header) render(flags HeaderRenderConfig) string { } k := data_keys[data_keys.len - 1] sb.write_string(h.data[k].join(',')) - sb.write_string('\n\r') + sb.write_string('\r\n') } } else { for k, v in h.data { @@ -538,7 +538,7 @@ pub fn (h Header) render(flags HeaderRenderConfig) string { sb.write_string(key) sb.write_string(': ') sb.write_string(v.join(',')) - sb.write_string('\n\r') + sb.write_string('\r\n') } } res := sb.str() diff --git a/vlib/net/http/header_test.v b/vlib/net/http/header_test.v index 2aca042769..3426113430 100644 --- a/vlib/net/http/header_test.v +++ b/vlib/net/http/header_test.v @@ -181,16 +181,16 @@ fn test_render_version() ? { h.add(.accept, 'baz') s1_0 := h.render(version: .v1_0) - assert s1_0.contains('accept: foo\n\r') - assert s1_0.contains('Accept: bar,baz\n\r') + assert s1_0.contains('accept: foo\r\n') + assert s1_0.contains('Accept: bar,baz\r\n') s1_1 := h.render(version: .v1_1) - assert s1_1.contains('accept: foo\n\r') - assert s1_1.contains('Accept: bar,baz\n\r') + assert s1_1.contains('accept: foo\r\n') + assert s1_1.contains('Accept: bar,baz\r\n') s2_0 := h.render(version: .v2_0) - assert s2_0.contains('accept: foo\n\r') - assert s2_0.contains('accept: bar,baz\n\r') + assert s2_0.contains('accept: foo\r\n') + assert s2_0.contains('accept: bar,baz\r\n') } fn test_render_coerce() ? { @@ -201,16 +201,16 @@ fn test_render_coerce() ? { h.add(.host, 'host') s1_0 := h.render(version: .v1_1, coerce: true) - assert s1_0.contains('accept: foo,bar,baz\n\r') - assert s1_0.contains('Host: host\n\r') + assert s1_0.contains('accept: foo,bar,baz\r\n') + assert s1_0.contains('Host: host\r\n') s1_1 := h.render(version: .v1_1, coerce: true) - assert s1_1.contains('accept: foo,bar,baz\n\r') - assert s1_1.contains('Host: host\n\r') + assert s1_1.contains('accept: foo,bar,baz\r\n') + assert s1_1.contains('Host: host\r\n') s2_0 := h.render(version: .v2_0, coerce: true) - assert s2_0.contains('accept: foo,bar,baz\n\r') - assert s2_0.contains('host: host\n\r') + assert s2_0.contains('accept: foo,bar,baz\r\n') + assert s2_0.contains('host: host\r\n') } fn test_render_canonicalize() ? { @@ -221,19 +221,19 @@ fn test_render_canonicalize() ? { h.add(.host, 'host') s1_0 := h.render(version: .v1_1, canonicalize: true) - assert s1_0.contains('Accept: foo\n\r') - assert s1_0.contains('Accept: bar,baz\n\r') - assert s1_0.contains('Host: host\n\r') + assert s1_0.contains('Accept: foo\r\n') + assert s1_0.contains('Accept: bar,baz\r\n') + assert s1_0.contains('Host: host\r\n') s1_1 := h.render(version: .v1_1, canonicalize: true) - assert s1_1.contains('Accept: foo\n\r') - assert s1_1.contains('Accept: bar,baz\n\r') - assert s1_1.contains('Host: host\n\r') + assert s1_1.contains('Accept: foo\r\n') + assert s1_1.contains('Accept: bar,baz\r\n') + assert s1_1.contains('Host: host\r\n') s2_0 := h.render(version: .v2_0, canonicalize: true) - assert s2_0.contains('accept: foo\n\r') - assert s2_0.contains('accept: bar,baz\n\r') - assert s2_0.contains('host: host\n\r') + assert s2_0.contains('accept: foo\r\n') + assert s2_0.contains('accept: bar,baz\r\n') + assert s2_0.contains('host: host\r\n') } fn test_render_coerce_canonicalize() ? { @@ -244,16 +244,16 @@ fn test_render_coerce_canonicalize() ? { h.add(.host, 'host') s1_0 := h.render(version: .v1_1, coerce: true, canonicalize: true) - assert s1_0.contains('Accept: foo,bar,baz\n\r') - assert s1_0.contains('Host: host\n\r') + assert s1_0.contains('Accept: foo,bar,baz\r\n') + assert s1_0.contains('Host: host\r\n') s1_1 := h.render(version: .v1_1, coerce: true, canonicalize: true) - assert s1_1.contains('Accept: foo,bar,baz\n\r') - assert s1_1.contains('Host: host\n\r') + assert s1_1.contains('Accept: foo,bar,baz\r\n') + assert s1_1.contains('Host: host\r\n') s2_0 := h.render(version: .v2_0, coerce: true, canonicalize: true) - assert s2_0.contains('accept: foo,bar,baz\n\r') - assert s2_0.contains('host: host\n\r') + assert s2_0.contains('accept: foo,bar,baz\r\n') + assert s2_0.contains('host: host\r\n') } fn test_str() ? { @@ -263,6 +263,6 @@ fn test_str() ? { h.add_custom('X-custom', 'Hello') ? // key order is not guaranteed - assert h.str() == 'Accept: text/html,image/jpeg\n\rX-custom: Hello\n\r' - || h.str() == 'X-custom: Hello\n\rAccept:text/html,image/jpeg\n\r' + assert h.str() == 'Accept: text/html,image/jpeg\r\nX-custom: Hello\r\n' + || h.str() == 'X-custom: Hello\r\nAccept:text/html,image/jpeg\r\n' } diff --git a/vlib/net/http/http.v b/vlib/net/http/http.v index 50d92eef20..0d940cf8a1 100644 --- a/vlib/net/http/http.v +++ b/vlib/net/http/http.v @@ -4,9 +4,6 @@ module http import net.urllib -import net.http.chunked -import net -import io const ( max_redirects = 4 @@ -14,21 +11,6 @@ const ( bufsize = 1536 ) -// Request holds information about an HTTP request -pub struct Request { -pub mut: - version Version = .v1_1 - method Method - header Header - cookies map[string]string - data string - url string - user_agent string = 'v.http' - verbose bool - user_ptr voidptr - ws_func voidptr -} - // FetchConfig holds configurations of fetch pub struct FetchConfig { pub mut: @@ -41,15 +23,6 @@ pub mut: verbose bool } -// Response represents the result of the request -pub struct Response { -pub: - text string - header Header - cookies map[string]string - status_code int -} - pub fn new_request(method Method, url_ string, data string) ?Request { url := if method == .get { url_ + '?' + data } else { url_ } // println('new req() method=$method url="$url" dta="$data"') @@ -88,7 +61,7 @@ pub fn post_json(url string, data string) ?Response { // post_form sends a POST HTTP request to the URL with X-WWW-FORM-URLENCODED data pub fn post_form(url string, data map[string]string) ?Response { - return fetch_with_method(.post, url, + return fetch_with_method(.post, url, header: new_header({key: .content_type, value: 'application/x-www-form-urlencoded'}) data: url_encode_form_data(data) ) @@ -182,169 +155,6 @@ fn build_url_from_fetch(_url string, config FetchConfig) ?string { return url.str() } -fn (mut req Request) free() { - unsafe { req.header.free() } -} - -fn (mut resp Response) free() { - unsafe { resp.header.data.free() } -} - -// add_header adds the key and value of an HTTP request header -// To add a custom header, use add_custom_header -pub fn (mut req Request) add_header(key CommonHeader, val string) { - req.header.add(key, val) -} - -// add_custom_header adds the key and value of an HTTP request header -// This method may fail if the key contains characters that are not permitted -pub fn (mut req Request) add_custom_header(key string, val string) ? { - return req.header.add_custom(key, val) -} - -// do will send the HTTP request and returns `http.Response` as soon as the response is recevied -pub fn (req &Request) do() ?Response { - mut url := urllib.parse(req.url) or { return error('http.Request.do: invalid url $req.url') } - mut rurl := url - mut resp := Response{} - mut no_redirects := 0 - for { - if no_redirects == http.max_redirects { - return error('http.request.do: maximum number of redirects reached ($http.max_redirects)') - } - qresp := req.method_and_url_to_response(req.method, rurl) ? - resp = qresp - if resp.status_code !in [301, 302, 303, 307, 308] { - break - } - // follow any redirects - mut redirect_url := resp.header.get(.location) or { '' } - if redirect_url.len > 0 && redirect_url[0] == `/` { - url.set_path(redirect_url) or { - return error('http.request.do: invalid path in redirect: "$redirect_url"') - } - redirect_url = url.str() - } - qrurl := urllib.parse(redirect_url) or { - return error('http.request.do: invalid URL in redirect "$redirect_url"') - } - rurl = qrurl - no_redirects++ - } - return resp -} - -fn (req &Request) method_and_url_to_response(method Method, url urllib.URL) ?Response { - host_name := url.hostname() - scheme := url.scheme - p := url.path.trim_left('/') - path := if url.query().len > 0 { '/$p?$url.query().encode()' } else { '/$p' } - mut nport := url.port().int() - if nport == 0 { - if scheme == 'http' { - nport = 80 - } - if scheme == 'https' { - nport = 443 - } - } - // println('fetch $method, $scheme, $host_name, $nport, $path ') - if scheme == 'https' { - // println('ssl_do( $nport, $method, $host_name, $path )') - res := req.ssl_do(nport, method, host_name, path) ? - return res - } else if scheme == 'http' { - // println('http_do( $nport, $method, $host_name, $path )') - res := req.http_do('$host_name:$nport', method, path) ? - return res - } - return error('http.request.method_and_url_to_response: unsupported scheme: "$scheme"') -} - -pub fn parse_response(resp string) Response { - mut header := new_header() - // TODO: Cookie data type - mut cookies := map[string]string{} - first_header := resp.all_before('\n') - mut status_code := 0 - if first_header.contains('HTTP/') { - val := first_header.find_between(' ', ' ') - status_code = val.int() - } - mut text := '' - // Build resp header map and separate the body - mut nl_pos := 3 - mut i := 1 - for { - old_pos := nl_pos - nl_pos = resp.index_after('\n', nl_pos + 1) - if nl_pos == -1 { - break - } - h := resp[old_pos + 1..nl_pos] - // End of headers - if h.len <= 1 { - text = resp[nl_pos + 1..] - break - } - i++ - pos := h.index(':') or { continue } - mut key := h[..pos] - val := h[pos + 2..].trim_space() - header.add_custom(key, val) or { eprintln('$err; skipping header') } - } - // set cookies - for cookie in header.values(.set_cookie) { - parts := cookie.split_nth('=', 2) - cookies[parts[0]] = parts[1] - } - if header.get(.transfer_encoding) or { '' } == 'chunked' || header.get(.content_length) or { '' } == '' { - text = chunked.decode(text) - } - return Response{ - status_code: status_code - header: header - cookies: cookies - text: text - } -} - -fn (req &Request) build_request_headers(method Method, host_name string, path string) string { - ua := req.user_agent - mut uheaders := []string{} - if !req.header.contains(.host) { - uheaders << 'Host: $host_name\r\n' - } - if !req.header.contains(.user_agent) { - uheaders << 'User-Agent: $ua\r\n' - } - if req.data.len > 0 && !req.header.contains(.content_length) { - uheaders << 'Content-Length: $req.data.len\r\n' - } - for key in req.header.keys() { - if key == CommonHeader.cookie.str() { - continue - } - val := req.header.custom_values(key).join('; ') - uheaders << '$key: $val\r\n' - } - uheaders << req.build_request_cookies_header() - version := if req.version == .unknown { Version.v1_1 } else { req.version } - return '$method $path $version\r\n' + uheaders.join('') + 'Connection: close\r\n\r\n' + req.data -} - -fn (req &Request) build_request_cookies_header() string { - if req.cookies.keys().len < 1 { - return '' - } - mut cookie := []string{} - for key, val in req.cookies { - cookie << '$key=$val' - } - cookie << req.header.values(.cookie) - return 'Cookie: ' + cookie.join('; ') + '\r\n' -} - // unescape_url is deprecated, use urllib.query_unescape() instead pub fn unescape_url(s string) string { panic('http.unescape_url() was replaced with urllib.query_unescape()') @@ -364,26 +174,3 @@ pub fn unescape(s string) string { pub fn escape(s string) string { panic('http.escape() was replaced with http.escape_url()') } - -fn (req &Request) http_do(host string, method Method, path string) ?Response { - host_name, _ := net.split_address(host) ? - s := req.build_request_headers(method, host_name, path) - mut client := net.dial_tcp(host) ? - // TODO this really needs to be exposed somehow - client.write(s.bytes()) ? - $if trace_http_request ? { - eprintln('> $s') - } - mut bytes := io.read_all(reader: client) ? - client.close() ? - response_text := bytes.bytestr() - $if trace_http_response ? { - eprintln('< $response_text') - } - return parse_response(response_text) -} - -// referer returns 'Referer' header value of the given request -pub fn (req &Request) referer() string { - return req.header.get(.referer) or { '' } -} diff --git a/vlib/net/http/request.v b/vlib/net/http/request.v new file mode 100644 index 0000000000..c8a1ede53e --- /dev/null +++ b/vlib/net/http/request.v @@ -0,0 +1,326 @@ +// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module http + +import io +import net +import net.urllib +import strings + +// Request holds information about an HTTP request (either received by +// a server or to be sent by a client) +pub struct Request { +pub mut: + version Version = .v1_1 + method Method + header Header + cookies map[string]string + data string + url string + user_agent string = 'v.http' + verbose bool + user_ptr voidptr + ws_func voidptr +} + +fn (mut req Request) free() { + unsafe { req.header.free() } +} + +// add_header adds the key and value of an HTTP request header +// To add a custom header, use add_custom_header +pub fn (mut req Request) add_header(key CommonHeader, val string) { + req.header.add(key, val) +} + +// add_custom_header adds the key and value of an HTTP request header +// This method may fail if the key contains characters that are not permitted +pub fn (mut req Request) add_custom_header(key string, val string) ? { + return req.header.add_custom(key, val) +} + +// do will send the HTTP request and returns `http.Response` as soon as the response is recevied +pub fn (req &Request) do() ?Response { + mut url := urllib.parse(req.url) or { return error('http.Request.do: invalid url $req.url') } + mut rurl := url + mut resp := Response{} + mut no_redirects := 0 + for { + if no_redirects == http.max_redirects { + return error('http.request.do: maximum number of redirects reached ($http.max_redirects)') + } + qresp := req.method_and_url_to_response(req.method, rurl) ? + resp = qresp + if resp.status_code !in [301, 302, 303, 307, 308] { + break + } + // follow any redirects + mut redirect_url := resp.header.get(.location) or { '' } + if redirect_url.len > 0 && redirect_url[0] == `/` { + url.set_path(redirect_url) or { + return error('http.request.do: invalid path in redirect: "$redirect_url"') + } + redirect_url = url.str() + } + qrurl := urllib.parse(redirect_url) or { + return error('http.request.do: invalid URL in redirect "$redirect_url"') + } + rurl = qrurl + no_redirects++ + } + return resp +} + +fn (req &Request) method_and_url_to_response(method Method, url urllib.URL) ?Response { + host_name := url.hostname() + scheme := url.scheme + p := url.path.trim_left('/') + path := if url.query().len > 0 { '/$p?$url.query().encode()' } else { '/$p' } + mut nport := url.port().int() + if nport == 0 { + if scheme == 'http' { + nport = 80 + } + if scheme == 'https' { + nport = 443 + } + } + // println('fetch $method, $scheme, $host_name, $nport, $path ') + if scheme == 'https' { + // println('ssl_do( $nport, $method, $host_name, $path )') + res := req.ssl_do(nport, method, host_name, path) ? + return res + } else if scheme == 'http' { + // println('http_do( $nport, $method, $host_name, $path )') + res := req.http_do('$host_name:$nport', method, path) ? + return res + } + return error('http.request.method_and_url_to_response: unsupported scheme: "$scheme"') +} + +fn (req &Request) build_request_headers(method Method, host_name string, path string) string { + ua := req.user_agent + mut uheaders := []string{} + if !req.header.contains(.host) { + uheaders << 'Host: $host_name\r\n' + } + if !req.header.contains(.user_agent) { + uheaders << 'User-Agent: $ua\r\n' + } + if req.data.len > 0 && !req.header.contains(.content_length) { + uheaders << 'Content-Length: $req.data.len\r\n' + } + for key in req.header.keys() { + if key == CommonHeader.cookie.str() { + continue + } + val := req.header.custom_values(key).join('; ') + uheaders << '$key: $val\r\n' + } + uheaders << req.build_request_cookies_header() + version := if req.version == .unknown { Version.v1_1 } else { req.version } + return '$method $path $version\r\n' + uheaders.join('') + 'Connection: close\r\n\r\n' + req.data +} + +fn (req &Request) build_request_cookies_header() string { + if req.cookies.keys().len < 1 { + return '' + } + mut cookie := []string{} + for key, val in req.cookies { + cookie << '$key=$val' + } + cookie << req.header.values(.cookie) + return 'Cookie: ' + cookie.join('; ') + '\r\n' +} + +fn (req &Request) http_do(host string, method Method, path string) ?Response { + host_name, _ := net.split_address(host) ? + s := req.build_request_headers(method, host_name, path) + mut client := net.dial_tcp(host) ? + // TODO this really needs to be exposed somehow + client.write(s.bytes()) ? + $if trace_http_request ? { + eprintln('> $s') + } + mut bytes := io.read_all(reader: client) ? + client.close() ? + response_text := bytes.bytestr() + $if trace_http_response ? { + eprintln('< $response_text') + } + return parse_response(response_text) +} + +// referer returns 'Referer' header value of the given request +pub fn (req &Request) referer() string { + return req.header.get(.referer) or { '' } +} + +// Parse a raw HTTP request into a Request object +pub fn parse_request(mut reader io.BufferedReader) ?Request { + // request line + mut line := reader.read_line() ? + method, target, version := parse_request_line(line) ? + + // headers + mut header := new_header() + line = reader.read_line() ? + for line != '' { + key, value := parse_header(line) ? + header.add_custom(key, value) ? + line = reader.read_line() ? + } + header.coerce(canonicalize: true) + + // body + mut body := []byte{} + if length := header.get(.content_length) { + n := length.int() + if n > 0 { + body = []byte{len: n} + mut count := 0 + for count < body.len { + count += reader.read(mut body[count..]) or { break } + } + } + } + + return Request{ + method: method + url: target.str() + header: header + data: body.bytestr() + version: version + } +} + +fn parse_request_line(s string) ?(Method, urllib.URL, Version) { + words := s.split(' ') + if words.len != 3 { + return error('malformed request line') + } + method := method_from_str(words[0]) + target := urllib.parse(words[1]) ? + version := version_from_str(words[2]) + if version == .unknown { + return error('unsupported version') + } + + return method, target, version +} + +fn parse_header(s string) ?(string, string) { + if !s.contains(':') { + return error('missing colon in header') + } + words := s.split_nth(':', 2) + // TODO: parse quoted text according to the RFC + return words[0], words[1].trim_left(' \t') +} + +// Parse URL encoded key=value&key=value forms +fn parse_form(body string) map[string]string { + words := body.split('&') + mut form := map[string]string{} + for word in words { + kv := word.split_nth('=', 2) + if kv.len != 2 { + continue + } + key := urllib.query_unescape(kv[0]) or { continue } + val := urllib.query_unescape(kv[1]) or { continue } + form[key] = val + } + return form + // } + // todo: parse form-data and application/json + // ... +} + +struct FileData { +pub: + filename string + content_type string + data string +} + +struct UnexpectedExtraAttributeError { + msg string + code int +} + +struct MultiplePathAttributesError { + msg string = 'Expected at most one path attribute' + code int +} + + +fn parse_multipart_form(body string, boundary string) (map[string]string, map[string][]FileData) { + sections := body.split(boundary) + fields := sections[1..sections.len - 1] + mut form := map[string]string{} + mut files := map[string][]FileData{} + + for field in fields { + // TODO: do not split into lines; do same parsing for HTTP body + lines := field.split_into_lines()[1..] + disposition := parse_disposition(lines[0]) + // Grab everything between the double quotes + name := disposition['name'] or { continue } + // Parse files + // TODO: filename* + if 'filename' in disposition { + filename := disposition['filename'] + // Parse Content-Type header + if lines.len == 1 || !lines[1].to_lower().starts_with('content-type:') { + continue + } + mut ct := lines[1].split_nth(':', 2)[1] + ct = ct.trim_left(' \t') + data := lines_to_string(field.len, lines, 3, lines.len - 1) + files[name] << FileData{ + filename: filename + content_type: ct + data: data + } + continue + } + data := lines_to_string(field.len, lines, 2, lines.len - 1) + form[name] = data + } + return form, files +} + +// Parse the Content-Disposition header of a multipart form +// Returns a map of the key="value" pairs +// Example: parse_disposition('Content-Disposition: form-data; name="a"; filename="b"') == {'name': 'a', 'filename': 'b'} +fn parse_disposition(line string) map[string]string { + mut data := map[string]string{} + for word in line.split(';') { + kv := word.split_nth('=', 2) + if kv.len != 2 { + continue + } + key, value := kv[0].to_lower().trim_left(' \t'), kv[1] + if value.starts_with('"') && value.ends_with('"') { + data[key] = value[1..value.len - 1] + } else { + data[key] = value + } + } + return data +} + +[manualfree] +fn lines_to_string(len int, lines []string, start int, end int) string { + mut sb := strings.new_builder(len) + for i in start .. end { + sb.writeln(lines[i]) + } + sb.cut_last(1) // last newline + res := sb.str() + unsafe { sb.free() } + return res +} diff --git a/vlib/net/http/request_test.v b/vlib/net/http/request_test.v new file mode 100644 index 0000000000..3f1ff205a3 --- /dev/null +++ b/vlib/net/http/request_test.v @@ -0,0 +1,138 @@ +module http + +import io + +struct StringReader { + text string +mut: + place int +} + +fn (mut s StringReader) read(mut buf []byte) ?int { + if s.place >= s.text.len { + return none + } + max_bytes := 100 + end := if s.place + max_bytes >= s.text.len { s.text.len } else { s.place + max_bytes } + n := copy(buf, s.text[s.place..end].bytes()) + s.place += n + return n +} + +fn reader(s string) &io.BufferedReader { + return io.new_buffered_reader( + reader: &StringReader{ + text: s + } + ) +} + +fn test_parse_request_not_http() { + parse_request(mut reader('hello')) or { return } + panic('should not have parsed') +} + +fn test_parse_request_no_headers() { + req := parse_request(mut reader('GET / HTTP/1.1\r\n\r\n')) or { panic('did not parse: $err') } + assert req.method == .get + assert req.url == '/' + assert req.version == .v1_1 +} + +fn test_parse_request_two_headers() { + req := parse_request(mut reader('GET / HTTP/1.1\r\nTest1: a\r\nTest2: B\r\n\r\n')) or { + panic('did not parse: $err') + } + assert req.header.custom_values('Test1') == ['a'] + assert req.header.custom_values('Test2') == ['B'] +} + +fn test_parse_request_two_header_values() { + req := parse_request(mut reader('GET / HTTP/1.1\r\nTest1: a; b\r\nTest2: c\r\nTest2: d\r\n\r\n')) or { + panic('did not parse: $err') + } + assert req.header.custom_values('Test1') == ['a; b'] + assert req.header.custom_values('Test2') == ['c', 'd'] +} + +fn test_parse_request_body() { + req := parse_request(mut reader('GET / HTTP/1.1\r\nTest1: a\r\nTest2: b\r\nContent-Length: 4\r\n\r\nbodyabc')) or { + panic('did not parse: $err') + } + assert req.data == 'body' +} + +fn test_parse_request_line() { + method, target, version := parse_request_line('GET /target HTTP/1.1') or { + panic('did not parse: $err') + } + assert method == .get + assert target.str() == '/target' + assert version == .v1_1 +} + +fn test_parse_form() { + assert parse_form('foo=bar&bar=baz') == map{ + 'foo': 'bar' + 'bar': 'baz' + } + assert parse_form('foo=bar=&bar=baz') == map{ + 'foo': 'bar=' + 'bar': 'baz' + } + assert parse_form('foo=bar%3D&bar=baz') == map{ + 'foo': 'bar=' + 'bar': 'baz' + } + assert parse_form('foo=b%26ar&bar=baz') == map{ + 'foo': 'b&ar' + 'bar': 'baz' + } + assert parse_form('a=b& c=d') == map{ + 'a': 'b' + ' c': 'd' + } + assert parse_form('a=b&c= d ') == map{ + 'a': 'b' + 'c': ' d ' + } +} + +fn test_parse_multipart_form() { + boundary := '6844a625b1f0b299' + names := ['foo', 'fooz'] + file := 'bar.v' + ct := 'application/octet-stream' + contents := ['baz', 'buzz'] + data := "--------------------------$boundary +Content-Disposition: form-data; name=\"${names[0]}\"; filename=\"$file\" +Content-Type: $ct + +${contents[0]} +--------------------------$boundary +Content-Disposition: form-data; name=\"${names[1]}\" + +${contents[1]} +--------------------------$boundary-- +" + form, files := parse_multipart_form(data, boundary) + assert files == map{ + names[0]: [FileData{ + filename: file + content_type: ct + data: contents[0] + }] + } + + assert form == map{ + names[1]: contents[1] + } +} + +fn test_parse_large_body() ? { + body := 'A'.repeat(101) // greater than max_bytes + req := 'GET / HTTP/1.1\r\nContent-Length: $body.len\r\n\r\n$body' + result := parse_request(mut reader(req)) ? + assert result.data.len == body.len + assert result.data == body +} diff --git a/vlib/net/http/response.v b/vlib/net/http/response.v new file mode 100644 index 0000000000..06237668fa --- /dev/null +++ b/vlib/net/http/response.v @@ -0,0 +1,78 @@ +// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module http + +import chunked + +// Response represents the result of the request +pub struct Response { +pub mut: + text string + header Header + cookies map[string]string + status_code int + version Version +} + +fn (mut resp Response) free() { + unsafe { resp.header.data.free() } +} + +// Formats resp to bytes suitable for HTTP response transmission +pub fn (resp Response) bytes() []byte { + // TODO: build []byte directly; this uses two allocations + // TODO: cookies + return ('$resp.version $resp.status_code ${status_from_int(resp.status_code).str()}\r\n' + + '${resp.header.render(version: resp.version)}\r\n' + + '$resp.text').bytes() +} + +// Parse a raw HTTP response into a Response object +pub fn parse_response(resp string) Response { + mut header := new_header() + // TODO: Cookie data type + mut cookies := map[string]string{} + first_header := resp.all_before('\n') + mut status_code := 0 + if first_header.contains('HTTP/') { + val := first_header.find_between(' ', ' ') + status_code = val.int() + } + mut text := '' + // Build resp header map and separate the body + mut nl_pos := 3 + mut i := 1 + for { + old_pos := nl_pos + nl_pos = resp.index_after('\n', nl_pos + 1) + if nl_pos == -1 { + break + } + h := resp[old_pos + 1..nl_pos] + // End of headers + if h.len <= 1 { + text = resp[nl_pos + 1..] + break + } + i++ + pos := h.index(':') or { continue } + mut key := h[..pos] + val := h[pos + 2..].trim_space() + header.add_custom(key, val) or { eprintln('$err; skipping header') } + } + // set cookies + for cookie in header.values(.set_cookie) { + parts := cookie.split_nth('=', 2) + cookies[parts[0]] = parts[1] + } + if header.get(.transfer_encoding) or { '' } == 'chunked' || header.get(.content_length) or { '' } == '' { + text = chunked.decode(text) + } + return Response{ + status_code: status_code + header: header + cookies: cookies + text: text + } +} diff --git a/vlib/net/http/server.v b/vlib/net/http/server.v new file mode 100644 index 0000000000..147fa8a0d3 --- /dev/null +++ b/vlib/net/http/server.v @@ -0,0 +1,73 @@ +// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module http + +import io +import net +import time + +pub struct Server { +pub mut: + port int = 8080 + handler fn(Request) Response + read_timeout time.Duration = 30 * time.second + write_timeout time.Duration = 30 * time.second +} + +pub fn (mut s Server) listen_and_serve() ? { + if voidptr(s.handler) == 0 { + eprintln('Server handler not set, using debug handler') + s.handler = fn(req Request) Response { + $if debug { + eprintln('[$time.now()] $req.method $req.url\n\r$req.header\n\r$req.data - 200 OK') + } $else { + eprintln('[$time.now()] $req.method $req.url - 200') + } + return Response{ + version: req.version + text: req.data + header: req.header + cookies: req.cookies + status_code: int(Status.ok) + } + } + } + mut l := net.listen_tcp(s.port) ? + eprintln('Listening on :$s.port') + for { + mut conn := l.accept() or { + eprintln('accept() failed: $err; skipping') + continue + } + conn.set_read_timeout(s.read_timeout) + conn.set_write_timeout(s.write_timeout) + // TODO: make concurrent + s.parse_and_respond(mut conn) + } +} + +fn (mut s Server) parse_and_respond(mut conn net.TcpConn) { + defer { + conn.close() or { eprintln('close() failed: $err') } + } + + mut reader := io.new_buffered_reader(reader: conn) + defer { + reader.free() + } + req := parse_request(mut reader) or { + $if debug { + // only show in debug mode to prevent abuse + eprintln('error parsing request: $err') + } + return + } + mut resp := s.handler(req) + if resp.version == .unknown { + resp.version = req.version + } + conn.write(resp.bytes()) or { + eprintln('error sending response: $err') + } +}