v/vlib/net/http/http.v

390 lines
11 KiB
V

// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module http
import net.urllib
import net.http.chunked
import net
import io
const (
max_redirects = 4
content_type_default = 'text/plain'
bufsize = 1536
)
// Request holds information about an HTTP request
pub struct Request {
pub mut:
version Version = .v1_1
method Method
header Header
cookies map[string]string
data string
url string
user_agent string = 'v.http'
verbose bool
user_ptr voidptr
ws_func voidptr
}
// FetchConfig holds configurations of fetch
pub struct FetchConfig {
pub mut:
method Method
header Header
data string
params map[string]string
cookies map[string]string
user_agent string = 'v.http'
verbose bool
}
// Response represents the result of the request
pub struct Response {
pub:
text string
header Header
cookies map[string]string
status_code int
}
pub fn new_request(method Method, url_ string, data string) ?Request {
url := if method == .get { url_ + '?' + data } else { url_ }
// println('new req() method=$method url="$url" dta="$data"')
return Request{
method: method
url: url
data: data
/*
headers: {
'Accept-Encoding': 'compress'
}
*/
}
}
// get sends a GET HTTP request to the URL
pub fn get(url string) ?Response {
return fetch_with_method(.get, url, FetchConfig{})
}
// post sends a POST HTTP request to the URL with a string data
pub fn post(url string, data string) ?Response {
return fetch_with_method(.post, url,
data: data
header: new_header({key: .content_type, value: http.content_type_default})
)
}
// post_json sends a POST HTTP request to the URL with a JSON data
pub fn post_json(url string, data string) ?Response {
return fetch_with_method(.post, url,
data: data
header: new_header({key: .content_type, value: 'application/json'})
)
}
// post_form sends a POST HTTP request to the URL with X-WWW-FORM-URLENCODED data
pub fn post_form(url string, data map[string]string) ?Response {
return fetch_with_method(.post, url,
header: new_header({key: .content_type, value: 'application/x-www-form-urlencoded'})
data: url_encode_form_data(data)
)
}
// put sends a PUT HTTP request to the URL with a string data
pub fn put(url string, data string) ?Response {
return fetch_with_method(.put, url,
data: data
header: new_header({key: .content_type, value: http.content_type_default})
)
}
// patch sends a PATCH HTTP request to the URL with a string data
pub fn patch(url string, data string) ?Response {
return fetch_with_method(.patch, url,
data: data
header: new_header({key: .content_type, value: http.content_type_default})
)
}
// head sends a HEAD HTTP request to the URL
pub fn head(url string) ?Response {
return fetch_with_method(.head, url, FetchConfig{})
}
// delete sends a DELETE HTTP request to the URL
pub fn delete(url string) ?Response {
return fetch_with_method(.delete, url, FetchConfig{})
}
// fetch sends an HTTP request to the URL with the given method and configurations
pub fn fetch(_url string, config FetchConfig) ?Response {
if _url == '' {
return error('http.fetch: empty url')
}
url := build_url_from_fetch(_url, config) or { return error('http.fetch: invalid url $_url') }
data := config.data
req := Request{
method: config.method
url: url
data: data
header: config.header
cookies: config.cookies
user_agent: config.user_agent
ws_func: 0
user_ptr: 0
verbose: config.verbose
}
res := req.do() ?
return res
}
// get_text sends a GET HTTP request to the URL and returns the text content of the response
pub fn get_text(url string) string {
resp := fetch(url, method: .get) or { return '' }
return resp.text
}
// url_encode_form_data converts mapped data to an URL encoded string
pub fn url_encode_form_data(data map[string]string) string {
mut pieces := []string{}
for key_, value_ in data {
key := urllib.query_escape(key_)
value := urllib.query_escape(value_)
pieces << '$key=$value'
}
return pieces.join('&')
}
fn fetch_with_method(method Method, url string, _config FetchConfig) ?Response {
mut config := _config
config.method = method
return fetch(url, config)
}
fn build_url_from_fetch(_url string, config FetchConfig) ?string {
mut url := urllib.parse(_url) ?
if config.params.len == 0 {
return url.str()
}
mut pieces := []string{cap: config.params.len}
for key, val in config.params {
pieces << '$key=$val'
}
mut query := pieces.join('&')
if url.raw_query.len > 1 {
query = url.raw_query + '&' + query
}
url.raw_query = query
return url.str()
}
fn (mut req Request) free() {
unsafe { req.header.free() }
}
fn (mut resp Response) free() {
unsafe { resp.header.data.free() }
}
// add_header adds the key and value of an HTTP request header
// To add a custom header, use add_custom_header
pub fn (mut req Request) add_header(key CommonHeader, val string) {
req.header.add(key, val)
}
// add_custom_header adds the key and value of an HTTP request header
// This method may fail if the key contains characters that are not permitted
pub fn (mut req Request) add_custom_header(key string, val string) ? {
return req.header.add_custom(key, val)
}
// do will send the HTTP request and returns `http.Response` as soon as the response is recevied
pub fn (req &Request) do() ?Response {
mut url := urllib.parse(req.url) or { return error('http.Request.do: invalid url $req.url') }
mut rurl := url
mut resp := Response{}
mut no_redirects := 0
for {
if no_redirects == http.max_redirects {
return error('http.request.do: maximum number of redirects reached ($http.max_redirects)')
}
qresp := req.method_and_url_to_response(req.method, rurl) ?
resp = qresp
if resp.status_code !in [301, 302, 303, 307, 308] {
break
}
// follow any redirects
mut redirect_url := resp.header.get(.location) or { '' }
if redirect_url.len > 0 && redirect_url[0] == `/` {
url.set_path(redirect_url) or {
return error('http.request.do: invalid path in redirect: "$redirect_url"')
}
redirect_url = url.str()
}
qrurl := urllib.parse(redirect_url) or {
return error('http.request.do: invalid URL in redirect "$redirect_url"')
}
rurl = qrurl
no_redirects++
}
return resp
}
fn (req &Request) method_and_url_to_response(method Method, url urllib.URL) ?Response {
host_name := url.hostname()
scheme := url.scheme
p := url.path.trim_left('/')
path := if url.query().len > 0 { '/$p?$url.query().encode()' } else { '/$p' }
mut nport := url.port().int()
if nport == 0 {
if scheme == 'http' {
nport = 80
}
if scheme == 'https' {
nport = 443
}
}
// println('fetch $method, $scheme, $host_name, $nport, $path ')
if scheme == 'https' {
// println('ssl_do( $nport, $method, $host_name, $path )')
res := req.ssl_do(nport, method, host_name, path) ?
return res
} else if scheme == 'http' {
// println('http_do( $nport, $method, $host_name, $path )')
res := req.http_do('$host_name:$nport', method, path) ?
return res
}
return error('http.request.method_and_url_to_response: unsupported scheme: "$scheme"')
}
pub fn parse_response(resp string) Response {
mut header := new_header()
// TODO: Cookie data type
mut cookies := map[string]string{}
first_header := resp.all_before('\n')
mut status_code := 0
if first_header.contains('HTTP/') {
val := first_header.find_between(' ', ' ')
status_code = val.int()
}
mut text := ''
// Build resp header map and separate the body
mut nl_pos := 3
mut i := 1
for {
old_pos := nl_pos
nl_pos = resp.index_after('\n', nl_pos + 1)
if nl_pos == -1 {
break
}
h := resp[old_pos + 1..nl_pos]
// End of headers
if h.len <= 1 {
text = resp[nl_pos + 1..]
break
}
i++
pos := h.index(':') or { continue }
mut key := h[..pos]
val := h[pos + 2..].trim_space()
header.add_custom(key, val) or { eprintln('$err; skipping header') }
}
// set cookies
for cookie in header.values(.set_cookie) {
parts := cookie.split_nth('=', 2)
cookies[parts[0]] = parts[1]
}
if header.get(.transfer_encoding) or { '' } == 'chunked' || header.get(.content_length) or { '' } == '' {
text = chunked.decode(text)
}
return Response{
status_code: status_code
header: header
cookies: cookies
text: text
}
}
fn (req &Request) build_request_headers(method Method, host_name string, path string) string {
ua := req.user_agent
mut uheaders := []string{}
if !req.header.contains(.host) {
uheaders << 'Host: $host_name\r\n'
}
if !req.header.contains(.user_agent) {
uheaders << 'User-Agent: $ua\r\n'
}
if req.data.len > 0 && !req.header.contains(.content_length) {
uheaders << 'Content-Length: $req.data.len\r\n'
}
for key in req.header.keys() {
if key == CommonHeader.cookie.str() {
continue
}
val := req.header.custom_values(key).join('; ')
uheaders << '$key: $val\r\n'
}
uheaders << req.build_request_cookies_header()
version := if req.version == .unknown { Version.v1_1 } else { req.version }
return '$method $path $version\r\n' + uheaders.join('') + 'Connection: close\r\n\r\n' + req.data
}
fn (req &Request) build_request_cookies_header() string {
if req.cookies.keys().len < 1 {
return ''
}
mut cookie := []string{}
for key, val in req.cookies {
cookie << '$key=$val'
}
cookie << req.header.values(.cookie)
return 'Cookie: ' + cookie.join('; ') + '\r\n'
}
// unescape_url is deprecated, use urllib.query_unescape() instead
pub fn unescape_url(s string) string {
panic('http.unescape_url() was replaced with urllib.query_unescape()')
}
// escape_url is deprecated, use urllib.query_escape() instead
pub fn escape_url(s string) string {
panic('http.escape_url() was replaced with urllib.query_escape()')
}
// unescape is deprecated, use urllib.query_escape() instead
pub fn unescape(s string) string {
panic('http.unescape() was replaced with http.unescape_url()')
}
// escape is deprecated, use urllib.query_unescape() instead
pub fn escape(s string) string {
panic('http.escape() was replaced with http.escape_url()')
}
fn (req &Request) http_do(host string, method Method, path string) ?Response {
host_name, _ := net.split_address(host) ?
s := req.build_request_headers(method, host_name, path)
mut client := net.dial_tcp(host) ?
// TODO this really needs to be exposed somehow
client.write(s.bytes()) ?
$if trace_http_request ? {
eprintln('> $s')
}
mut bytes := io.read_all(reader: client) ?
client.close() ?
response_text := bytes.bytestr()
$if trace_http_response ? {
eprintln('< $response_text')
}
return parse_response(response_text)
}
// referer returns 'Referer' header value of the given request
pub fn (req &Request) referer() string {
return req.header.get(.referer) or { '' }
}