vweb: router refactor (#12041)

pull/12061/head
Anton Zavodchikov 2021-10-03 18:26:44 +05:00 committed by GitHub
parent 9be16eba63
commit 895daf297f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 179 additions and 421 deletions

74
vlib/vweb/parse.v 100644
View File

@ -0,0 +1,74 @@
module vweb
import net.urllib
import net.http
// Parsing function attributes for methods and path.
fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
if attrs.len == 0 {
return [http.Method.get], '/$name'
}
mut x := attrs.clone()
mut methods := []http.Method{}
mut path := ''
for i := 0; i < x.len; {
attr := x[i]
attru := attr.to_upper()
m := http.method_from_str(attru)
if attru == 'GET' || m != .get {
methods << m
x.delete(i)
continue
}
if attr.starts_with('/') {
if path != '' {
return IError(http.MultiplePathAttributesError{})
}
path = attr
x.delete(i)
continue
}
i++
}
if x.len > 0 {
return IError(http.UnexpectedExtraAttributeError{
msg: 'Encountered unexpected extra attributes: $x'
})
}
if methods.len == 0 {
methods = [http.Method.get]
}
if path == '' {
path = '/$name'
}
// Make path lowercase for case-insensitive comparisons
return methods, path.to_lower()
}
fn parse_query_from_url(url urllib.URL) map[string]string {
mut query := map[string]string{}
for k, v in url.query().data {
query[k] = v.data[0]
}
return query
}
fn parse_form_from_request(request http.Request) ?(map[string]string, map[string][]http.FileData) {
mut form := map[string]string{}
mut files := map[string][]http.FileData{}
if request.method in methods_with_form {
ct := request.header.get(.content_type) or { '' }.split(';').map(it.trim_left(' \t'))
if 'multipart/form-data' in ct {
boundary := ct.filter(it.starts_with('boundary='))
if boundary.len != 1 {
return error('detected more that one form-data boundary')
}
form, files = http.parse_multipart_form(request.data, boundary[0][9..])
} else {
form = http.parse_form(request.data)
}
}
return form, files
}

View File

@ -1,121 +0,0 @@
module vweb
import io
import strings
import net.http
import net.urllib
fn parse_request(mut reader io.BufferedReader) ?http.Request {
// request line
mut line := reader.read_line() ?
method, target, version := parse_request_line(line) ?
// headers
mut header := http.new_header()
line = reader.read_line() ?
for line != '' {
key, value := parse_header(line) ?
header.add_custom(key, value) ?
line = reader.read_line() ?
}
header.coerce(canonicalize: true)
// body
mut body := []byte{}
if length := header.get(.content_length) {
n := length.int()
if n > 0 {
body = []byte{len: n}
mut count := 0
for count < body.len {
count += reader.read(mut body[count..]) or { break }
}
}
}
return http.Request{
method: method
url: target.str()
header: header
data: body.bytestr()
version: version
}
}
fn parse_request_line(s string) ?(http.Method, urllib.URL, http.Version) {
words := s.split(' ')
if words.len != 3 {
return error('malformed request line')
}
method := http.method_from_str(words[0])
target := urllib.parse(words[1]) ?
version := http.version_from_str(words[2])
if version == .unknown {
return error('unsupported version')
}
return method, target, version
}
fn parse_header(s string) ?(string, string) {
if !s.contains(':') {
return error('missing colon in header')
}
words := s.split_nth(':', 2)
// TODO: parse quoted text according to the RFC
return words[0], words[1].trim_left(' \t')
}
// Parse URL encoded key=value&key=value forms
fn parse_form(body string) map[string]string {
words := body.split('&')
mut form := map[string]string{}
for word in words {
kv := word.split_nth('=', 2)
if kv.len != 2 {
continue
}
key := urllib.query_unescape(kv[0]) or { continue }
val := urllib.query_unescape(kv[1]) or { continue }
form[key] = val
}
return form
// }
// todo: parse form-data and application/json
// ...
}
// Parse the Content-Disposition header of a multipart form
// Returns a map of the key="value" pairs
// Example: parse_disposition('Content-Disposition: form-data; name="a"; filename="b"') == {'name': 'a', 'filename': 'b'}
fn parse_disposition(line string) map[string]string {
mut data := map[string]string{}
for word in line.split(';') {
kv := word.split_nth('=', 2)
if kv.len != 2 {
continue
}
key, value := kv[0].to_lower().trim_left(' \t'), kv[1].trim_right('\r')
if value.starts_with('"') && value.ends_with('"') {
data[key] = value[1..value.len - 1]
} else {
data[key] = value
}
}
return data
}
[manualfree]
fn lines_to_string(len int, lines []string, start int, end int) string {
mut sb := strings.new_builder(len)
for i in start .. end {
sb.writeln(lines[i])
}
sb.cut_last(1) // last newline
if sb.last_n(1) == '\r' {
sb.cut_last(1)
}
res := sb.str()
unsafe { sb.free() }
return res
}

View File

@ -1,141 +0,0 @@
module vweb
import net.http
import io
struct StringReader {
text string
mut:
place int
}
fn (mut s StringReader) read(mut buf []byte) ?int {
if s.place >= s.text.len {
return none
}
max_bytes := 100
end := if s.place + max_bytes >= s.text.len { s.text.len } else { s.place + max_bytes }
n := copy(buf, s.text[s.place..end].bytes())
s.place += n
return n
}
fn reader(s string) &io.BufferedReader {
return io.new_buffered_reader(
reader: &StringReader{
text: s
}
)
}
fn test_parse_request_not_http() {
mut reader_ := reader('hello')
parse_request(mut reader_) or { return }
panic('should not have parsed')
}
fn test_parse_request_no_headers() {
mut reader_ := reader('GET / HTTP/1.1\r\n\r\n')
req := parse_request(mut reader_) or { panic('did not parse: $err') }
assert req.method == .get
assert req.url == '/'
assert req.version == .v1_1
}
fn test_parse_request_two_headers() {
mut reader_ := reader('GET / HTTP/1.1\r\nTest1: a\r\nTest2: B\r\n\r\n')
req := parse_request(mut reader_) or { panic('did not parse: $err') }
assert req.header.custom_values('Test1') == ['a']
assert req.header.custom_values('Test2') == ['B']
}
fn test_parse_request_two_header_values() {
mut reader_ := reader('GET / HTTP/1.1\r\nTest1: a; b\r\nTest2: c\r\nTest2: d\r\n\r\n')
req := parse_request(mut reader_) or { panic('did not parse: $err') }
assert req.header.custom_values('Test1') == ['a; b']
assert req.header.custom_values('Test2') == ['c', 'd']
}
fn test_parse_request_body() {
mut reader_ := reader('GET / HTTP/1.1\r\nTest1: a\r\nTest2: b\r\nContent-Length: 4\r\n\r\nbodyabc')
req := parse_request(mut reader_) or { panic('did not parse: $err') }
assert req.data == 'body'
}
fn test_parse_request_line() {
method, target, version := parse_request_line('GET /target HTTP/1.1') or {
panic('did not parse: $err')
}
assert method == .get
assert target.str() == '/target'
assert version == .v1_1
}
fn test_parse_form() {
assert parse_form('foo=bar&bar=baz') == {
'foo': 'bar'
'bar': 'baz'
}
assert parse_form('foo=bar=&bar=baz') == {
'foo': 'bar='
'bar': 'baz'
}
assert parse_form('foo=bar%3D&bar=baz') == {
'foo': 'bar='
'bar': 'baz'
}
assert parse_form('foo=b%26ar&bar=baz') == {
'foo': 'b&ar'
'bar': 'baz'
}
assert parse_form('a=b& c=d') == {
'a': 'b'
' c': 'd'
}
assert parse_form('a=b&c= d ') == {
'a': 'b'
'c': ' d '
}
}
fn test_parse_multipart_form() {
boundary := '6844a625b1f0b299'
names := ['foo', 'fooz']
file := 'bar.v'
ct := 'application/octet-stream'
contents := ['baz', 'buzz']
data := "--------------------------$boundary
Content-Disposition: form-data; name=\"${names[0]}\"; filename=\"$file\"
Content-Type: $ct
${contents[0]}
--------------------------$boundary
Content-Disposition: form-data; name=\"${names[1]}\"
${contents[1]}
--------------------------$boundary--
"
form, files := http.parse_multipart_form(data, boundary)
assert files == {
names[0]: [
http.FileData{
filename: file
content_type: ct
data: contents[0]
},
]
}
assert form == {
names[1]: contents[1]
}
}
fn test_parse_large_body() ? {
body := 'ABCEF\r\n'.repeat(1024 * 1024) // greater than max_bytes
req := 'GET / HTTP/1.1\r\nContent-Length: $body.len\r\n\r\n$body'
mut reader_ := reader(req)
result := parse_request(mut reader_) ?
assert result.data.len == body.len
assert result.data == body
}

View File

@ -10,6 +10,13 @@ import net.http
import net.urllib
import time
// A type which don't get filtered inside templates
pub type RawHtml = string
// A dummy structure that returns from routes to indicate that you actually sent something to a user
[noinit]
pub struct Result {}
pub const (
methods_with_form = [http.Method.post, .put, .patch]
headers_close = http.new_custom_header_from_map({
@ -128,24 +135,37 @@ pub const (
default_port = 8080
)
// The Context struct represents the Context which hold the HTTP request and response.
// It has fields for the query, form, files.
pub struct Context {
mut:
content_type string = 'text/plain'
status string = '200 OK'
pub:
// HTTP Request
req http.Request
// TODO Response
pub mut:
done bool
// time.ticks() from start of vweb connection handle.
// You can use it to determine how much time is spent on your request.
page_gen_start i64
// TCP connection to client.
// But beware, do not store it for further use, after request processing vweb will close connection.
conn &net.TcpConn
static_files map[string]string
static_mime_types map[string]string
form map[string]string
query map[string]string
files map[string][]http.FileData
header http.Header // response headers
done bool
page_gen_start i64
form_error string
// Map containing query params for the route.
// Example: `http://localhost:3000/index?q=vpm&order_by=desc => { 'q': 'vpm', 'order_by': 'desc' }
query map[string]string
// Multipart-form fields.
form map[string]string
// Files from multipart-form.
files map[string][]http.FileData
header http.Header // response headers
// ? It doesn't seem to be used anywhere
form_error string
}
struct FileData {
@ -155,22 +175,21 @@ pub:
data string
}
struct UnexpectedExtraAttributeError {
msg string
code int
struct Route {
methods []http.Method
path string
}
struct MultiplePathAttributesError {
msg string = 'Expected at most one path attribute'
code int
}
// declaring init_server in your App struct is optional
// Defining this method is optional.
// This method called at server start.
// You can use it for initializing globals.
pub fn (ctx Context) init_server() {
eprintln('init_server() has been deprecated, please init your web app in `fn main()`')
}
// declaring before_request in your App struct is optional
// Defining this method is optional.
// This method called before every request (aka middleware).
// Probably you can use it for check user session cookie or add header.
pub fn (ctx Context) before_request() {}
pub struct Cookie {
@ -181,10 +200,6 @@ pub struct Cookie {
http_only bool
}
[noinit]
pub struct Result {
}
// vweb intern function
[manualfree]
pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bool {
@ -330,13 +345,6 @@ pub fn (ctx &Context) get_header(key string) string {
return ctx.req.header.get_custom(key) or { '' }
}
/*
pub fn run<T>(port int) {
mut x := &T{}
run_app(mut x, port)
}
*/
interface DbInterface {
db voidptr
}
@ -344,25 +352,22 @@ interface DbInterface {
// run_app
[manualfree]
pub fn run<T>(global_app &T, port int) {
// mut global_app := &T{}
// mut app := &T{}
// run_app<T>(mut app, port)
mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code $err') }
// Parsing methods attributes
mut routes := map[string]Route{}
$for method in T.methods {
http_methods, route_path := parse_attrs(method.name, method.attrs) or {
eprintln('error parsing method attributes: $err')
return
}
routes[method.name] = Route{
methods: http_methods
path: route_path
}
}
println('[Vweb] Running app on http://localhost:$port')
// app.Context = Context{
// conn: 0
//}
// app.init_server()
// unsafe {
// global_app.init_server()
//}
//$for method in T.methods {
//$if method.return_type is Result {
// check routes for validity
//}
//}
for {
// Create a new app object for each connection, copy global data like db connections
mut request_app := &T{}
@ -377,20 +382,17 @@ pub fn run<T>(global_app &T, port int) {
}
}
request_app.Context = global_app.Context // copy the context ref that contains static files map etc
// request_app.Context = Context{
// conn: 0
//}
mut conn := l.accept() or {
// failures should not panic
eprintln('accept() failed with error: $err.msg')
continue
}
go handle_conn<T>(mut conn, mut request_app)
go handle_conn<T>(mut conn, mut request_app, routes)
}
}
[manualfree]
fn handle_conn<T>(mut conn net.TcpConn, mut app T) {
fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
conn.set_read_timeout(30 * time.second)
conn.set_write_timeout(30 * time.second)
defer {
@ -399,88 +401,77 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T) {
free(app)
}
}
mut reader := io.new_buffered_reader(reader: conn)
defer {
reader.free()
}
page_gen_start := time.ticks()
req := parse_request(mut reader) or {
// Request parse
req := http.parse_request(mut reader) or {
// Prevents errors from being thrown when BufferedReader is empty
if '$err' != 'none' {
eprintln('error parsing request: $err')
}
return
}
app.Context = Context{
req: req
conn: conn
form: map[string]string{}
static_files: app.static_files
static_mime_types: app.static_mime_types
page_gen_start: page_gen_start
}
if req.method in vweb.methods_with_form {
ct := req.header.get(.content_type) or { '' }.split(';').map(it.trim_left(' \t'))
if 'multipart/form-data' in ct {
boundary := ct.filter(it.starts_with('boundary='))
if boundary.len != 1 {
send_string(mut conn, vweb.http_400.bytestr()) or {}
return
}
form, files := http.parse_multipart_form(req.data, boundary[0][9..])
for k, v in form {
app.form[k] = v
}
for k, v in files {
app.files[k] = v
}
} else {
form := parse_form(req.data)
for k, v in form {
app.form[k] = v
}
}
}
// Serve a static file if it is one
// TODO: get the real path
url := urllib.parse(app.req.url) or {
// URL Parse
url := urllib.parse(req.url) or {
eprintln('error parsing path: $err')
return
}
// Query parse
query := parse_query_from_url(url)
url_words := url.path.split('/').filter(it != '')
// Form parse
form, files := parse_form_from_request(req) or {
// Bad request
conn.write(vweb.http_400.bytes()) or {}
return
}
app.Context = Context{
req: req
page_gen_start: page_gen_start
conn: conn
query: query
form: form
files: files
static_files: app.static_files
static_mime_types: app.static_mime_types
}
// Calling middleware...
app.before_request()
// Static handling
if serve_if_static<T>(mut app, url) {
// successfully served a static file
return
}
app.before_request()
// Call the right action
$if debug {
println('route matching...')
}
url_words := url.path.split('/').filter(it != '')
// copy query args to app.query
for k, v in url.query().data {
app.query[k] = v.data[0]
}
// Route matching
$for method in T.methods {
$if method.return_type is Result {
mut method_args := []string{}
// TODO: move to server start
http_methods, route_path := parse_attrs(method.name, method.attrs) or {
eprintln('error parsing method attributes: $err')
return
route := routes[method.name] or {
eprintln('parsed attributes for the `$method.name` are not found, skipping...')
Route{}
}
// Used for route matching
route_words := route_path.split('/').filter(it != '')
// Skip if the HTTP request method does not match the attributes
if app.req.method in http_methods {
if req.method in route.methods {
// Used for route matching
route_words := route.path.split('/').filter(it != '')
// Route immediate matches first
// For example URL `/register` matches route `/:user`, but `fn register()`
// should be called first.
if !route_path.contains('/:') && url_words == route_words {
if !route.path.contains('/:') && url_words == route_words {
// We found a match
app.$method()
return
@ -492,7 +483,7 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T) {
}
if params := route_matches(url_words, route_words) {
method_args = params.clone()
method_args := params.clone()
if method_args.len != method.args.len {
eprintln('warning: uneven parameters count ($method.args.len) in `$method.name`, compared to the vweb route `$method.attrs` ($method_args.len)')
}
@ -502,9 +493,8 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T) {
}
}
}
// site not found
// send_string(mut conn, vweb.http_404.bytestr()) or {}
app.not_found()
// Route not found
conn.write(vweb.http_404.bytes()) or {}
}
fn route_matches(url_words []string, route_words []string) ?[]string {
@ -549,50 +539,6 @@ fn route_matches(url_words []string, route_words []string) ?[]string {
return params
}
// parse function attribute list for methods and a path
fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
if attrs.len == 0 {
return [http.Method.get], '/$name'
}
mut x := attrs.clone()
mut methods := []http.Method{}
mut path := ''
for i := 0; i < x.len; {
attr := x[i]
attru := attr.to_upper()
m := http.method_from_str(attru)
if attru == 'GET' || m != .get {
methods << m
x.delete(i)
continue
}
if attr.starts_with('/') {
if path != '' {
return IError(&MultiplePathAttributesError{})
}
path = attr
x.delete(i)
continue
}
i++
}
if x.len > 0 {
return IError(&UnexpectedExtraAttributeError{
msg: 'Encountered unexpected extra attributes: $x'
})
}
if methods.len == 0 {
methods = [http.Method.get]
}
if path == '' {
path = '/$name'
}
// Make path lowercase for case-insensitive comparisons
return methods, path.to_lower()
}
// check if request is for a static file and serves it
// returns true if we served a static file, false otherwise
[manualfree]
@ -696,6 +642,13 @@ pub fn not_found() Result {
return Result{}
}
fn send_string(mut conn net.TcpConn, s string) ? {
conn.write(s.bytes()) ?
}
// Do not delete.
// It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside vweb templates
// TODO: move it to template render
fn filter(s string) string {
return s.replace_each([
'<',
@ -706,10 +659,3 @@ fn filter(s string) string {
'&amp;',
])
}
// A type which don't get filtered inside templates
pub type RawHtml = string
fn send_string(mut conn net.TcpConn, s string) ? {
conn.write(s.bytes()) ?
}