vweb: router refactor (#12041)

vlib/vweb/parse.v

@@ -0,0 +1,74 @@
+module vweb
+
+import net.urllib
+import net.http
+
+// Parsing function attributes for methods and path.
+fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
+	if attrs.len == 0 {
+		return [http.Method.get], '/$name'
+	}
+
+	mut x := attrs.clone()
+	mut methods := []http.Method{}
+	mut path := ''
+
+	for i := 0; i < x.len; {
+		attr := x[i]
+		attru := attr.to_upper()
+		m := http.method_from_str(attru)
+		if attru == 'GET' || m != .get {
+			methods << m
+			x.delete(i)
+			continue
+		}
+		if attr.starts_with('/') {
+			if path != '' {
+				return IError(http.MultiplePathAttributesError{})
+			}
+			path = attr
+			x.delete(i)
+			continue
+		}
+		i++
+	}
+	if x.len > 0 {
+		return IError(http.UnexpectedExtraAttributeError{
+			msg: 'Encountered unexpected extra attributes: $x'
+		})
+	}
+	if methods.len == 0 {
+		methods = [http.Method.get]
+	}
+	if path == '' {
+		path = '/$name'
+	}
+	// Make path lowercase for case-insensitive comparisons
+	return methods, path.to_lower()
+}
+
+fn parse_query_from_url(url urllib.URL) map[string]string {
+	mut query := map[string]string{}
+	for k, v in url.query().data {
+		query[k] = v.data[0]
+	}
+	return query
+}
+
+fn parse_form_from_request(request http.Request) ?(map[string]string, map[string][]http.FileData) {
+	mut form := map[string]string{}
+	mut files := map[string][]http.FileData{}
+	if request.method in methods_with_form {
+		ct := request.header.get(.content_type) or { '' }.split(';').map(it.trim_left(' \t'))
+		if 'multipart/form-data' in ct {
+			boundary := ct.filter(it.starts_with('boundary='))
+			if boundary.len != 1 {
+				return error('detected more that one form-data boundary')
+			}
+			form, files = http.parse_multipart_form(request.data, boundary[0][9..])
+		} else {
+			form = http.parse_form(request.data)
+		}
+	}
+	return form, files
+}

vlib/vweb/request.v

@@ -1,121 +0,0 @@
-module vweb
-
-import io
-import strings
-import net.http
-import net.urllib
-
-fn parse_request(mut reader io.BufferedReader) ?http.Request {
-	// request line
-	mut line := reader.read_line() ?
-	method, target, version := parse_request_line(line) ?
-
-	// headers
-	mut header := http.new_header()
-	line = reader.read_line() ?
-	for line != '' {
-		key, value := parse_header(line) ?
-		header.add_custom(key, value) ?
-		line = reader.read_line() ?
-	}
-	header.coerce(canonicalize: true)
-
-	// body
-	mut body := []byte{}
-	if length := header.get(.content_length) {
-		n := length.int()
-		if n > 0 {
-			body = []byte{len: n}
-			mut count := 0
-			for count < body.len {
-				count += reader.read(mut body[count..]) or { break }
-			}
-		}
-	}
-
-	return http.Request{
-		method: method
-		url: target.str()
-		header: header
-		data: body.bytestr()
-		version: version
-	}
-}
-
-fn parse_request_line(s string) ?(http.Method, urllib.URL, http.Version) {
-	words := s.split(' ')
-	if words.len != 3 {
-		return error('malformed request line')
-	}
-	method := http.method_from_str(words[0])
-	target := urllib.parse(words[1]) ?
-	version := http.version_from_str(words[2])
-	if version == .unknown {
-		return error('unsupported version')
-	}
-
-	return method, target, version
-}
-
-fn parse_header(s string) ?(string, string) {
-	if !s.contains(':') {
-		return error('missing colon in header')
-	}
-	words := s.split_nth(':', 2)
-	// TODO: parse quoted text according to the RFC
-	return words[0], words[1].trim_left(' \t')
-}
-
-// Parse URL encoded key=value&key=value forms
-fn parse_form(body string) map[string]string {
-	words := body.split('&')
-	mut form := map[string]string{}
-	for word in words {
-		kv := word.split_nth('=', 2)
-		if kv.len != 2 {
-			continue
-		}
-		key := urllib.query_unescape(kv[0]) or { continue }
-		val := urllib.query_unescape(kv[1]) or { continue }
-		form[key] = val
-	}
-	return form
-	// }
-	// todo: parse form-data and application/json
-	// ...
-}
-
-// Parse the Content-Disposition header of a multipart form
-// Returns a map of the key="value" pairs
-// Example: parse_disposition('Content-Disposition: form-data; name="a"; filename="b"') == {'name': 'a', 'filename': 'b'}
-fn parse_disposition(line string) map[string]string {
-	mut data := map[string]string{}
-	for word in line.split(';') {
-		kv := word.split_nth('=', 2)
-		if kv.len != 2 {
-			continue
-		}
-		key, value := kv[0].to_lower().trim_left(' \t'), kv[1].trim_right('\r')
-		if value.starts_with('"') && value.ends_with('"') {
-			data[key] = value[1..value.len - 1]
-		} else {
-			data[key] = value
-		}
-	}
-	return data
-}
-
-[manualfree]
-fn lines_to_string(len int, lines []string, start int, end int) string {
-	mut sb := strings.new_builder(len)
-	for i in start .. end {
-		sb.writeln(lines[i])
-	}
-	sb.cut_last(1) // last newline
-	if sb.last_n(1) == '\r' {
-		sb.cut_last(1)
-	}
-	res := sb.str()
-	unsafe { sb.free() }
-	return res
-}

vlib/vweb/request_test.v

@@ -1,141 +0,0 @@
-module vweb
-
-import net.http
-import io
-
-struct StringReader {
-	text string
-mut:
-	place int
-}
-
-fn (mut s StringReader) read(mut buf []byte) ?int {
-	if s.place >= s.text.len {
-		return none
-	}
-	max_bytes := 100
-	end := if s.place + max_bytes >= s.text.len { s.text.len } else { s.place + max_bytes }
-	n := copy(buf, s.text[s.place..end].bytes())
-	s.place += n
-	return n
-}
-
-fn reader(s string) &io.BufferedReader {
-	return io.new_buffered_reader(
-		reader: &StringReader{
-			text: s
-		}
-	)
-}
-
-fn test_parse_request_not_http() {
-	mut reader_ := reader('hello')
-	parse_request(mut reader_) or { return }
-	panic('should not have parsed')
-}
-
-fn test_parse_request_no_headers() {
-	mut reader_ := reader('GET / HTTP/1.1\r\n\r\n')
-	req := parse_request(mut reader_) or { panic('did not parse: $err') }
-	assert req.method == .get
-	assert req.url == '/'
-	assert req.version == .v1_1
-}
-
-fn test_parse_request_two_headers() {
-	mut reader_ := reader('GET / HTTP/1.1\r\nTest1: a\r\nTest2:  B\r\n\r\n')
-	req := parse_request(mut reader_) or { panic('did not parse: $err') }
-	assert req.header.custom_values('Test1') == ['a']
-	assert req.header.custom_values('Test2') == ['B']
-}
-
-fn test_parse_request_two_header_values() {
-	mut reader_ := reader('GET / HTTP/1.1\r\nTest1: a; b\r\nTest2: c\r\nTest2: d\r\n\r\n')
-	req := parse_request(mut reader_) or { panic('did not parse: $err') }
-	assert req.header.custom_values('Test1') == ['a; b']
-	assert req.header.custom_values('Test2') == ['c', 'd']
-}
-
-fn test_parse_request_body() {
-	mut reader_ := reader('GET / HTTP/1.1\r\nTest1: a\r\nTest2: b\r\nContent-Length: 4\r\n\r\nbodyabc')
-	req := parse_request(mut reader_) or { panic('did not parse: $err') }
-	assert req.data == 'body'
-}
-
-fn test_parse_request_line() {
-	method, target, version := parse_request_line('GET /target HTTP/1.1') or {
-		panic('did not parse: $err')
-	}
-	assert method == .get
-	assert target.str() == '/target'
-	assert version == .v1_1
-}
-
-fn test_parse_form() {
-	assert parse_form('foo=bar&bar=baz') == {
-		'foo': 'bar'
-		'bar': 'baz'
-	}
-	assert parse_form('foo=bar=&bar=baz') == {
-		'foo': 'bar='
-		'bar': 'baz'
-	}
-	assert parse_form('foo=bar%3D&bar=baz') == {
-		'foo': 'bar='
-		'bar': 'baz'
-	}
-	assert parse_form('foo=b%26ar&bar=baz') == {
-		'foo': 'b&ar'
-		'bar': 'baz'
-	}
-	assert parse_form('a=b& c=d') == {
-		'a':  'b'
-		' c': 'd'
-	}
-	assert parse_form('a=b&c= d ') == {
-		'a': 'b'
-		'c': ' d '
-	}
-}
-
-fn test_parse_multipart_form() {
-	boundary := '6844a625b1f0b299'
-	names := ['foo', 'fooz']
-	file := 'bar.v'
-	ct := 'application/octet-stream'
-	contents := ['baz', 'buzz']
-	data := "--------------------------$boundary
-Content-Disposition: form-data; name=\"${names[0]}\"; filename=\"$file\"
-Content-Type: $ct
-
-${contents[0]}
---------------------------$boundary
-Content-Disposition: form-data; name=\"${names[1]}\"
-
-${contents[1]}
---------------------------$boundary--
-"
-	form, files := http.parse_multipart_form(data, boundary)
-	assert files == {
-		names[0]: [
-			http.FileData{
-				filename: file
-				content_type: ct
-				data: contents[0]
-			},
-		]
-	}
-
-	assert form == {
-		names[1]: contents[1]
-	}
-}
-
-fn test_parse_large_body() ? {
-	body := 'ABCEF\r\n'.repeat(1024 * 1024) // greater than max_bytes
-	req := 'GET / HTTP/1.1\r\nContent-Length: $body.len\r\n\r\n$body'
-	mut reader_ := reader(req)
-	result := parse_request(mut reader_) ?
-	assert result.data.len == body.len
-	assert result.data == body
-}

vlib/vweb/vweb.v

@@ -10,6 +10,13 @@ import net.http
 import net.urllib
 import time
 
+// A type which don't get filtered inside templates
+pub type RawHtml = string
+
+// A dummy structure that returns from routes to indicate that you actually sent something to a user
+[noinit]
+pub struct Result {}
+
 pub const (
 	methods_with_form = [http.Method.post, .put, .patch]
 	headers_close     = http.new_custom_header_from_map({
@@ -128,24 +135,37 @@ pub const (
 	default_port       = 8080
 )
 
+// The Context struct represents the Context which hold the HTTP request and response.
+// It has fields for the query, form, files.
 pub struct Context {
 mut:
 	content_type string = 'text/plain'
 	status       string = '200 OK'
 pub:
+	// HTTP Request
 	req http.Request
 	// TODO Response
 pub mut:
+	done bool
+	// time.ticks() from start of vweb connection handle.
+	// You can use it to determine how much time is spent on your request.
+	page_gen_start i64
+	// TCP connection to client.
+	// But beware, do not store it for further use, after request processing vweb will close connection.
 	conn              &net.TcpConn
 	static_files      map[string]string
 	static_mime_types map[string]string
-	form              map[string]string
+	// Map containing query params for the route.
-	query             map[string]string
+	// Example: `http://localhost:3000/index?q=vpm&order_by=desc => { 'q': 'vpm', 'order_by': 'desc' }
-	files             map[string][]http.FileData
+	query map[string]string
-	header            http.Header // response headers
+	// Multipart-form fields.
-	done              bool
+	form map[string]string
-	page_gen_start    i64
+	// Files from multipart-form.
-	form_error        string
+	files map[string][]http.FileData
+
+	header http.Header // response headers
+	// ? It doesn't seem to be used anywhere
+	form_error string
 }
 
 struct FileData {
@@ -155,22 +175,21 @@ pub:
 	data         string
 }
 
-struct UnexpectedExtraAttributeError {
+struct Route {
-	msg  string
+	methods []http.Method
-	code int
+	path    string
 }
 
-struct MultiplePathAttributesError {
+// Defining this method is optional.
-	msg  string = 'Expected at most one path attribute'
+// This method called at server start.
-	code int
+// You can use it for initializing globals.
-}
-
-// declaring init_server in your App struct is optional
 pub fn (ctx Context) init_server() {
 	eprintln('init_server() has been deprecated, please init your web app in `fn main()`')
 }
 
-// declaring before_request in your App struct is optional
+// Defining this method is optional.
+// This method called before every request (aka middleware).
+// Probably you can use it for check user session cookie or add header.
 pub fn (ctx Context) before_request() {}
 
 pub struct Cookie {
@@ -181,10 +200,6 @@ pub struct Cookie {
 	http_only bool
 }
 
-[noinit]
-pub struct Result {
-}
-
 // vweb intern function
 [manualfree]
 pub fn (mut ctx Context) send_response_to_client(mimetype string, res string) bool {
@@ -330,13 +345,6 @@ pub fn (ctx &Context) get_header(key string) string {
 	return ctx.req.header.get_custom(key) or { '' }
 }
 
-/*
-pub fn run<T>(port int) {
-	mut x := &T{}
-	run_app(mut x, port)
-}
-*/
-
 interface DbInterface {
 	db voidptr
 }
@@ -344,25 +352,22 @@ interface DbInterface {
 // run_app
 [manualfree]
 pub fn run<T>(global_app &T, port int) {
-	// mut global_app := &T{}
-	// mut app := &T{}
-	// run_app<T>(mut app, port)
-
 	mut l := net.listen_tcp(.ip6, ':$port') or { panic('failed to listen $err.code $err') }
 
+	// Parsing methods attributes
+	mut routes := map[string]Route{}
+	$for method in T.methods {
+		http_methods, route_path := parse_attrs(method.name, method.attrs) or {
+			eprintln('error parsing method attributes: $err')
+			return
+		}
+
+		routes[method.name] = Route{
+			methods: http_methods
+			path: route_path
+		}
+	}
 	println('[Vweb] Running app on http://localhost:$port')
-	// app.Context = Context{
-	// conn: 0
-	//}
-	// app.init_server()
-	// unsafe {
-	// global_app.init_server()
-	//}
-	//$for method in T.methods {
-	//$if method.return_type is Result {
-	// check routes for validity
-	//}
-	//}
 	for {
 		// Create a new app object for each connection, copy global data like db connections
 		mut request_app := &T{}
@@ -377,20 +382,17 @@ pub fn run<T>(global_app &T, port int) {
 			}
 		}
 		request_app.Context = global_app.Context // copy the context ref that contains static files map etc
-		// request_app.Context = Context{
-		// conn: 0
-		//}
 		mut conn := l.accept() or {
 			// failures should not panic
 			eprintln('accept() failed with error: $err.msg')
 			continue
 		}
-		go handle_conn<T>(mut conn, mut request_app)
+		go handle_conn<T>(mut conn, mut request_app, routes)
 	}
 }
 
 [manualfree]
-fn handle_conn<T>(mut conn net.TcpConn, mut app T) {
+fn handle_conn<T>(mut conn net.TcpConn, mut app T, routes map[string]Route) {
 	conn.set_read_timeout(30 * time.second)
 	conn.set_write_timeout(30 * time.second)
 	defer {
@@ -399,88 +401,77 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T) {
 			free(app)
 		}
 	}
+
 	mut reader := io.new_buffered_reader(reader: conn)
 	defer {
 		reader.free()
 	}
+
 	page_gen_start := time.ticks()
-	req := parse_request(mut reader) or {
+
+	// Request parse
+	req := http.parse_request(mut reader) or {
 		// Prevents errors from being thrown when BufferedReader is empty
 		if '$err' != 'none' {
 			eprintln('error parsing request: $err')
 		}
 		return
 	}
-	app.Context = Context{
+
-		req: req
+	// URL Parse
-		conn: conn
+	url := urllib.parse(req.url) or {
-		form: map[string]string{}
-		static_files: app.static_files
-		static_mime_types: app.static_mime_types
-		page_gen_start: page_gen_start
-	}
-	if req.method in vweb.methods_with_form {
-		ct := req.header.get(.content_type) or { '' }.split(';').map(it.trim_left(' \t'))
-		if 'multipart/form-data' in ct {
-			boundary := ct.filter(it.starts_with('boundary='))
-			if boundary.len != 1 {
-				send_string(mut conn, vweb.http_400.bytestr()) or {}
-				return
-			}
-			form, files := http.parse_multipart_form(req.data, boundary[0][9..])
-			for k, v in form {
-				app.form[k] = v
-			}
-			for k, v in files {
-				app.files[k] = v
-			}
-		} else {
-			form := parse_form(req.data)
-			for k, v in form {
-				app.form[k] = v
-			}
-		}
-	}
-	// Serve a static file if it is one
-	// TODO: get the real path
-	url := urllib.parse(app.req.url) or {
 		eprintln('error parsing path: $err')
 		return
 	}
+
+	// Query parse
+	query := parse_query_from_url(url)
+	url_words := url.path.split('/').filter(it != '')
+
+	// Form parse
+	form, files := parse_form_from_request(req) or {
+		// Bad request
+		conn.write(vweb.http_400.bytes()) or {}
+		return
+	}
+
+	app.Context = Context{
+		req: req
+		page_gen_start: page_gen_start
+		conn: conn
+		query: query
+		form: form
+		files: files
+		static_files: app.static_files
+		static_mime_types: app.static_mime_types
+	}
+
+	// Calling middleware...
+	app.before_request()
+
+	// Static handling
 	if serve_if_static<T>(mut app, url) {
 		// successfully served a static file
 		return
 	}
 
-	app.before_request()
+	// Route matching
-	// Call the right action
-	$if debug {
-		println('route matching...')
-	}
-	url_words := url.path.split('/').filter(it != '')
-	// copy query args to app.query
-	for k, v in url.query().data {
-		app.query[k] = v.data[0]
-	}
-
 	$for method in T.methods {
 		$if method.return_type is Result {
-			mut method_args := []string{}
+			route := routes[method.name] or {
-			// TODO: move to server start
+				eprintln('parsed attributes for the `$method.name` are not found, skipping...')
-			http_methods, route_path := parse_attrs(method.name, method.attrs) or {
+				Route{}
-				eprintln('error parsing method attributes: $err')
-				return
 			}
 
-			// Used for route matching
-			route_words := route_path.split('/').filter(it != '')
-
 			// Skip if the HTTP request method does not match the attributes
-			if app.req.method in http_methods {
+			if req.method in route.methods {
+				// Used for route matching
+				route_words := route.path.split('/').filter(it != '')
+
 				// Route immediate matches first
 				// For example URL `/register` matches route `/:user`, but `fn register()`
 				// should be called first.
-				if !route_path.contains('/:') && url_words == route_words {
+				if !route.path.contains('/:') && url_words == route_words {
 					// We found a match
 					app.$method()
 					return
@@ -492,7 +483,7 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T) {
 				}
 
 				if params := route_matches(url_words, route_words) {
-					method_args = params.clone()
+					method_args := params.clone()
 					if method_args.len != method.args.len {
 						eprintln('warning: uneven parameters count ($method.args.len) in `$method.name`, compared to the vweb route `$method.attrs` ($method_args.len)')
 					}
@@ -502,9 +493,8 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T) {
 			}
 		}
 	}
-	// site not found
+	// Route not found
-	// send_string(mut conn, vweb.http_404.bytestr()) or {}
+	conn.write(vweb.http_404.bytes()) or {}
-	app.not_found()
 }
 
 fn route_matches(url_words []string, route_words []string) ?[]string {
@@ -549,50 +539,6 @@ fn route_matches(url_words []string, route_words []string) ?[]string {
 	return params
 }
 
-// parse function attribute list for methods and a path
-fn parse_attrs(name string, attrs []string) ?([]http.Method, string) {
-	if attrs.len == 0 {
-		return [http.Method.get], '/$name'
-	}
-
-	mut x := attrs.clone()
-	mut methods := []http.Method{}
-	mut path := ''
-
-	for i := 0; i < x.len; {
-		attr := x[i]
-		attru := attr.to_upper()
-		m := http.method_from_str(attru)
-		if attru == 'GET' || m != .get {
-			methods << m
-			x.delete(i)
-			continue
-		}
-		if attr.starts_with('/') {
-			if path != '' {
-				return IError(&MultiplePathAttributesError{})
-			}
-			path = attr
-			x.delete(i)
-			continue
-		}
-		i++
-	}
-	if x.len > 0 {
-		return IError(&UnexpectedExtraAttributeError{
-			msg: 'Encountered unexpected extra attributes: $x'
-		})
-	}
-	if methods.len == 0 {
-		methods = [http.Method.get]
-	}
-	if path == '' {
-		path = '/$name'
-	}
-	// Make path lowercase for case-insensitive comparisons
-	return methods, path.to_lower()
-}
-
 // check if request is for a static file and serves it
 // returns true if we served a static file, false otherwise
 [manualfree]
@@ -696,6 +642,13 @@ pub fn not_found() Result {
 	return Result{}
 }
 
+fn send_string(mut conn net.TcpConn, s string) ? {
+	conn.write(s.bytes()) ?
+}
+
+// Do not delete.
+// It used by `vlib/v/gen/c/str_intp.v:130` for string interpolation inside vweb templates
+// TODO: move it to template render
 fn filter(s string) string {
 	return s.replace_each([
 		'<',
@@ -706,10 +659,3 @@ fn filter(s string) string {
 		'&amp;',
 	])
 }
-
-// A type which don't get filtered inside templates
-pub type RawHtml = string
-
-fn send_string(mut conn net.TcpConn, s string) ? {
-	conn.write(s.bytes()) ?
-}