From 1967f395f15b28647b49386535a0bea5e16099fa Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 9 Jan 2022 17:44:47 +0100 Subject: [PATCH] Started logging stuff; added vweb source code --- .gitignore | 2 + Makefile | 2 +- vieter/main.v | 54 ++- vieter/vweb/README.md | 141 ++++++ vieter/vweb/assets/assets.v | 201 ++++++++ vieter/vweb/assets/assets_test.v | 179 +++++++ vieter/vweb/parse.v | 74 +++ vieter/vweb/route_test.v | 282 +++++++++++ vieter/vweb/sse/sse.v | 77 +++ vieter/vweb/tests/vweb_test.v | 300 ++++++++++++ vieter/vweb/tests/vweb_test_server.v | 118 +++++ vieter/vweb/vweb.v | 698 +++++++++++++++++++++++++++ vieter/vweb/vweb_app_test.v | 78 +++ 13 files changed, 2196 insertions(+), 10 deletions(-) create mode 100644 vieter/vweb/README.md create mode 100644 vieter/vweb/assets/assets.v create mode 100644 vieter/vweb/assets/assets_test.v create mode 100644 vieter/vweb/parse.v create mode 100644 vieter/vweb/route_test.v create mode 100644 vieter/vweb/sse/sse.v create mode 100644 vieter/vweb/tests/vweb_test.v create mode 100644 vieter/vweb/tests/vweb_test_server.v create mode 100644 vieter/vweb/vweb.v create mode 100644 vieter/vweb/vweb_app_test.v diff --git a/.gitignore b/.gitignore index 4114c9d..8d15c59 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ vieter/vieter # Ignore testing files *.pkg* + +vieter.log diff --git a/Makefile b/Makefile index 0a22bbd..f936e4c 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: run run: - API_KEY=test REPO_DIR=data v watch run vieter + API_KEY=test REPO_DIR=data LOG_LEVEL=DEBUG v watch run vieter .PHONY: fmt fmt: diff --git a/vieter/main.v b/vieter/main.v index b4b4640..967dc9f 100644 --- a/vieter/main.v +++ b/vieter/main.v @@ -2,13 +2,15 @@ module main import vweb import os +import log const port = 8000 struct App { vweb.Context - api_key string [required] - repo_dir string [required] + api_key string [required; vweb_global] + repo_dir string [required; vweb_global] + logger log.Log [required; vweb_global] } [noreturn] @@ -17,21 +19,54 @@ fn exit_with_message(code int, msg string) { exit(code) } -[put; '/pkgs/:filename'] +[post; '/publish'] fn (mut app App) put_package(filename string) vweb.Result { - os.write_file('$app.repo_dir/$filename', app.req.data) or { - return app.text(err.msg) + for _, files in app.files { + for file in files { + filepath := os.join_path_single(app.repo_dir, file.filename) + + if os.exists(filepath) { + return app.text('File already exists.') + } + + os.write_file(filepath, file.data) or { + return app.text('Failed to upload file.') + } + + return app.text('yeet') + + } } - return app.text('yeet') + return app.text('done') } fn main() { + // Configure logger + log_level_str := os.getenv_opt('LOG_LEVEL') or { 'WARN' } + log_level := log.level_from_tag(log_level_str) or { + exit_with_message(1, 'Invalid log level. The allowed values are FATAL, ERROR, WARN, INFO & DEBUG.') } + log_file := os.getenv_opt('LOG_FILE') or { 'vieter.log' } + + mut logger := log.Log{ + level: log_level + } + logger.set_full_logpath(log_file) + logger.log_to_console_too() + defer { + logger.info('Flushing log file') + logger.flush() + logger.close() + } + // logger.set + logger.info('Logger set up.') + logger.flush() + + // Configure vweb server key := os.getenv_opt('API_KEY') or { exit_with_message(1, 'No API key was provided.') } repo_dir := os.getenv_opt('REPO_DIR') or { exit_with_message(1, 'No repo directory was configured.') } - println(repo_dir) // We create the upload directory during startup if !os.is_dir(repo_dir) { @@ -41,7 +76,8 @@ fn main() { } vweb.run(&App{ - api_key: key - repo_dir: repo_dir + api_key: key, + repo_dir: repo_dir, + logger: logger }, port) } diff --git a/vieter/vweb/README.md b/vieter/vweb/README.md new file mode 100644 index 0000000..850048c --- /dev/null +++ b/vieter/vweb/README.md @@ -0,0 +1,141 @@ +# vweb - the V Web Server # + +A simple yet powerful web server with built-in routing, parameter handling, +templating, and other features. + +## Alpha level software ## + +Some features may not be complete, and there may still be bugs. However, it is +still a very useful tool. The [gitly](https://gitly.org/) site is based on vweb. + +## Features ## + +- **Very fast** performance of C on the web. +- **Small binary** hello world website is <100 KB. +- **Easy to deploy** just one binary file that also includes all templates. + No need to install any dependencies. +- **Templates are precompiled** all errors are visible at compilation time, + not at runtime. + +There is no formal documentation yet - here is a simple +[example](https://github.com/vlang/v/tree/master/examples/vweb/vweb_example.v) + +There's also the V forum, [vorum](https://github.com/vlang/vorum) + +`vorum.v` contains all GET and POST actions. + +```v ignore +pub fn (app mut App) index() { + posts := app.find_all_posts() + $vweb.html() +} + +// TODO ['/post/:id/:title'] +// TODO `fn (app App) post(id int)` +pub fn (app App) post() { + id := app.get_post_id() + post := app.retrieve_post(id) or { + app.redirect('/') + return + } + comments := app.find_comments(id) + show_form := true + $vweb.html() +} +``` + +`index.html` is an example of the V template language: + +```html +@for post in posts +
+ @post.title + + @post.nr_comments + @post.time +
+@end +``` + +`$vweb.html()` compiles an HTML template into V during compilation, +and embeds the resulting code into the current action. + +That means that the template automatically has access to that action's entire environment. + +## Deploying vweb apps ## + +Everything, including HTML templates, is in one binary file. That's all you need to deploy. + +## Getting Started ## + +To start with vweb, you have to import the module `vweb`. After the import, +define a struct to hold vweb.Context (and any other variables your program will +need). + +The web server can be started by calling `vweb.run(&App{}, port)`. + +**Example:** + +```v ignore +import vweb + +struct App { + vweb.Context +} + +fn main() { + vweb.run(&App{}, 8080) +} +``` + +### Defining endpoints ### + +To add endpoints to your web server, you have to extend the `App` struct. +For routing you can either use auto-mapping of function names or specify the path as an attribute. +The function expects a response of the type `vweb.Result`. + +**Example:** + +```v ignore +// This endpoint can be accessed via http://localhost:port/hello +fn (mut app App) hello() vweb.Result { + return app.text('Hello') +} + +// This endpoint can be accessed via http://localhost:port/foo +["/foo"] +fn (mut app App) world() vweb.Result { + return app.text('World') +} +``` + +To create an HTTP POST endpoint, you simply add a `[post]` attribute before the function definition. + +**Example:** + +```v ignore +[post] +fn (mut app App) world() vweb.Result { + return app.text('World') +} +``` + +To pass a parameter to an endpoint, you simply define it inside +an attribute, e. g. `['/hello/:user]`. +After it is defined in the attribute, you have to add it as a function parameter. + +**Example:** + +```v ignore +['/hello/:user'] +fn (mut app App) hello_user(user string) vweb.Result { + return app.text('Hello $user') +} +``` + +You have access to the raw request data such as headers +or the request body by accessing `app` (which is `vweb.Context`). +If you want to read the request body, you can do that by calling `app.req.data`. +To read the request headers, you just call `app.req.header` and access the +header you want, e.g. `app.req.header.get(.content_type)`. See `struct Header` +for all available methods (`v doc net.http Header`). diff --git a/vieter/vweb/assets/assets.v b/vieter/vweb/assets/assets.v new file mode 100644 index 0000000..09a4ab9 --- /dev/null +++ b/vieter/vweb/assets/assets.v @@ -0,0 +1,201 @@ +module assets + +// this module provides an AssetManager for combining +// and caching javascript & css. +import os +import time +import crypto.md5 + +const ( + unknown_asset_type_error = 'vweb.assets: unknown asset type' +) + +struct AssetManager { +mut: + css []Asset + js []Asset +pub mut: + // when true assets will be minified + minify bool + // the directory to store the cached/combined files + cache_dir string +} + +struct Asset { + file_path string + last_modified time.Time +} + +// new_manager returns a new AssetManager +pub fn new_manager() &AssetManager { + return &AssetManager{} +} + +// add_css adds a css asset +pub fn (mut am AssetManager) add_css(file string) bool { + return am.add('css', file) +} + +// add_js adds a js asset +pub fn (mut am AssetManager) add_js(file string) bool { + return am.add('js', file) +} + +// combine_css returns the combined css as a string when to_file is false +// when to_file is true it combines the css to disk and returns the path of the file +pub fn (am AssetManager) combine_css(to_file bool) string { + return am.combine('css', to_file) +} + +// combine_js returns the combined js as a string when to_file is false +// when to_file is true it combines the css to disk and returns the path of the file +pub fn (am AssetManager) combine_js(to_file bool) string { + return am.combine('js', to_file) +} + +// include_css returns the html tag(s) for including the css files in a page. +// when combine is true the files are combined. +pub fn (am AssetManager) include_css(combine bool) string { + return am.include('css', combine) +} + +// include_js returns the html \n' + } + for asset in assets { + out += '\n' + } + } + return out +} + +// dont return option until size limit is removed +// fn (mut am AssetManager) add(asset_type, file string) ?bool { +fn (mut am AssetManager) add(asset_type string, file string) bool { + if !os.exists(file) { + // return error('vweb.assets: cannot add asset $file, it does not exist') + return false + } + asset := Asset{ + file_path: file + last_modified: time.Time{ + unix: os.file_last_mod_unix(file) + } + } + if asset_type == 'css' { + am.css << asset + } else if asset_type == 'js' { + am.js << asset + } else { + panic('$assets.unknown_asset_type_error ($asset_type).') + } + return true +} + +fn (am AssetManager) exists(asset_type string, file string) bool { + assets := am.get_assets(asset_type) + for asset in assets { + if asset.file_path == file { + return true + } + } + return false +} + +fn (am AssetManager) get_assets(asset_type string) []Asset { + if asset_type != 'css' && asset_type != 'js' { + panic('$assets.unknown_asset_type_error ($asset_type).') + } + assets := if asset_type == 'css' { am.css } else { am.js } + return assets +} + +// todo: implement proper minification +pub fn minify_css(css string) string { + mut lines := css.split('\n') + for i, _ in lines { + lines[i] = lines[i].trim_space() + } + return lines.join(' ') +} + +// todo: implement proper minification +pub fn minify_js(js string) string { + mut lines := js.split('\n') + for i, _ in lines { + lines[i] = lines[i].trim_space() + } + return lines.join(' ') +} diff --git a/vieter/vweb/assets/assets_test.v b/vieter/vweb/assets/assets_test.v new file mode 100644 index 0000000..6170f3c --- /dev/null +++ b/vieter/vweb/assets/assets_test.v @@ -0,0 +1,179 @@ +import vweb.assets +import os + +// clean_cache_dir used before and after tests that write to a cache directory. +// Because of parallel compilation and therefore test running, +// unique cache dirs are needed per test function. +fn clean_cache_dir(dir string) { + if os.is_dir(dir) { + os.rmdir_all(dir) or { panic(err) } + } +} + +fn base_cache_dir() string { + return os.join_path(os.temp_dir(), 'assets_test_cache') +} + +fn cache_dir(test_name string) string { + return os.join_path(base_cache_dir(), test_name) +} + +fn get_test_file_path(file string) string { + path := os.join_path(base_cache_dir(), file) + if !os.is_dir(base_cache_dir()) { + os.mkdir_all(base_cache_dir()) or { panic(err) } + } + if !os.exists(path) { + os.write_file(path, get_test_file_contents(file)) or { panic(err) } + } + return path +} + +fn get_test_file_contents(file string) string { + contents := match file { + 'test1.js' { '{"one": 1}\n' } + 'test2.js' { '{"two": 2}\n' } + 'test1.css' { '.one {\n\tcolor: #336699;\n}\n' } + 'test2.css' { '.two {\n\tcolor: #996633;\n}\n' } + else { 'wibble\n' } + } + return contents +} + +fn test_set_cache() { + mut am := assets.new_manager() + am.cache_dir = 'cache' +} + +fn test_set_minify() { + mut am := assets.new_manager() + am.minify = true +} + +fn test_add() { + mut am := assets.new_manager() + assert am.add('css', 'testx.css') == false + assert am.add('css', get_test_file_path('test1.css')) == true + assert am.add('js', get_test_file_path('test1.js')) == true + // assert am.add('css', get_test_file_path('test2.js')) == false // TODO: test extension on add +} + +fn test_add_css() { + mut am := assets.new_manager() + assert am.add_css('testx.css') == false + assert am.add_css(get_test_file_path('test1.css')) == true + // assert am.add_css(get_test_file_path('test1.js')) == false // TODO: test extension on add +} + +fn test_add_js() { + mut am := assets.new_manager() + assert am.add_js('testx.js') == false + assert am.add_css(get_test_file_path('test1.js')) == true + // assert am.add_css(get_test_file_path('test1.css')) == false // TODO: test extension on add +} + +fn test_combine_css() { + mut am := assets.new_manager() + am.cache_dir = cache_dir('test_combine_css') + clean_cache_dir(am.cache_dir) + am.add_css(get_test_file_path('test1.css')) + am.add_css(get_test_file_path('test2.css')) + // TODO: How do I test non-minified, is there a "here doc" format that keeps formatting? + am.minify = true + expected := '.one { color: #336699; } .two { color: #996633; } ' + actual := am.combine_css(false) + assert actual == expected + assert actual.contains(expected) + // Test cache path doesn't change when input files and minify setting do not. + path1 := am.combine_css(true) + clean_cache_dir(am.cache_dir) + path2 := am.combine_css(true) + assert path1 == path2 + clean_cache_dir(am.cache_dir) +} + +fn test_combine_js() { + mut am := assets.new_manager() + am.cache_dir = cache_dir('test_combine_js') + clean_cache_dir(am.cache_dir) + am.add_js(get_test_file_path('test1.js')) + am.add_js(get_test_file_path('test2.js')) + expected1 := '{"one": 1}' + expected2 := '{"two": 2}' + expected := expected1 + '\n' + expected2 + '\n' + actual := am.combine_js(false) + assert actual == expected + assert actual.contains(expected) + assert actual.contains(expected1) + assert actual.contains(expected2) + am.minify = true + clean_cache_dir(am.cache_dir) + expected3 := expected1 + ' ' + expected2 + ' ' + actual2 := am.combine_js(false) + assert actual2 == expected3 + assert actual2.contains(expected3) + // Test cache path doesn't change when input files and minify setting do not. + path1 := am.combine_js(true) + clean_cache_dir(am.cache_dir) + path2 := am.combine_js(true) + assert path1 == path2 + clean_cache_dir(am.cache_dir) +} + +fn test_include_css() { + mut am := assets.new_manager() + file1 := get_test_file_path('test1.css') + am.add_css(file1) + expected := '\n' + actual := am.include_css(false) + assert actual == expected + assert actual.contains(expected) + // Two lines of output. + file2 := get_test_file_path('test2.css') + am.add_css(file2) + am.cache_dir = cache_dir('test_include_css') + clean_cache_dir(am.cache_dir) + expected2 := expected + '\n' + actual2 := am.include_css(false) + assert actual2 == expected2 + assert actual2.contains(expected2) + // Combined output. + clean_cache_dir(am.cache_dir) + actual3 := am.include_css(true) + assert actual3.contains(expected2) == false + assert actual3.starts_with('\n' + actual := am.include_js(false) + assert actual == expected + assert actual.contains(expected) + // Two lines of output. + file2 := get_test_file_path('test2.js') + am.add_js(file2) + am.cache_dir = cache_dir('test_include_js') + clean_cache_dir(am.cache_dir) + expected2 := expected + '\n' + actual2 := am.include_js(false) + assert actual2 == expected2 + assert actual2.contains(expected2) + // Combined output. + clean_cache_dir(am.cache_dir) + actual3 := am.include_js(true) + assert actual3.contains(expected2) == false + assert actual3.starts_with('