From a43ebfeaf0a500a99f5aad224f63e3804aaddeb6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 11 Jan 2022 21:01:09 +0100 Subject: [PATCH] Initially working static file streaming! --- .woodpecker.yml | 1 - vieter/main.v | 10 +++---- vieter/repo.v | 10 ++++--- vieter/routes.v | 26 ++++++++++++++++-- vieter/web/web.v | 71 ++++++++++++++++++++++++++++++++++++++++++------ 5 files changed, 98 insertions(+), 20 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 1c6fb14..210ab28 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -7,6 +7,5 @@ pipeline: build: image: 'chewingbever/vlang:latest' commands: - - apt-get install -y --no-install-recommends openssl - v vieter diff --git a/vieter/main.v b/vieter/main.v index e11e2b8..d2c96e4 100644 --- a/vieter/main.v +++ b/vieter/main.v @@ -15,9 +15,9 @@ const db_name = 'pieter.db.tar.gz' struct App { web.Context pub: - api_key string [required; web_global] + api_key string [required; web_global] pub mut: - repo repo.Repo [required; web_global] + repo repo.Repo [required; web_global] } [noreturn] @@ -81,9 +81,9 @@ fn main() { } repo := repo.Repo{ - dir: repo_dir - name: db_name - } + dir: repo_dir + name: db_name + } // We create the upload directory during startup if !os.is_dir(repo.pkg_dir()) { diff --git a/vieter/repo.v b/vieter/repo.v index 55b38f3..6b307e8 100644 --- a/vieter/repo.v +++ b/vieter/repo.v @@ -6,7 +6,9 @@ const pkgs_subpath = 'pkgs' // Dummy struct to work around the fact that you can only share structs, maps & // arrays -pub struct Dummy { x int } +pub struct Dummy { + x int +} // Handles management of a repository. Package files are stored in '$dir/pkgs' // & moved there if necessary. @@ -14,17 +16,17 @@ pub struct Repo { mut: mutex shared Dummy pub: - dir string [required] + dir string [required] name string [required] } pub fn (r &Repo) pkg_dir() string { - return os.join_path_single(r.dir, pkgs_subpath) + return os.join_path_single(r.dir, repo.pkgs_subpath) } // Returns path to the given package, prepended with the repo's path. pub fn (r &Repo) pkg_path(pkg string) string { - return os.join_path(r.dir, pkgs_subpath, pkg) + return os.join_path(r.dir, repo.pkgs_subpath, pkg) } pub fn (r &Repo) exists(pkg string) bool { diff --git a/vieter/routes.v b/vieter/routes.v index 3dfb1ce..a8e5e1a 100644 --- a/vieter/routes.v +++ b/vieter/routes.v @@ -19,8 +19,31 @@ fn pretty_bytes(bytes int) string { return '${n:.2}${prefixes[i]}' } +fn is_pkg_name(s string) bool { + return s.contains('.pkg') +} + +['/:filename'; get] +fn (mut app App) get_root(filename string) web.Result { + mut full_path := '' + + if is_pkg_name(filename) { + full_path = os.join_path_single(app.repo.pkg_dir(), filename) + } else { + full_path = os.join_path_single(app.repo.dir, filename) + } + + return app.file(full_path) +} + ['/pkgs/:pkg'; put] fn (mut app App) put_package(pkg string) web.Result { + if !is_pkg_name(pkg) { + app.lwarn("Invalid package name '$pkg'.") + + return app.text('Invalid filename.') + } + if app.repo.exists(pkg) { app.lwarn("Duplicate package '$pkg'") @@ -33,7 +56,7 @@ fn (mut app App) put_package(pkg string) web.Result { app.ldebug("Uploading $length (${pretty_bytes(length.int())}) bytes to package '$pkg'.") // This is used to time how long it takes to upload a file - mut sw := time.new_stopwatch(time.StopWatchOptions{auto_start: true}) + mut sw := time.new_stopwatch(time.StopWatchOptions{ auto_start: true }) reader_to_file(mut app.reader, length.int(), pkg_path) or { app.lwarn("Failed to upload package '$pkg'") @@ -43,7 +66,6 @@ fn (mut app App) put_package(pkg string) web.Result { sw.stop() app.ldebug("Upload of package '$pkg' completed in ${sw.elapsed().seconds():.3}s.") - } else { app.lwarn("Tried to upload package '$pkg' without specifying a Content-Length.") return app.text("Content-Type header isn't set.") diff --git a/vieter/web/web.v b/vieter/web/web.v index 50fd087..39faaa7 100644 --- a/vieter/web/web.v +++ b/vieter/web/web.v @@ -257,20 +257,75 @@ pub fn (mut ctx Context) json_pretty(j T) Result { } // Response HTTP_OK with file as payload +// This function manually implements responses because it needs to stream the file contents pub fn (mut ctx Context) file(f_path string) Result { + if ctx.done { + return Result{} + } + ctx.done = true + + if !os.is_file(f_path) { + return ctx.not_found() + } + ext := os.file_ext(f_path) - data := os.read_file(f_path) or { - eprint(err.msg) + // data := os.read_file(f_path) or { + // eprint(err.msg) + // ctx.server_error(500) + // return Result{} + // } + // content_type := web.mime_types[ext] + // if content_type == '' { + // eprintln('no MIME type found for extension $ext') + // ctx.server_error(500) + + // return Result{} + // } + + // First, we return the headers for the request + + // We open the file before sending the headers in case reading fails + file_size := os.file_size(f_path) + + file := os.open(f_path) or { + eprintln(err.msg) ctx.server_error(500) return Result{} } - content_type := web.mime_types[ext] - if content_type == '' { - eprintln('no MIME type found for extension $ext') - ctx.server_error(500) - } else { - ctx.send_response_to_client(content_type, data) + + // build header + header := http.new_header_from_map({ + // http.CommonHeader.content_type: content_type + http.CommonHeader.content_length: file_size.str() + }).join(ctx.header) + + mut resp := http.Response{ + header: header.join(web.headers_close) } + resp.set_version(.v1_1) + resp.set_status(http.status_from_int(ctx.status.int())) + send_string(mut ctx.conn, resp.bytestr()) or { return Result{} } + + mut buf := []byte{len: 1_000_000} + mut bytes_left := file_size + + // Repeat as long as the stream still has data + for bytes_left > 0 { + // TODO check if just breaking here is safe + bytes_read := file.read(mut buf) or { break } + bytes_left -= u64(bytes_read) + + mut to_write := bytes_read + + for to_write > 0 { + // TODO don't just loop infinitely here + bytes_written := ctx.conn.write(buf[bytes_read - to_write..bytes_read]) or { continue } + + to_write = to_write - bytes_written + } + } + + ctx.done = true return Result{} }