From 473c7ec06b10735b82959a50df7bb470d12ded7a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 11 May 2022 10:03:43 +0200 Subject: [PATCH 01/17] feat(docker): start of new socket implementation --- src/docker/docker.v | 4 ---- src/docker/socket.v | 47 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 src/docker/socket.v diff --git a/src/docker/docker.v b/src/docker/docker.v index 305e825..ce01e7e 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -5,10 +5,6 @@ import net.urllib import net.http import json -const socket = '/var/run/docker.sock' - -const buf_len = 1024 - // send writes a request to the Docker socket, waits for a response & returns // it. fn send(req &string) ?http.Response { diff --git a/src/docker/socket.v b/src/docker/socket.v new file mode 100644 index 0000000..3a72f14 --- /dev/null +++ b/src/docker/socket.v @@ -0,0 +1,47 @@ +module docker + +import net.unix +import io +import net.http + +const socket = '/var/run/docker.sock' + +const buf_len = 10 * 1024 + +pub struct DockerDaemon { +mut: + socket &unix.StreamConn + reader &io.BufferedReader +} + +pub fn new_conn() ?DockerDaemon { + s := unix.connect_stream(socket) ? + + d := DockerDaemon{socket: s, reader: io.new_buffered_reader(reader: s)} + + return d +} + +fn (mut d DockerDaemon) send_request(req &string) ? { + d.socket.write_string(req) ? + d.socket.wait_for_write() ? +} + +// read_response_head consumes the socket's contents until it encounters +// '\n\n', after which it parses the response as an HTTP response. +fn (mut d DockerDaemon) read_response_head() ?http.Response { + mut c := 0 + mut buf := [buf_len]u8{len: docker.buf_len} + mut res := []u8{} + + for { + c = d.socket.read(mut buf) ? + res << buf[..c] + + if res#[-2..] == [u8(`\n`), `\n`] { + break + } + } + + return http.parse_response(res.bytestr()) +} From dd9958ea28982d39c4dba26036cb9f03c304e27d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 13 May 2022 21:28:24 +0200 Subject: [PATCH 02/17] refactor: ran vfmt with new defaults feat(docker): started work on new implementation --- src/docker/containers.v | 13 +++++-- src/docker/docker.v | 20 +++++----- src/docker/socket.v | 83 ++++++++++++++++++++++++++++++++++------- src/main.v | 2 +- 4 files changed, 88 insertions(+), 30 deletions(-) diff --git a/src/docker/containers.v b/src/docker/containers.v index 14ac12d..3b674e6 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -3,17 +3,22 @@ module docker import json import net.urllib import time +import net.http struct Container { id string [json: Id] names []string [json: Names] } -// containers returns a list of all currently running containers -pub fn containers() ?[]Container { - res := request('GET', urllib.parse('/v1.41/containers/json')?)? +pub fn (mut d DockerDaemon) containers() ?[]Container { + d.send_request('GET', urllib.parse('/v1.41/containers/json')?)? + res_header := d.read_response_head()? + content_length := res_header.header.get(http.CommonHeader.content_length)?.int() + res := d.read_response_body(content_length)? - return json.decode([]Container, res.text) or {} + data := json.decode([]Container, res)? + + return data } pub struct NewContainer { diff --git a/src/docker/docker.v b/src/docker/docker.v index ce01e7e..fa83d89 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -9,8 +9,8 @@ import json // it. fn send(req &string) ?http.Response { // Open a connection to the socket - mut s := unix.connect_stream(docker.socket) or { - return error('Failed to connect to socket ${docker.socket}.') + mut s := unix.connect_stream(socket) or { + return error('Failed to connect to socket ${socket}.') } defer { @@ -21,19 +21,19 @@ fn send(req &string) ?http.Response { } // Write the request to the socket - s.write_string(req) or { return error('Failed to write request to socket ${docker.socket}.') } + s.write_string(req) or { return error('Failed to write request to socket ${socket}.') } s.wait_for_write()? mut c := 0 - mut buf := []u8{len: docker.buf_len} + mut buf := []u8{len: buf_len} mut res := []u8{} for { - c = s.read(mut buf) or { return error('Failed to read data from socket ${docker.socket}.') } + c = s.read(mut buf) or { return error('Failed to read data from socket ${socket}.') } res << buf[..c] - if c < docker.buf_len { + if c < buf_len { break } } @@ -41,7 +41,7 @@ fn send(req &string) ?http.Response { // After reading the first part of the response, we parse it into an HTTP // response. If it isn't chunked, we return early with the data. parsed := http.parse_response(res.bytestr()) or { - return error('Failed to parse HTTP response from socket ${docker.socket}.') + return error('Failed to parse HTTP response from socket ${socket}.') } if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { @@ -55,12 +55,10 @@ fn send(req &string) ?http.Response { s.wait_for_write()? for { - c = s.read(mut buf) or { - return error('Failed to read data from socket ${docker.socket}.') - } + c = s.read(mut buf) or { return error('Failed to read data from socket ${socket}.') } res << buf[..c] - if c < docker.buf_len { + if c < buf_len { break } } diff --git a/src/docker/socket.v b/src/docker/socket.v index 3a72f14..7815435 100644 --- a/src/docker/socket.v +++ b/src/docker/socket.v @@ -3,10 +3,15 @@ module docker import net.unix import io import net.http +import strings +import net.urllib +import json -const socket = '/var/run/docker.sock' - -const buf_len = 10 * 1024 +const ( + socket = '/var/run/docker.sock' + buf_len = 10 * 1024 + http_separator = [u8(`\r`), `\n`, `\r`, `\n`] +) pub struct DockerDaemon { mut: @@ -14,34 +19,84 @@ mut: reader &io.BufferedReader } -pub fn new_conn() ?DockerDaemon { - s := unix.connect_stream(socket) ? +pub fn new_conn() ?&DockerDaemon { + s := unix.connect_stream(docker.socket)? - d := DockerDaemon{socket: s, reader: io.new_buffered_reader(reader: s)} + d := &DockerDaemon{ + socket: s + reader: io.new_buffered_reader(reader: s) + } return d } -fn (mut d DockerDaemon) send_request(req &string) ? { - d.socket.write_string(req) ? - d.socket.wait_for_write() ? +pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' + + d.socket.write_string(req)? +} + +pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL, content_type string, body string) ? { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' + + d.socket.write_string(req)? +} + +pub fn (mut d DockerDaemon) request_with_json(method string, url urllib.URL, data &T) ? { + body := json.encode(data) + + return request_with_body(method, url, 'application/json', body) } // read_response_head consumes the socket's contents until it encounters -// '\n\n', after which it parses the response as an HTTP response. -fn (mut d DockerDaemon) read_response_head() ?http.Response { +// '\r\n\r\n', after which it parses the response as an HTTP response. +pub fn (mut d DockerDaemon) read_response_head() ?http.Response { mut c := 0 - mut buf := [buf_len]u8{len: docker.buf_len} + mut buf := []u8{len: 4} mut res := []u8{} for { - c = d.socket.read(mut buf) ? + c = d.reader.read(mut buf)? res << buf[..c] - if res#[-2..] == [u8(`\n`), `\n`] { + mut i := 0 + mut match_len := 0 + + for i + match_len < c { + if buf[i + match_len] == docker.http_separator[match_len] { + match_len += 1 + } else { + i += match_len + 1 + match_len = 0 + } + } + + if match_len == 4 { break + } else if match_len > 0 { + mut buf2 := []u8{len: 4 - match_len} + c2 := d.reader.read(mut buf2)? + res << buf2[..c2] + + if buf2 == docker.http_separator[match_len..] { + break + } } } return http.parse_response(res.bytestr()) } + +pub fn (mut d DockerDaemon) read_response_body(length int) ?string { + mut buf := []u8{len: docker.buf_len} + mut c := 0 + mut builder := strings.new_builder(docker.buf_len) + + for builder.len < length { + c = d.reader.read(mut buf) or { break } + + builder.write(buf[..c])? + } + + return builder.str() +} diff --git a/src/main.v b/src/main.v index 41d0d33..db6d5ef 100644 --- a/src/main.v +++ b/src/main.v @@ -31,7 +31,7 @@ fn main() { logs.cmd(), ] } - app.setup() app.parse(os.args) + return } From 4c97489f8a7f868e2ea58afbe9fe3fd47477ed02 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 13 May 2022 22:03:06 +0200 Subject: [PATCH 03/17] feat(docker): partially migrate to new code --- src/build/build.v | 11 ++++++---- src/docker/containers.v | 47 ++++++++++++++++++++++++++++++++++++++--- src/docker/socket.v | 12 +++++++++-- src/main.v | 1 + 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 41f68e2..b470853 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -17,6 +17,8 @@ const build_image_repo = 'vieter-build' // makepkg with. The base image should be some Linux distribution that uses // Pacman as its package manager. pub fn create_build_image(base_image string) ?string { + mut dd := docker.new_conn() ? + commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' @@ -48,12 +50,13 @@ pub fn create_build_image(base_image string) ?string { // We pull the provided image docker.pull_image(image_name, image_tag)? - id := docker.create_container(c)? - docker.start_container(id)? + id := dd.create_container(c)?.id + /* id := docker.create_container(c)? */ + dd.start_container(id)? // This loop waits until the container has stopped, so we can remove it after for { - data := docker.inspect_container(id)? + data := dd.inspect_container(id)? if !data.state.running { break @@ -68,7 +71,7 @@ pub fn create_build_image(base_image string) ?string { // conflicts. tag := time.sys_mono_now().str() image := docker.create_image_from_container(id, 'vieter-build', tag)? - docker.remove_container(id)? + dd.remove_container(id)? return image.id } diff --git a/src/docker/containers.v b/src/docker/containers.v index 3b674e6..98116cc 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -12,15 +12,14 @@ struct Container { pub fn (mut d DockerDaemon) containers() ?[]Container { d.send_request('GET', urllib.parse('/v1.41/containers/json')?)? - res_header := d.read_response_head()? - content_length := res_header.header.get(http.CommonHeader.content_length)?.int() - res := d.read_response_body(content_length)? + _, res := d.read_response()? data := json.decode([]Container, res)? return data } +[params] pub struct NewContainer { image string [json: Image] entrypoint []string [json: Entrypoint] @@ -31,7 +30,25 @@ pub struct NewContainer { } struct CreatedContainer { +pub: id string [json: Id] + warnings []string [json: Warnings] +} + +pub fn (mut d DockerDaemon) create_container(c NewContainer) ?CreatedContainer { + d.send_request_with_json('POST', urllib.parse('/v1.41/containers/create')?, c)? + _, res := d.read_response()? + + data := json.decode(CreatedContainer, res)? + + return data +} + +pub fn (mut d DockerDaemon) start_container(id string) ?bool { + d.send_request('POST', urllib.parse('/v1.41/containers/$id/start')?)? + head := d.read_response_head() ? + + return head.status_code == 204 } // create_container creates a container defined by the given configuration. If @@ -72,6 +89,25 @@ pub mut: end_time time.Time [skip] } +pub fn (mut d DockerDaemon) inspect_container(id string) ?ContainerInspect { + d.send_request('GET', urllib.parse('/v1.41/containers/$id/json')?)? + head, body := d.read_response()? + + if head.status_code != 200 { + return error('Failed to inspect container.') + } + + mut data := json.decode(ContainerInspect, body)? + + data.state.start_time = time.parse_rfc3339(data.state.start_time_str)? + + if data.state.status == 'exited' { + data.state.end_time = time.parse_rfc3339(data.state.end_time_str)? + } + + return data +} + // inspect_container returns the result of inspecting a container with a given // ID. pub fn inspect_container(id string) ?ContainerInspect { @@ -92,6 +128,11 @@ pub fn inspect_container(id string) ?ContainerInspect { return data } +pub fn (mut d DockerDaemon) remove_container(id string) ? { + d.send_request('DELETE', urllib.parse('/v1.41/containers/$id')?)? + head := d.read_response_head() ? +} + // remove_container removes a container with a given ID. pub fn remove_container(id string) ?bool { res := request('DELETE', urllib.parse('/v1.41/containers/$id')?)? diff --git a/src/docker/socket.v b/src/docker/socket.v index 7815435..41f9f6d 100644 --- a/src/docker/socket.v +++ b/src/docker/socket.v @@ -42,10 +42,10 @@ pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL d.socket.write_string(req)? } -pub fn (mut d DockerDaemon) request_with_json(method string, url urllib.URL, data &T) ? { +pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib.URL, data &T) ? { body := json.encode(data) - return request_with_body(method, url, 'application/json', body) + return d.send_request_with_body(method, url, 'application/json', body) } // read_response_head consumes the socket's contents until it encounters @@ -100,3 +100,11 @@ pub fn (mut d DockerDaemon) read_response_body(length int) ?string { return builder.str() } + +pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { + head := d.read_response_head()? + content_length := head.header.get(http.CommonHeader.content_length)?.int() + res := d.read_response_body(content_length)? + + return head, res +} diff --git a/src/main.v b/src/main.v index db6d5ef..6b1e7bc 100644 --- a/src/main.v +++ b/src/main.v @@ -7,6 +7,7 @@ import build import console.git import console.logs import cron +import docker fn main() { mut app := cli.Command{ From da46b8b4ae4483fed98b911e0523182ffe2156e8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 13 May 2022 22:15:30 +0200 Subject: [PATCH 04/17] feat(docker): error when HTTP requests fail --- src/build/build.v | 1 + src/docker/containers.v | 42 ++++++++++++++++++++++++++++++++++------- src/docker/socket.v | 6 ++++++ src/docker/stream.v | 9 +++++++++ 4 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 src/docker/stream.v diff --git a/src/build/build.v b/src/build/build.v index b470853..ae10318 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -56,6 +56,7 @@ pub fn create_build_image(base_image string) ?string { // This loop waits until the container has stopped, so we can remove it after for { + println('wot') data := dd.inspect_container(id)? if !data.state.running { diff --git a/src/docker/containers.v b/src/docker/containers.v index 98116cc..e2da938 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -5,6 +5,10 @@ import net.urllib import time import net.http +struct DockerError { + message string +} + struct Container { id string [json: Id] names []string [json: Names] @@ -12,7 +16,13 @@ struct Container { pub fn (mut d DockerDaemon) containers() ?[]Container { d.send_request('GET', urllib.parse('/v1.41/containers/json')?)? - _, res := d.read_response()? + head, res := d.read_response()? + + if head.status_code != 200 { + data := json.decode(DockerError, res)? + + return error(data.message) + } data := json.decode([]Container, res)? @@ -37,18 +47,28 @@ pub: pub fn (mut d DockerDaemon) create_container(c NewContainer) ?CreatedContainer { d.send_request_with_json('POST', urllib.parse('/v1.41/containers/create')?, c)? - _, res := d.read_response()? + head, res := d.read_response()? + + if head.status_code != 201 { + data := json.decode(DockerError, res)? + + return error(data.message) + } data := json.decode(CreatedContainer, res)? return data } -pub fn (mut d DockerDaemon) start_container(id string) ?bool { +pub fn (mut d DockerDaemon) start_container(id string) ? { d.send_request('POST', urllib.parse('/v1.41/containers/$id/start')?)? - head := d.read_response_head() ? + head, body := d.read_response() ? - return head.status_code == 204 + if head.status_code != 204 { + data := json.decode(DockerError, body)? + + return error(data.message) + } } // create_container creates a container defined by the given configuration. If @@ -94,7 +114,9 @@ pub fn (mut d DockerDaemon) inspect_container(id string) ?ContainerInspect { head, body := d.read_response()? if head.status_code != 200 { - return error('Failed to inspect container.') + data := json.decode(DockerError, body)? + + return error(data.message) } mut data := json.decode(ContainerInspect, body)? @@ -130,7 +152,13 @@ pub fn inspect_container(id string) ?ContainerInspect { pub fn (mut d DockerDaemon) remove_container(id string) ? { d.send_request('DELETE', urllib.parse('/v1.41/containers/$id')?)? - head := d.read_response_head() ? + head, body := d.read_response() ? + + if head.status_code != 204 { + data := json.decode(DockerError, body)? + + return error(data.message) + } } // remove_container removes a container with a given ID. diff --git a/src/docker/socket.v b/src/docker/socket.v index 41f9f6d..749f14e 100644 --- a/src/docker/socket.v +++ b/src/docker/socket.v @@ -50,6 +50,8 @@ pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib. // read_response_head consumes the socket's contents until it encounters // '\r\n\r\n', after which it parses the response as an HTTP response. +// Importantly, this function never consumes past the HTTP separator, so the +// body can be read fully later on. pub fn (mut d DockerDaemon) read_response_head() ?http.Response { mut c := 0 mut buf := []u8{len: 4} @@ -88,6 +90,10 @@ pub fn (mut d DockerDaemon) read_response_head() ?http.Response { } pub fn (mut d DockerDaemon) read_response_body(length int) ?string { + if length == 0 { + return '' + } + mut buf := []u8{len: docker.buf_len} mut c := 0 mut builder := strings.new_builder(docker.buf_len) diff --git a/src/docker/stream.v b/src/docker/stream.v new file mode 100644 index 0000000..24c51c1 --- /dev/null +++ b/src/docker/stream.v @@ -0,0 +1,9 @@ +module docker + +import io + +struct ChunkedResponseStream { + reader io.Reader +} + + From 92cbea69d6e4f8929ff5f397b36d7852e7084172 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 14 May 2022 21:55:19 +0200 Subject: [PATCH 05/17] feat(docker): added ChunkedResponseReader implementation --- src/build/build.v | 6 +-- src/docker/containers.v | 6 +-- src/docker/images.v | 26 ++++++++++++ src/docker/socket.v | 35 ++++++++++------ src/docker/stream.v | 92 ++++++++++++++++++++++++++++++++++++++++- src/util/util.v | 16 +++++++ 6 files changed, 161 insertions(+), 20 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index ae10318..dd72c17 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -17,7 +17,7 @@ const build_image_repo = 'vieter-build' // makepkg with. The base image should be some Linux distribution that uses // Pacman as its package manager. pub fn create_build_image(base_image string) ?string { - mut dd := docker.new_conn() ? + mut dd := docker.new_conn()? commands := [ // Update repos & install required packages @@ -51,12 +51,12 @@ pub fn create_build_image(base_image string) ?string { docker.pull_image(image_name, image_tag)? id := dd.create_container(c)?.id - /* id := docker.create_container(c)? */ + // id := docker.create_container(c)? dd.start_container(id)? // This loop waits until the container has stopped, so we can remove it after for { - println('wot') + println('wot') data := dd.inspect_container(id)? if !data.state.running { diff --git a/src/docker/containers.v b/src/docker/containers.v index e2da938..0027747 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -41,7 +41,7 @@ pub struct NewContainer { struct CreatedContainer { pub: - id string [json: Id] + id string [json: Id] warnings []string [json: Warnings] } @@ -62,7 +62,7 @@ pub fn (mut d DockerDaemon) create_container(c NewContainer) ?CreatedContainer { pub fn (mut d DockerDaemon) start_container(id string) ? { d.send_request('POST', urllib.parse('/v1.41/containers/$id/start')?)? - head, body := d.read_response() ? + head, body := d.read_response()? if head.status_code != 204 { data := json.decode(DockerError, body)? @@ -152,7 +152,7 @@ pub fn inspect_container(id string) ?ContainerInspect { pub fn (mut d DockerDaemon) remove_container(id string) ? { d.send_request('DELETE', urllib.parse('/v1.41/containers/$id')?)? - head, body := d.read_response() ? + head, body := d.read_response()? if head.status_code != 204 { data := json.decode(DockerError, body)? diff --git a/src/docker/images.v b/src/docker/images.v index 2e873fa..2249113 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -9,6 +9,32 @@ pub: id string [json: Id] } +pub fn (mut d DockerDaemon) pull_image(image string, tag string) ? { + d.send_request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)? + head := d.read_response_head()? + + if head.status_code != 200 { + content_length := head.header.get(http.CommonHeader.content_length)?.int() + body := d.read_response_body(content_length)? + data := json.decode(DockerError, body)? + + return error(data.message) + } + + mut body := d.get_chunked_response_reader() + + mut buf := []u8{len: 1024} + + for { + c := body.read(mut buf)? + + if c == 0 { + break + } + print(buf[..c].bytestr()) + } +} + // pull_image pulls tries to pull the image for the given image & tag pub fn pull_image(image string, tag string) ?http.Response { return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?) diff --git a/src/docker/socket.v b/src/docker/socket.v index 749f14e..444d9a9 100644 --- a/src/docker/socket.v +++ b/src/docker/socket.v @@ -6,11 +6,13 @@ import net.http import strings import net.urllib import json +import util const ( - socket = '/var/run/docker.sock' - buf_len = 10 * 1024 - http_separator = [u8(`\r`), `\n`, `\r`, `\n`] + socket = '/var/run/docker.sock' + buf_len = 10 * 1024 + http_separator = [u8(`\r`), `\n`, `\r`, `\n`] + http_chunk_separator = [u8(`\r`), `\n`] ) pub struct DockerDaemon { @@ -61,17 +63,18 @@ pub fn (mut d DockerDaemon) read_response_head() ?http.Response { c = d.reader.read(mut buf)? res << buf[..c] - mut i := 0 - mut match_len := 0 + match_len := util.match_array_in_array(buf[..c], docker.http_separator) + // mut i := 0 + // mut match_len := 0 - for i + match_len < c { - if buf[i + match_len] == docker.http_separator[match_len] { - match_len += 1 - } else { - i += match_len + 1 - match_len = 0 - } - } + // for i + match_len < c { + // if buf[i + match_len] == docker.http_separator[match_len] { + // match_len += 1 + // } else { + // i += match_len + 1 + // match_len = 0 + // } + //} if match_len == 4 { break @@ -114,3 +117,9 @@ pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { return head, res } + +pub fn (mut d DockerDaemon) get_chunked_response_reader() &ChunkedResponseReader { + r := new_chunked_response_reader(d.reader) + + return r +} diff --git a/src/docker/stream.v b/src/docker/stream.v index 24c51c1..fbc48ba 100644 --- a/src/docker/stream.v +++ b/src/docker/stream.v @@ -1,9 +1,99 @@ module docker import io +import util +import encoding.binary +import encoding.hex -struct ChunkedResponseStream { +struct ChunkedResponseReader { +mut: reader io.Reader + // buf []u8 + // offset int + // len int + bytes_left_in_chunk u64 + end_of_stream bool + started bool } +pub fn new_chunked_response_reader(reader io.Reader) &ChunkedResponseReader { + r := &ChunkedResponseReader{ + reader: reader + } + return r +} + +// We satisfy the io.Reader interface +pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { + if r.end_of_stream { + return none + } + + if r.bytes_left_in_chunk == 0 { + r.bytes_left_in_chunk = r.read_chunk_size()? + + if r.end_of_stream { + return none + } + } + + mut c := 0 + + if buf.len > r.bytes_left_in_chunk { + c = r.reader.read(mut buf[..r.bytes_left_in_chunk])? + } else { + c = r.reader.read(mut buf)? + } + + r.bytes_left_in_chunk -= u64(c) + + return c +} + +fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { + mut buf := []u8{len: 2} + mut res := []u8{} + + if r.started { + // Each chunk ends with a `\r\n` which we want to skip first + r.reader.read(mut buf) ? + } + + r.started = true + + for { + c := r.reader.read(mut buf)? + res << buf[..c] + + match_len := util.match_array_in_array(buf[..c], http_chunk_separator) + + if match_len == http_chunk_separator.len { + break + } + + if match_len > 0 { + mut buf2 := []u8{len: 2 - match_len} + c2 := r.reader.read(mut buf2)? + res << buf2[..c2] + + if buf2 == http_chunk_separator[match_len..] { + break + } + } + } + + mut num_data := hex.decode(res#[..-2].bytestr())? + + for num_data.len < 8 { + num_data.insert(0, 0) + } + + num := binary.big_endian_u64(num_data) + + if num == 0 { + r.end_of_stream = true + } + + return num +} diff --git a/src/util/util.v b/src/util/util.v index 6602621..f9f58ed 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -92,3 +92,19 @@ pub fn pretty_bytes(bytes int) string { return '${n:.2}${util.prefixes[i]}' } + +pub fn match_array_in_array(a1 []T, a2 []T) int { + mut i := 0 + mut match_len := 0 + + for i + match_len < a1.len { + if a1[i + match_len] == a2[match_len] { + match_len += 1 + } else { + i += match_len + 1 + match_len = 0 + } + } + + return match_len +} From f22ed29631a3e5d09b9496bb6bba255d4b5dc400 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 14 May 2022 22:39:52 +0200 Subject: [PATCH 06/17] feat(docker): added StreamFormatReader --- src/docker/containers.v | 15 +++++++++ src/docker/socket.v | 7 ++++ src/docker/stream.v | 73 ++++++++++++++++++++++++++++++++++------- 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/src/docker/containers.v b/src/docker/containers.v index 0027747..ff48c69 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -168,6 +168,21 @@ pub fn remove_container(id string) ?bool { return res.status_code == 204 } +pub fn (mut d DockerDaemon) get_container_logs(id string) ?&StreamFormatReader { + d.send_request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)? + head := d.read_response_head()? + + if head.status_code != 200 { + content_length := head.header.get(http.CommonHeader.content_length)?.int() + body := d.read_response_body(content_length)? + data := json.decode(DockerError, body)? + + return error(data.message) + } + + return d.get_stream_format_reader() +} + // get_container_logs retrieves the logs for a Docker container, both stdout & // stderr. pub fn get_container_logs(id string) ?string { diff --git a/src/docker/socket.v b/src/docker/socket.v index 444d9a9..f6dfeb1 100644 --- a/src/docker/socket.v +++ b/src/docker/socket.v @@ -123,3 +123,10 @@ pub fn (mut d DockerDaemon) get_chunked_response_reader() &ChunkedResponseReader return r } + +pub fn (mut d DockerDaemon) get_stream_format_reader() &StreamFormatReader { + r := new_chunked_response_reader(d.reader) + r2 := new_stream_format_reader(r) + + return r2 +} diff --git a/src/docker/stream.v b/src/docker/stream.v index fbc48ba..8822fd3 100644 --- a/src/docker/stream.v +++ b/src/docker/stream.v @@ -7,13 +7,10 @@ import encoding.hex struct ChunkedResponseReader { mut: - reader io.Reader - // buf []u8 - // offset int - // len int + reader io.Reader bytes_left_in_chunk u64 end_of_stream bool - started bool + started bool } pub fn new_chunked_response_reader(reader io.Reader) &ChunkedResponseReader { @@ -39,7 +36,7 @@ pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { } mut c := 0 - + if buf.len > r.bytes_left_in_chunk { c = r.reader.read(mut buf[..r.bytes_left_in_chunk])? } else { @@ -56,8 +53,8 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { mut res := []u8{} if r.started { - // Each chunk ends with a `\r\n` which we want to skip first - r.reader.read(mut buf) ? + // Each chunk ends with a `\r\n` which we want to skip first + r.reader.read(mut buf)? } r.started = true @@ -85,9 +82,9 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { mut num_data := hex.decode(res#[..-2].bytestr())? - for num_data.len < 8 { - num_data.insert(0, 0) - } + for num_data.len < 8 { + num_data.insert(0, 0) + } num := binary.big_endian_u64(num_data) @@ -97,3 +94,57 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { return num } + +struct StreamFormatReader { + stdout bool + stderr bool + stdin bool +mut: + reader io.Reader + bytes_left_in_chunk u32 + end_of_stream bool +} + +pub fn new_stream_format_reader(reader io.Reader) &StreamFormatReader { + r := &StreamFormatReader{ + reader: reader + } + + return r +} + +pub fn (mut r StreamFormatReader) read(mut buf []u8) ?int { + if r.end_of_stream { + return none + } + + if r.bytes_left_in_chunk == 0 { + r.bytes_left_in_chunk = r.read_chunk_size()? + + if r.end_of_stream { + return none + } + } + + mut c := 0 + + if buf.len > r.bytes_left_in_chunk { + c = r.reader.read(mut buf[..r.bytes_left_in_chunk])? + } else { + c = r.reader.read(mut buf)? + } + + r.bytes_left_in_chunk -= u32(c) + + return c +} + +fn (mut r StreamFormatReader) read_chunk_size() ?u32 { + mut buf := []u8{len: 8} + + r.reader.read(mut buf)? + + num := binary.big_endian_u32(buf[4..]) + + return num +} From 1811ebbe3f3549b1034274a6a74675af4ef8469f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 14 May 2022 23:42:20 +0200 Subject: [PATCH 07/17] doc: documented new Docker code --- src/docker/containers.v | 7 +++++++ src/docker/images.v | 1 + src/docker/socket.v | 33 +++++++++++++++++++-------------- src/docker/stream.v | 24 ++++++++++++++++++++---- src/util/util.v | 3 +++ 5 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/docker/containers.v b/src/docker/containers.v index ff48c69..05c9cc7 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -14,6 +14,7 @@ struct Container { names []string [json: Names] } +// containers returns a list of all containers. pub fn (mut d DockerDaemon) containers() ?[]Container { d.send_request('GET', urllib.parse('/v1.41/containers/json')?)? head, res := d.read_response()? @@ -45,6 +46,7 @@ pub: warnings []string [json: Warnings] } +// create_container creates a new container with the given config. pub fn (mut d DockerDaemon) create_container(c NewContainer) ?CreatedContainer { d.send_request_with_json('POST', urllib.parse('/v1.41/containers/create')?, c)? head, res := d.read_response()? @@ -60,6 +62,7 @@ pub fn (mut d DockerDaemon) create_container(c NewContainer) ?CreatedContainer { return data } +// start_container starts the container with the given id. pub fn (mut d DockerDaemon) start_container(id string) ? { d.send_request('POST', urllib.parse('/v1.41/containers/$id/start')?)? head, body := d.read_response()? @@ -109,6 +112,7 @@ pub mut: end_time time.Time [skip] } +// inspect_container returns detailed information for a given container. pub fn (mut d DockerDaemon) inspect_container(id string) ?ContainerInspect { d.send_request('GET', urllib.parse('/v1.41/containers/$id/json')?)? head, body := d.read_response()? @@ -150,6 +154,7 @@ pub fn inspect_container(id string) ?ContainerInspect { return data } +// remove_container removes the container with the given id. pub fn (mut d DockerDaemon) remove_container(id string) ? { d.send_request('DELETE', urllib.parse('/v1.41/containers/$id')?)? head, body := d.read_response()? @@ -168,6 +173,8 @@ pub fn remove_container(id string) ?bool { return res.status_code == 204 } +// get_container_logs returns a reader object allowing access to the +// container's logs. pub fn (mut d DockerDaemon) get_container_logs(id string) ?&StreamFormatReader { d.send_request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)? head := d.read_response_head()? diff --git a/src/docker/images.v b/src/docker/images.v index 2249113..974b408 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -9,6 +9,7 @@ pub: id string [json: Id] } +// pull_image pulls the given image:tag. pub fn (mut d DockerDaemon) pull_image(image string, tag string) ? { d.send_request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)? head := d.read_response_head()? diff --git a/src/docker/socket.v b/src/docker/socket.v index f6dfeb1..b1e3080 100644 --- a/src/docker/socket.v +++ b/src/docker/socket.v @@ -21,6 +21,7 @@ mut: reader &io.BufferedReader } +// new_conn creates a new connection to the Docker daemon. pub fn new_conn() ?&DockerDaemon { s := unix.connect_stream(docker.socket)? @@ -32,18 +33,22 @@ pub fn new_conn() ?&DockerDaemon { return d } +// send_request sends an HTTP request without body. pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' d.socket.write_string(req)? } +// send_request_with_body sends an HTTP request with the given body. pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL, content_type string, body string) ? { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' d.socket.write_string(req)? } +// send_request_with_json is a convenience wrapper around +// send_request_with_body that encodes the input as JSON. pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib.URL, data &T) ? { body := json.encode(data) @@ -52,8 +57,8 @@ pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib. // read_response_head consumes the socket's contents until it encounters // '\r\n\r\n', after which it parses the response as an HTTP response. -// Importantly, this function never consumes past the HTTP separator, so the -// body can be read fully later on. +// Importantly, this function never consumes the reader past the HTTP +// separator, so the body can be read fully later on. pub fn (mut d DockerDaemon) read_response_head() ?http.Response { mut c := 0 mut buf := []u8{len: 4} @@ -64,21 +69,12 @@ pub fn (mut d DockerDaemon) read_response_head() ?http.Response { res << buf[..c] match_len := util.match_array_in_array(buf[..c], docker.http_separator) - // mut i := 0 - // mut match_len := 0 - - // for i + match_len < c { - // if buf[i + match_len] == docker.http_separator[match_len] { - // match_len += 1 - // } else { - // i += match_len + 1 - // match_len = 0 - // } - //} if match_len == 4 { break - } else if match_len > 0 { + } + + if match_len > 0 { mut buf2 := []u8{len: 4 - match_len} c2 := d.reader.read(mut buf2)? res << buf2[..c2] @@ -92,6 +88,8 @@ pub fn (mut d DockerDaemon) read_response_head() ?http.Response { return http.parse_response(res.bytestr()) } +// read_response_body reads `length` bytes from the stream. It can be used when +// the response encoding isn't chunked to fully read it. pub fn (mut d DockerDaemon) read_response_body(length int) ?string { if length == 0 { return '' @@ -110,6 +108,9 @@ pub fn (mut d DockerDaemon) read_response_body(length int) ?string { return builder.str() } +// read_response is a convenience function combining read_response_head & +// read_response_body. It can be used when you know for certain the response +// won't be chunked. pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { head := d.read_response_head()? content_length := head.header.get(http.CommonHeader.content_length)?.int() @@ -118,12 +119,16 @@ pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { return head, res } +// get_chunked_response_reader returns a ChunkedResponseReader using the socket +// as reader. pub fn (mut d DockerDaemon) get_chunked_response_reader() &ChunkedResponseReader { r := new_chunked_response_reader(d.reader) return r } +// get_stream_format_reader returns a StreamFormatReader using the socket as +// reader. pub fn (mut d DockerDaemon) get_stream_format_reader() &StreamFormatReader { r := new_chunked_response_reader(d.reader) r2 := new_stream_format_reader(r) diff --git a/src/docker/stream.v b/src/docker/stream.v index 8822fd3..f20fe18 100644 --- a/src/docker/stream.v +++ b/src/docker/stream.v @@ -5,6 +5,8 @@ import util import encoding.binary import encoding.hex +// ChunkedResponseReader parses an underlying HTTP chunked response, exposing +// it as if it was a continuous stream of data. struct ChunkedResponseReader { mut: reader io.Reader @@ -13,6 +15,8 @@ mut: started bool } +// new_chunked_response_reader creates a new ChunkedResponseReader on the heap +// with the provided reader. pub fn new_chunked_response_reader(reader io.Reader) &ChunkedResponseReader { r := &ChunkedResponseReader{ reader: reader @@ -21,7 +25,7 @@ pub fn new_chunked_response_reader(reader io.Reader) &ChunkedResponseReader { return r } -// We satisfy the io.Reader interface +// read satisfies the io.Reader interface. pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { if r.end_of_stream { return none @@ -37,6 +41,8 @@ pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { mut c := 0 + // Make sure we don't read more than we can safely read. This is to avoid + // the underlying reader from becoming out of sync with our parsing: if buf.len > r.bytes_left_in_chunk { c = r.reader.read(mut buf[..r.bytes_left_in_chunk])? } else { @@ -48,6 +54,9 @@ pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { return c } +// read_chunk_size advances the reader & reads the size of the next HTTP chunk. +// This function should only be called if the previous chunk has been +// completely consumed. fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { mut buf := []u8{len: 2} mut res := []u8{} @@ -80,6 +89,7 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { } } + // The length of the next chunk is provided as a hexadecimal mut num_data := hex.decode(res#[..-2].bytestr())? for num_data.len < 8 { @@ -88,6 +98,8 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { num := binary.big_endian_u64(num_data) + // This only occurs for the very last chunk, which always reports a size of + // 0. if num == 0 { r.end_of_stream = true } @@ -95,16 +107,17 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { return num } +// StreamFormatReader parses an underlying stream of Docker logs, removing the +// header bytes. struct StreamFormatReader { - stdout bool - stderr bool - stdin bool mut: reader io.Reader bytes_left_in_chunk u32 end_of_stream bool } +// new_stream_format_reader creates a new StreamFormatReader using the given +// reader. pub fn new_stream_format_reader(reader io.Reader) &StreamFormatReader { r := &StreamFormatReader{ reader: reader @@ -113,6 +126,7 @@ pub fn new_stream_format_reader(reader io.Reader) &StreamFormatReader { return r } +// read satisfies the io.Reader interface. pub fn (mut r StreamFormatReader) read(mut buf []u8) ?int { if r.end_of_stream { return none @@ -139,6 +153,8 @@ pub fn (mut r StreamFormatReader) read(mut buf []u8) ?int { return c } +// read_chunk_size advances the reader & reads the header bytes for the length +// of the next chunk. fn (mut r StreamFormatReader) read_chunk_size() ?u32 { mut buf := []u8{len: 8} diff --git a/src/util/util.v b/src/util/util.v index f9f58ed..f805e6d 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -93,6 +93,9 @@ pub fn pretty_bytes(bytes int) string { return '${n:.2}${util.prefixes[i]}' } +// match_array_in_array returns how many elements of a2 overlap with a1. For +// example, if a1 = "abcd" & a2 = "cd", the result will be 2. If the match is +// not at the end of a1, the result is 0. pub fn match_array_in_array(a1 []T, a2 []T) int { mut i := 0 mut match_len := 0 From 79fd9c1f8741f15cda9cb03200fa37576b047e91 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 15 May 2022 00:23:52 +0200 Subject: [PATCH 08/17] fix(docker): read_response now handles chunked data --- src/build/build.v | 1 - src/docker/images.v | 5 +---- src/docker/socket.v | 22 +++++++++++++++++++--- src/util/util.v | 11 +++++++++++ 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index dd72c17..98c56f5 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -56,7 +56,6 @@ pub fn create_build_image(base_image string) ?string { // This loop waits until the container has stopped, so we can remove it after for { - println('wot') data := dd.inspect_container(id)? if !data.state.running { diff --git a/src/docker/images.v b/src/docker/images.v index 974b408..ab735f2 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -27,11 +27,8 @@ pub fn (mut d DockerDaemon) pull_image(image string, tag string) ? { mut buf := []u8{len: 1024} for { - c := body.read(mut buf)? + c := body.read(mut buf) or { break } - if c == 0 { - break - } print(buf[..c].bytestr()) } } diff --git a/src/docker/socket.v b/src/docker/socket.v index b1e3080..dfa7ea7 100644 --- a/src/docker/socket.v +++ b/src/docker/socket.v @@ -38,6 +38,9 @@ pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' d.socket.write_string(req)? + + // When starting a new request, the reader needs to be reset. + d.reader = io.new_buffered_reader(reader: d.socket) } // send_request_with_body sends an HTTP request with the given body. @@ -45,6 +48,9 @@ pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' d.socket.write_string(req)? + + // When starting a new request, the reader needs to be reset. + d.reader = io.new_buffered_reader(reader: d.socket) } // send_request_with_json is a convenience wrapper around @@ -108,11 +114,21 @@ pub fn (mut d DockerDaemon) read_response_body(length int) ?string { return builder.str() } -// read_response is a convenience function combining read_response_head & -// read_response_body. It can be used when you know for certain the response -// won't be chunked. +// read_response is a convenience function which always consumes the entire +// response & returns it. It should only be used when we're certain that the +// result isn't too large. pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { head := d.read_response_head()? + + if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' { + mut builder := strings.new_builder(1024) + mut body := d.get_chunked_response_reader() + + util.reader_to_writer(mut body, mut builder) ? + + return head, builder.str() + } + content_length := head.header.get(http.CommonHeader.content_length)?.int() res := d.read_response_body(content_length)? diff --git a/src/util/util.v b/src/util/util.v index f805e6d..7aabc1b 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -23,6 +23,17 @@ pub fn exit_with_message(code int, msg string) { exit(code) } +// reader_to_writer tries to consume the entire reader & write it to the writer. +pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { + mut buf := []u8{len: 10 * 1024} + + for { + c := reader.read(mut buf) or { break } + + writer.write(buf) or { break } + } +} + // reader_to_file writes the contents of a BufferedReader to a file pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { mut file := os.create(path)? From e041682feae9d058d20e44dba58e185ba0310443 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 15 May 2022 09:56:23 +0200 Subject: [PATCH 09/17] feat(docker): fully migrate build commands to new code --- src/build/build.v | 43 +++++++++++++++++++++++++++++++++---------- src/docker/images.v | 33 ++++++++++++++++++++++++++++++--- src/docker/socket.v | 7 ++++++- src/main.v | 1 - src/util/util.v | 2 +- 5 files changed, 70 insertions(+), 16 deletions(-) diff --git a/src/build/build.v b/src/build/build.v index 98c56f5..0de91a6 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -6,6 +6,8 @@ import time import os import db import client +import strings +import util const container_build_dir = '/build' @@ -19,6 +21,10 @@ const build_image_repo = 'vieter-build' pub fn create_build_image(base_image string) ?string { mut dd := docker.new_conn()? + defer { + dd.close() or {} + } + commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' @@ -48,7 +54,7 @@ pub fn create_build_image(base_image string) ?string { image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' } // We pull the provided image - docker.pull_image(image_name, image_tag)? + dd.pull_image(image_name, image_tag)? id := dd.create_container(c)?.id // id := docker.create_container(c)? @@ -70,7 +76,7 @@ pub fn create_build_image(base_image string) ?string { // TODO also add the base image's name into the image name to prevent // conflicts. tag := time.sys_mono_now().str() - image := docker.create_image_from_container(id, 'vieter-build', tag)? + image := dd.create_image_from_container(id, 'vieter-build', tag)? dd.remove_container(id)? return image.id @@ -88,6 +94,12 @@ pub: // provided GitRepo. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult { + mut dd := docker.new_conn()? + + defer { + dd.close() or {} + } + build_arch := os.uname().machine // TODO what to do with PKGBUILDs that build multiple packages? @@ -115,27 +127,31 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db user: 'builder:builder' } - id := docker.create_container(c)? - docker.start_container(id)? + id := dd.create_container(c)?.id + dd.start_container(id)? - mut data := docker.inspect_container(id)? + mut data := dd.inspect_container(id)? // This loop waits until the container has stopped, so we can remove it after for data.state.running { time.sleep(1 * time.second) - data = docker.inspect_container(id)? + data = dd.inspect_container(id)? } - logs := docker.get_container_logs(id)? + mut logs_stream := dd.get_container_logs(id)? - docker.remove_container(id)? + // Read in the entire stream + mut logs_builder := strings.new_builder(10 * 1024) + util.reader_to_writer(mut logs_stream, mut logs_builder)? + + dd.remove_container(id)? return BuildResult{ start_time: data.state.start_time end_time: data.state.end_time exit_code: data.state.exit_code - logs: logs + logs: logs_builder.str() } } @@ -153,7 +169,14 @@ fn build(conf Config, repo_id int) ? { res := build_repo(conf.address, conf.api_key, image_id, repo)? println('Removing build image...') - docker.remove_image(image_id)? + + mut dd := docker.new_conn()? + + defer { + dd.close() or {} + } + + dd.remove_image(image_id)? println('Uploading logs to Vieter...') c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, diff --git a/src/docker/images.v b/src/docker/images.v index ab735f2..51620af 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -22,14 +22,13 @@ pub fn (mut d DockerDaemon) pull_image(image string, tag string) ? { return error(data.message) } + // Keep reading the body until the pull has completed mut body := d.get_chunked_response_reader() mut buf := []u8{len: 1024} for { - c := body.read(mut buf) or { break } - - print(buf[..c].bytestr()) + body.read(mut buf) or { break } } } @@ -38,6 +37,22 @@ pub fn pull_image(image string, tag string) ?http.Response { return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?) } +// create_image_from_container creates a new image from a container. +pub fn (mut d DockerDaemon) create_image_from_container(id string, repo string, tag string) ?Image { + d.send_request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? + head, body := d.read_response()? + + if head.status_code != 201 { + data := json.decode(DockerError, body)? + + return error(data.message) + } + + data := json.decode(Image, body)? + + return data +} + // create_image_from_container creates a new image from a container with the // given repo & tag, given the container's ID. pub fn create_image_from_container(id string, repo string, tag string) ?Image { @@ -56,3 +71,15 @@ pub fn remove_image(id string) ?bool { return res.status_code == 200 } + +// remove_image removes the image with the given id. +pub fn (mut d DockerDaemon) remove_image(id string) ? { + d.send_request('DELETE', urllib.parse('/v1.41/images/$id')?)? + head, body := d.read_response()? + + if head.status_code != 200 { + data := json.decode(DockerError, body)? + + return error(data.message) + } +} diff --git a/src/docker/socket.v b/src/docker/socket.v index dfa7ea7..2ee9bff 100644 --- a/src/docker/socket.v +++ b/src/docker/socket.v @@ -33,6 +33,11 @@ pub fn new_conn() ?&DockerDaemon { return d } +// close closes the underlying socket connection. +pub fn (mut d DockerDaemon) close() ? { + d.socket.close()? +} + // send_request sends an HTTP request without body. pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' @@ -124,7 +129,7 @@ pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { mut builder := strings.new_builder(1024) mut body := d.get_chunked_response_reader() - util.reader_to_writer(mut body, mut builder) ? + util.reader_to_writer(mut body, mut builder)? return head, builder.str() } diff --git a/src/main.v b/src/main.v index 6b1e7bc..db6d5ef 100644 --- a/src/main.v +++ b/src/main.v @@ -7,7 +7,6 @@ import build import console.git import console.logs import cron -import docker fn main() { mut app := cli.Command{ diff --git a/src/util/util.v b/src/util/util.v index 7aabc1b..324fb3d 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -28,7 +28,7 @@ pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { mut buf := []u8{len: 10 * 1024} for { - c := reader.read(mut buf) or { break } + reader.read(mut buf) or { break } writer.write(buf) or { break } } From ce67208fbdda00d226c26358d5c42a1833c232f6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 15 May 2022 10:01:12 +0200 Subject: [PATCH 10/17] refactor(docker): remove old code --- src/cron/daemon/daemon.v | 13 ++- src/docker/containers.v | 70 ---------------- src/docker/docker.v | 175 +++++++++++++++++++++++++++------------ src/docker/images.v | 24 ------ src/docker/socket.v | 158 ----------------------------------- 5 files changed, 131 insertions(+), 309 deletions(-) delete mode 100644 src/docker/socket.v diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index ade8fcb..da3b46e 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -253,14 +253,21 @@ fn (mut d Daemon) rebuild_base_image() bool { fn (mut d Daemon) clean_old_base_images() { mut i := 0 + mut dd := docker.new_conn() or { + d.lerror('Failed to connect to Docker socket.') + return + } + + defer { + dd.close() or {} + } + for i < d.builder_images.len - 1 { // For each builder image, we try to remove it by calling the Docker // API. If the function returns an error or false, that means the image // wasn't deleted. Therefore, we move the index over. If the function // returns true, the array's length has decreased by one so we don't // move the index. - if !docker.remove_image(d.builder_images[i]) or { false } { - i += 1 - } + dd.remove_image(d.builder_images[i]) or { i += 1 } } } diff --git a/src/docker/containers.v b/src/docker/containers.v index 05c9cc7..7605452 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -30,7 +30,6 @@ pub fn (mut d DockerDaemon) containers() ?[]Container { return data } -[params] pub struct NewContainer { image string [json: Image] entrypoint []string [json: Entrypoint] @@ -74,26 +73,6 @@ pub fn (mut d DockerDaemon) start_container(id string) ? { } } -// create_container creates a container defined by the given configuration. If -// successful, it returns the ID of the newly created container. -pub fn create_container(c &NewContainer) ?string { - res := request_with_json('POST', urllib.parse('/v1.41/containers/create')?, c)? - - if res.status_code != 201 { - return error('Failed to create container.') - } - - return json.decode(CreatedContainer, res.text)?.id -} - -// start_container starts a container with a given ID. It returns whether the -// container was started or not. -pub fn start_container(id string) ?bool { - res := request('POST', urllib.parse('/v1.41/containers/$id/start')?)? - - return res.status_code == 204 -} - struct ContainerInspect { pub mut: state ContainerState [json: State] @@ -134,26 +113,6 @@ pub fn (mut d DockerDaemon) inspect_container(id string) ?ContainerInspect { return data } -// inspect_container returns the result of inspecting a container with a given -// ID. -pub fn inspect_container(id string) ?ContainerInspect { - res := request('GET', urllib.parse('/v1.41/containers/$id/json')?)? - - if res.status_code != 200 { - return error('Failed to inspect container.') - } - - mut data := json.decode(ContainerInspect, res.text)? - - data.state.start_time = time.parse_rfc3339(data.state.start_time_str)? - - if data.state.status == 'exited' { - data.state.end_time = time.parse_rfc3339(data.state.end_time_str)? - } - - return data -} - // remove_container removes the container with the given id. pub fn (mut d DockerDaemon) remove_container(id string) ? { d.send_request('DELETE', urllib.parse('/v1.41/containers/$id')?)? @@ -166,13 +125,6 @@ pub fn (mut d DockerDaemon) remove_container(id string) ? { } } -// remove_container removes a container with a given ID. -pub fn remove_container(id string) ?bool { - res := request('DELETE', urllib.parse('/v1.41/containers/$id')?)? - - return res.status_code == 204 -} - // get_container_logs returns a reader object allowing access to the // container's logs. pub fn (mut d DockerDaemon) get_container_logs(id string) ?&StreamFormatReader { @@ -189,25 +141,3 @@ pub fn (mut d DockerDaemon) get_container_logs(id string) ?&StreamFormatReader { return d.get_stream_format_reader() } - -// get_container_logs retrieves the logs for a Docker container, both stdout & -// stderr. -pub fn get_container_logs(id string) ?string { - res := request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)? - mut res_bytes := res.text.bytes() - - // Docker uses a special "stream" format for their logs, so we have to - // clean up the data. - mut index := 0 - - for index < res_bytes.len { - // The reverse is required because V reads in the bytes differently - t := res_bytes[index + 4..index + 8].reverse() - len_length := unsafe { *(&u32(&t[0])) } - - res_bytes.delete_many(index, 8) - index += int(len_length) - } - - return res_bytes.bytestr() -} diff --git a/src/docker/docker.v b/src/docker/docker.v index fa83d89..2ee9bff 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -1,91 +1,158 @@ module docker import net.unix -import net.urllib +import io import net.http +import strings +import net.urllib import json +import util -// send writes a request to the Docker socket, waits for a response & returns -// it. -fn send(req &string) ?http.Response { - // Open a connection to the socket - mut s := unix.connect_stream(socket) or { - return error('Failed to connect to socket ${socket}.') +const ( + socket = '/var/run/docker.sock' + buf_len = 10 * 1024 + http_separator = [u8(`\r`), `\n`, `\r`, `\n`] + http_chunk_separator = [u8(`\r`), `\n`] +) + +pub struct DockerDaemon { +mut: + socket &unix.StreamConn + reader &io.BufferedReader +} + +// new_conn creates a new connection to the Docker daemon. +pub fn new_conn() ?&DockerDaemon { + s := unix.connect_stream(docker.socket)? + + d := &DockerDaemon{ + socket: s + reader: io.new_buffered_reader(reader: s) } - defer { - // This or is required because otherwise, the V compiler segfaults for - // some reason - // https://github.com/vlang/v/issues/13534 - s.close() or {} - } + return d +} - // Write the request to the socket - s.write_string(req) or { return error('Failed to write request to socket ${socket}.') } +// close closes the underlying socket connection. +pub fn (mut d DockerDaemon) close() ? { + d.socket.close()? +} - s.wait_for_write()? +// send_request sends an HTTP request without body. +pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' + d.socket.write_string(req)? + + // When starting a new request, the reader needs to be reset. + d.reader = io.new_buffered_reader(reader: d.socket) +} + +// send_request_with_body sends an HTTP request with the given body. +pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL, content_type string, body string) ? { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' + + d.socket.write_string(req)? + + // When starting a new request, the reader needs to be reset. + d.reader = io.new_buffered_reader(reader: d.socket) +} + +// send_request_with_json is a convenience wrapper around +// send_request_with_body that encodes the input as JSON. +pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib.URL, data &T) ? { + body := json.encode(data) + + return d.send_request_with_body(method, url, 'application/json', body) +} + +// read_response_head consumes the socket's contents until it encounters +// '\r\n\r\n', after which it parses the response as an HTTP response. +// Importantly, this function never consumes the reader past the HTTP +// separator, so the body can be read fully later on. +pub fn (mut d DockerDaemon) read_response_head() ?http.Response { mut c := 0 - mut buf := []u8{len: buf_len} + mut buf := []u8{len: 4} mut res := []u8{} for { - c = s.read(mut buf) or { return error('Failed to read data from socket ${socket}.') } + c = d.reader.read(mut buf)? res << buf[..c] - if c < buf_len { + match_len := util.match_array_in_array(buf[..c], docker.http_separator) + + if match_len == 4 { break } - } - // After reading the first part of the response, we parse it into an HTTP - // response. If it isn't chunked, we return early with the data. - parsed := http.parse_response(res.bytestr()) or { - return error('Failed to parse HTTP response from socket ${socket}.') - } + if match_len > 0 { + mut buf2 := []u8{len: 4 - match_len} + c2 := d.reader.read(mut buf2)? + res << buf2[..c2] - if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { - return parsed - } - - // We loop until we've encountered the end of the chunked response - // A chunked HTTP response always ends with '0\r\n\r\n'. - for res.len < 5 || res#[-5..] != [u8(`0`), `\r`, `\n`, `\r`, `\n`] { - // Wait for the server to respond - s.wait_for_write()? - - for { - c = s.read(mut buf) or { return error('Failed to read data from socket ${socket}.') } - res << buf[..c] - - if c < buf_len { + if buf2 == docker.http_separator[match_len..] { break } } } - // Decode chunked response return http.parse_response(res.bytestr()) } -// request_with_body sends a request to the Docker socket with the given body. -fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' +// read_response_body reads `length` bytes from the stream. It can be used when +// the response encoding isn't chunked to fully read it. +pub fn (mut d DockerDaemon) read_response_body(length int) ?string { + if length == 0 { + return '' + } - return send(req) + mut buf := []u8{len: docker.buf_len} + mut c := 0 + mut builder := strings.new_builder(docker.buf_len) + + for builder.len < length { + c = d.reader.read(mut buf) or { break } + + builder.write(buf[..c])? + } + + return builder.str() } -// request sends a request to the Docker socket with an empty body. -fn request(method string, url urllib.URL) ?http.Response { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' +// read_response is a convenience function which always consumes the entire +// response & returns it. It should only be used when we're certain that the +// result isn't too large. +pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { + head := d.read_response_head()? - return send(req) + if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' { + mut builder := strings.new_builder(1024) + mut body := d.get_chunked_response_reader() + + util.reader_to_writer(mut body, mut builder)? + + return head, builder.str() + } + + content_length := head.header.get(http.CommonHeader.content_length)?.int() + res := d.read_response_body(content_length)? + + return head, res } -// request_with_json sends a request to the Docker socket with a given JSON -// payload -pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { - body := json.encode(data) +// get_chunked_response_reader returns a ChunkedResponseReader using the socket +// as reader. +pub fn (mut d DockerDaemon) get_chunked_response_reader() &ChunkedResponseReader { + r := new_chunked_response_reader(d.reader) - return request_with_body(method, url, 'application/json', body) + return r +} + +// get_stream_format_reader returns a StreamFormatReader using the socket as +// reader. +pub fn (mut d DockerDaemon) get_stream_format_reader() &StreamFormatReader { + r := new_chunked_response_reader(d.reader) + r2 := new_stream_format_reader(r) + + return r2 } diff --git a/src/docker/images.v b/src/docker/images.v index 51620af..5866905 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -32,11 +32,6 @@ pub fn (mut d DockerDaemon) pull_image(image string, tag string) ? { } } -// pull_image pulls tries to pull the image for the given image & tag -pub fn pull_image(image string, tag string) ?http.Response { - return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?) -} - // create_image_from_container creates a new image from a container. pub fn (mut d DockerDaemon) create_image_from_container(id string, repo string, tag string) ?Image { d.send_request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? @@ -53,25 +48,6 @@ pub fn (mut d DockerDaemon) create_image_from_container(id string, repo string, return data } -// create_image_from_container creates a new image from a container with the -// given repo & tag, given the container's ID. -pub fn create_image_from_container(id string, repo string, tag string) ?Image { - res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? - - if res.status_code != 201 { - return error('Failed to create image from container.') - } - - return json.decode(Image, res.text) or {} -} - -// remove_image removes the image with the given ID. -pub fn remove_image(id string) ?bool { - res := request('DELETE', urllib.parse('/v1.41/images/$id')?)? - - return res.status_code == 200 -} - // remove_image removes the image with the given id. pub fn (mut d DockerDaemon) remove_image(id string) ? { d.send_request('DELETE', urllib.parse('/v1.41/images/$id')?)? diff --git a/src/docker/socket.v b/src/docker/socket.v deleted file mode 100644 index 2ee9bff..0000000 --- a/src/docker/socket.v +++ /dev/null @@ -1,158 +0,0 @@ -module docker - -import net.unix -import io -import net.http -import strings -import net.urllib -import json -import util - -const ( - socket = '/var/run/docker.sock' - buf_len = 10 * 1024 - http_separator = [u8(`\r`), `\n`, `\r`, `\n`] - http_chunk_separator = [u8(`\r`), `\n`] -) - -pub struct DockerDaemon { -mut: - socket &unix.StreamConn - reader &io.BufferedReader -} - -// new_conn creates a new connection to the Docker daemon. -pub fn new_conn() ?&DockerDaemon { - s := unix.connect_stream(docker.socket)? - - d := &DockerDaemon{ - socket: s - reader: io.new_buffered_reader(reader: s) - } - - return d -} - -// close closes the underlying socket connection. -pub fn (mut d DockerDaemon) close() ? { - d.socket.close()? -} - -// send_request sends an HTTP request without body. -pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - - d.socket.write_string(req)? - - // When starting a new request, the reader needs to be reset. - d.reader = io.new_buffered_reader(reader: d.socket) -} - -// send_request_with_body sends an HTTP request with the given body. -pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL, content_type string, body string) ? { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' - - d.socket.write_string(req)? - - // When starting a new request, the reader needs to be reset. - d.reader = io.new_buffered_reader(reader: d.socket) -} - -// send_request_with_json is a convenience wrapper around -// send_request_with_body that encodes the input as JSON. -pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib.URL, data &T) ? { - body := json.encode(data) - - return d.send_request_with_body(method, url, 'application/json', body) -} - -// read_response_head consumes the socket's contents until it encounters -// '\r\n\r\n', after which it parses the response as an HTTP response. -// Importantly, this function never consumes the reader past the HTTP -// separator, so the body can be read fully later on. -pub fn (mut d DockerDaemon) read_response_head() ?http.Response { - mut c := 0 - mut buf := []u8{len: 4} - mut res := []u8{} - - for { - c = d.reader.read(mut buf)? - res << buf[..c] - - match_len := util.match_array_in_array(buf[..c], docker.http_separator) - - if match_len == 4 { - break - } - - if match_len > 0 { - mut buf2 := []u8{len: 4 - match_len} - c2 := d.reader.read(mut buf2)? - res << buf2[..c2] - - if buf2 == docker.http_separator[match_len..] { - break - } - } - } - - return http.parse_response(res.bytestr()) -} - -// read_response_body reads `length` bytes from the stream. It can be used when -// the response encoding isn't chunked to fully read it. -pub fn (mut d DockerDaemon) read_response_body(length int) ?string { - if length == 0 { - return '' - } - - mut buf := []u8{len: docker.buf_len} - mut c := 0 - mut builder := strings.new_builder(docker.buf_len) - - for builder.len < length { - c = d.reader.read(mut buf) or { break } - - builder.write(buf[..c])? - } - - return builder.str() -} - -// read_response is a convenience function which always consumes the entire -// response & returns it. It should only be used when we're certain that the -// result isn't too large. -pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { - head := d.read_response_head()? - - if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' { - mut builder := strings.new_builder(1024) - mut body := d.get_chunked_response_reader() - - util.reader_to_writer(mut body, mut builder)? - - return head, builder.str() - } - - content_length := head.header.get(http.CommonHeader.content_length)?.int() - res := d.read_response_body(content_length)? - - return head, res -} - -// get_chunked_response_reader returns a ChunkedResponseReader using the socket -// as reader. -pub fn (mut d DockerDaemon) get_chunked_response_reader() &ChunkedResponseReader { - r := new_chunked_response_reader(d.reader) - - return r -} - -// get_stream_format_reader returns a StreamFormatReader using the socket as -// reader. -pub fn (mut d DockerDaemon) get_stream_format_reader() &StreamFormatReader { - r := new_chunked_response_reader(d.reader) - r2 := new_stream_format_reader(r) - - return r2 -} From 97cdaa18e1bb435ca512e81b70cb4fa3bc13f414 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 16 May 2022 13:03:44 +0200 Subject: [PATCH 11/17] refactor(docker): split stream separator code into own function --- CHANGELOG.md | 2 ++ src/docker/docker.v | 23 +---------------------- src/docker/stream.v | 23 ++--------------------- src/util/util.v | 36 ++++++++++++++++++++++++++++++++++-- 4 files changed, 39 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e17cd6..c86761c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Web API for adding & querying build logs * CLI commands to access build logs API * Cron build logs are uploaded to above API +* Proper ASCII table output in CLI ### Changed @@ -20,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Official Arch packages are now split between `vieter` & `vieter-git` * `vieter` is the latest release * `vieter-git` is the latest commit on the dev branch +* Full refactor of Docker socket code ## [0.3.0-alpha.1](https://git.rustybever.be/vieter/vieter/src/tag/0.3.0-alpha.1) diff --git a/src/docker/docker.v b/src/docker/docker.v index 2ee9bff..f612d1f 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -71,30 +71,9 @@ pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib. // Importantly, this function never consumes the reader past the HTTP // separator, so the body can be read fully later on. pub fn (mut d DockerDaemon) read_response_head() ?http.Response { - mut c := 0 - mut buf := []u8{len: 4} mut res := []u8{} - for { - c = d.reader.read(mut buf)? - res << buf[..c] - - match_len := util.match_array_in_array(buf[..c], docker.http_separator) - - if match_len == 4 { - break - } - - if match_len > 0 { - mut buf2 := []u8{len: 4 - match_len} - c2 := d.reader.read(mut buf2)? - res << buf2[..c2] - - if buf2 == docker.http_separator[match_len..] { - break - } - } - } + util.read_until_separator(mut d.reader, mut res, http_separator) ? return http.parse_response(res.bytestr()) } diff --git a/src/docker/stream.v b/src/docker/stream.v index f20fe18..ed73098 100644 --- a/src/docker/stream.v +++ b/src/docker/stream.v @@ -59,7 +59,6 @@ pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { // completely consumed. fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { mut buf := []u8{len: 2} - mut res := []u8{} if r.started { // Each chunk ends with a `\r\n` which we want to skip first @@ -68,26 +67,8 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { r.started = true - for { - c := r.reader.read(mut buf)? - res << buf[..c] - - match_len := util.match_array_in_array(buf[..c], http_chunk_separator) - - if match_len == http_chunk_separator.len { - break - } - - if match_len > 0 { - mut buf2 := []u8{len: 2 - match_len} - c2 := r.reader.read(mut buf2)? - res << buf2[..c2] - - if buf2 == http_chunk_separator[match_len..] { - break - } - } - } + mut res := []u8{} + util.read_until_separator(mut r.reader, mut res, http_chunk_separator) ? // The length of the next chunk is provided as a hexadecimal mut num_data := hex.decode(res#[..-2].bytestr())? diff --git a/src/util/util.v b/src/util/util.v index 324fb3d..9cf3011 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -28,9 +28,14 @@ pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { mut buf := []u8{len: 10 * 1024} for { - reader.read(mut buf) or { break } + bytes_read := reader.read(mut buf) or { break } + mut bytes_written := 0 - writer.write(buf) or { break } + for bytes_written < bytes_read { + c := writer.write(buf[bytes_written..bytes_read]) or { break } + + bytes_written += c + } } } @@ -122,3 +127,30 @@ pub fn match_array_in_array(a1 []T, a2 []T) int { return match_len } + +// read_until_separator consumes an io.Reader until it encounters some +// separator array. The data read is stored inside the provided res array. +pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? { + mut buf := []u8{len: sep.len} + + for { + c := reader.read(mut buf)? + res << buf[..c] + + match_len := match_array_in_array(buf[..c], sep) + + if match_len == sep.len { + break + } + + if match_len > 0 { + match_left := sep.len - match_len + c2 := reader.read(mut buf[..match_left])? + res << buf[..c2] + + if buf[..c2] == sep[match_len..] { + break + } + } + } +} From 1d3c7a1651c238c6818a3434f230a4a54ab7840c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 16 May 2022 14:09:21 +0200 Subject: [PATCH 12/17] refactor(docker): renamed DockerDaemon to DockerConn --- src/docker/README.md | 5 +++++ src/docker/containers.v | 12 ++++++------ src/docker/docker.v | 26 +++++++++++++------------- src/docker/images.v | 6 +++--- src/docker/stream.v | 2 +- 5 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 src/docker/README.md diff --git a/src/docker/README.md b/src/docker/README.md new file mode 100644 index 0000000..5236277 --- /dev/null +++ b/src/docker/README.md @@ -0,0 +1,5 @@ +# docker + +This module implements part of the Docker Engine API v1.41 +([documentation](https://docs.docker.com/engine/api/v1.41/)) using socket-based +HTTP communication. diff --git a/src/docker/containers.v b/src/docker/containers.v index 7605452..62c8031 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -15,7 +15,7 @@ struct Container { } // containers returns a list of all containers. -pub fn (mut d DockerDaemon) containers() ?[]Container { +pub fn (mut d DockerConn) containers() ?[]Container { d.send_request('GET', urllib.parse('/v1.41/containers/json')?)? head, res := d.read_response()? @@ -46,7 +46,7 @@ pub: } // create_container creates a new container with the given config. -pub fn (mut d DockerDaemon) create_container(c NewContainer) ?CreatedContainer { +pub fn (mut d DockerConn) create_container(c NewContainer) ?CreatedContainer { d.send_request_with_json('POST', urllib.parse('/v1.41/containers/create')?, c)? head, res := d.read_response()? @@ -62,7 +62,7 @@ pub fn (mut d DockerDaemon) create_container(c NewContainer) ?CreatedContainer { } // start_container starts the container with the given id. -pub fn (mut d DockerDaemon) start_container(id string) ? { +pub fn (mut d DockerConn) start_container(id string) ? { d.send_request('POST', urllib.parse('/v1.41/containers/$id/start')?)? head, body := d.read_response()? @@ -92,7 +92,7 @@ pub mut: } // inspect_container returns detailed information for a given container. -pub fn (mut d DockerDaemon) inspect_container(id string) ?ContainerInspect { +pub fn (mut d DockerConn) inspect_container(id string) ?ContainerInspect { d.send_request('GET', urllib.parse('/v1.41/containers/$id/json')?)? head, body := d.read_response()? @@ -114,7 +114,7 @@ pub fn (mut d DockerDaemon) inspect_container(id string) ?ContainerInspect { } // remove_container removes the container with the given id. -pub fn (mut d DockerDaemon) remove_container(id string) ? { +pub fn (mut d DockerConn) remove_container(id string) ? { d.send_request('DELETE', urllib.parse('/v1.41/containers/$id')?)? head, body := d.read_response()? @@ -127,7 +127,7 @@ pub fn (mut d DockerDaemon) remove_container(id string) ? { // get_container_logs returns a reader object allowing access to the // container's logs. -pub fn (mut d DockerDaemon) get_container_logs(id string) ?&StreamFormatReader { +pub fn (mut d DockerConn) get_container_logs(id string) ?&StreamFormatReader { d.send_request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)? head := d.read_response_head()? diff --git a/src/docker/docker.v b/src/docker/docker.v index f612d1f..4ce1ea6 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -15,17 +15,17 @@ const ( http_chunk_separator = [u8(`\r`), `\n`] ) -pub struct DockerDaemon { +pub struct DockerConn { mut: socket &unix.StreamConn reader &io.BufferedReader } // new_conn creates a new connection to the Docker daemon. -pub fn new_conn() ?&DockerDaemon { +pub fn new_conn() ?&DockerConn { s := unix.connect_stream(docker.socket)? - d := &DockerDaemon{ + d := &DockerConn{ socket: s reader: io.new_buffered_reader(reader: s) } @@ -34,12 +34,12 @@ pub fn new_conn() ?&DockerDaemon { } // close closes the underlying socket connection. -pub fn (mut d DockerDaemon) close() ? { +pub fn (mut d DockerConn) close() ? { d.socket.close()? } // send_request sends an HTTP request without body. -pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { +pub fn (mut d DockerConn) send_request(method string, url urllib.URL) ? { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' d.socket.write_string(req)? @@ -49,7 +49,7 @@ pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { } // send_request_with_body sends an HTTP request with the given body. -pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL, content_type string, body string) ? { +pub fn (mut d DockerConn) send_request_with_body(method string, url urllib.URL, content_type string, body string) ? { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' d.socket.write_string(req)? @@ -60,7 +60,7 @@ pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL // send_request_with_json is a convenience wrapper around // send_request_with_body that encodes the input as JSON. -pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib.URL, data &T) ? { +pub fn (mut d DockerConn) send_request_with_json(method string, url urllib.URL, data &T) ? { body := json.encode(data) return d.send_request_with_body(method, url, 'application/json', body) @@ -70,17 +70,17 @@ pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib. // '\r\n\r\n', after which it parses the response as an HTTP response. // Importantly, this function never consumes the reader past the HTTP // separator, so the body can be read fully later on. -pub fn (mut d DockerDaemon) read_response_head() ?http.Response { +pub fn (mut d DockerConn) read_response_head() ?http.Response { mut res := []u8{} - util.read_until_separator(mut d.reader, mut res, http_separator) ? + util.read_until_separator(mut d.reader, mut res, docker.http_separator)? return http.parse_response(res.bytestr()) } // read_response_body reads `length` bytes from the stream. It can be used when // the response encoding isn't chunked to fully read it. -pub fn (mut d DockerDaemon) read_response_body(length int) ?string { +pub fn (mut d DockerConn) read_response_body(length int) ?string { if length == 0 { return '' } @@ -101,7 +101,7 @@ pub fn (mut d DockerDaemon) read_response_body(length int) ?string { // read_response is a convenience function which always consumes the entire // response & returns it. It should only be used when we're certain that the // result isn't too large. -pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { +pub fn (mut d DockerConn) read_response() ?(http.Response, string) { head := d.read_response_head()? if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' { @@ -121,7 +121,7 @@ pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { // get_chunked_response_reader returns a ChunkedResponseReader using the socket // as reader. -pub fn (mut d DockerDaemon) get_chunked_response_reader() &ChunkedResponseReader { +pub fn (mut d DockerConn) get_chunked_response_reader() &ChunkedResponseReader { r := new_chunked_response_reader(d.reader) return r @@ -129,7 +129,7 @@ pub fn (mut d DockerDaemon) get_chunked_response_reader() &ChunkedResponseReader // get_stream_format_reader returns a StreamFormatReader using the socket as // reader. -pub fn (mut d DockerDaemon) get_stream_format_reader() &StreamFormatReader { +pub fn (mut d DockerConn) get_stream_format_reader() &StreamFormatReader { r := new_chunked_response_reader(d.reader) r2 := new_stream_format_reader(r) diff --git a/src/docker/images.v b/src/docker/images.v index 5866905..cab7f34 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -10,7 +10,7 @@ pub: } // pull_image pulls the given image:tag. -pub fn (mut d DockerDaemon) pull_image(image string, tag string) ? { +pub fn (mut d DockerConn) pull_image(image string, tag string) ? { d.send_request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)? head := d.read_response_head()? @@ -33,7 +33,7 @@ pub fn (mut d DockerDaemon) pull_image(image string, tag string) ? { } // create_image_from_container creates a new image from a container. -pub fn (mut d DockerDaemon) create_image_from_container(id string, repo string, tag string) ?Image { +pub fn (mut d DockerConn) create_image_from_container(id string, repo string, tag string) ?Image { d.send_request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? head, body := d.read_response()? @@ -49,7 +49,7 @@ pub fn (mut d DockerDaemon) create_image_from_container(id string, repo string, } // remove_image removes the image with the given id. -pub fn (mut d DockerDaemon) remove_image(id string) ? { +pub fn (mut d DockerConn) remove_image(id string) ? { d.send_request('DELETE', urllib.parse('/v1.41/images/$id')?)? head, body := d.read_response()? diff --git a/src/docker/stream.v b/src/docker/stream.v index ed73098..dff1784 100644 --- a/src/docker/stream.v +++ b/src/docker/stream.v @@ -68,7 +68,7 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { r.started = true mut res := []u8{} - util.read_until_separator(mut r.reader, mut res, http_chunk_separator) ? + util.read_until_separator(mut r.reader, mut res, http_chunk_separator)? // The length of the next chunk is provided as a hexadecimal mut num_data := hex.decode(res#[..-2].bytestr())? From 055b168ff12d1db28ad0dc918a3790cd523597c5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 16 May 2022 14:22:53 +0200 Subject: [PATCH 13/17] refactor(util): split into two files --- src/build/build.v | 7 ++-- src/docker/README.md | 2 - src/docker/stream.v | 4 +- src/env/env.v | 13 +++--- src/util/README.md | 2 + src/util/stream.v | 95 ++++++++++++++++++++++++++++++++++++++++++ src/util/util.v | 98 ++------------------------------------------ 7 files changed, 114 insertions(+), 107 deletions(-) create mode 100644 src/util/README.md create mode 100644 src/util/stream.v diff --git a/src/build/build.v b/src/build/build.v index 0de91a6..2784c26 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -9,9 +9,10 @@ import client import strings import util -const container_build_dir = '/build' - -const build_image_repo = 'vieter-build' +const ( + container_build_dir = '/build' + build_image_repo = 'vieter-build' +) // create_build_image creates a builder image given some base image which can // then be used to build & package Arch images. It mostly just updates the diff --git a/src/docker/README.md b/src/docker/README.md index 5236277..4cc8971 100644 --- a/src/docker/README.md +++ b/src/docker/README.md @@ -1,5 +1,3 @@ -# docker - This module implements part of the Docker Engine API v1.41 ([documentation](https://docs.docker.com/engine/api/v1.41/)) using socket-based HTTP communication. diff --git a/src/docker/stream.v b/src/docker/stream.v index dff1784..02fb972 100644 --- a/src/docker/stream.v +++ b/src/docker/stream.v @@ -58,9 +58,9 @@ pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { // This function should only be called if the previous chunk has been // completely consumed. fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { - mut buf := []u8{len: 2} - if r.started { + mut buf := []u8{len: 2} + // Each chunk ends with a `\r\n` which we want to skip first r.reader.read(mut buf)? } diff --git a/src/env/env.v b/src/env/env.v index d145931..3d07d23 100644 --- a/src/env/env.v +++ b/src/env/env.v @@ -3,12 +3,13 @@ module env import os import toml -// The prefix that every environment variable should have -const prefix = 'VIETER_' - -// The suffix an environment variable in order for it to be loaded from a file -// instead -const file_suffix = '_FILE' +const ( + // The prefix that every environment variable should have + prefix = 'VIETER_' + // The suffix an environment variable in order for it to be loaded from a file + // instead + file_suffix = '_FILE' +) fn get_env_var(field_name string) ?string { env_var_name := '$env.prefix$field_name.to_upper()' diff --git a/src/util/README.md b/src/util/README.md new file mode 100644 index 0000000..529e412 --- /dev/null +++ b/src/util/README.md @@ -0,0 +1,2 @@ +This module defines a few useful functions used throughout the codebase that +don't specifically fit inside a module. diff --git a/src/util/stream.v b/src/util/stream.v new file mode 100644 index 0000000..06397aa --- /dev/null +++ b/src/util/stream.v @@ -0,0 +1,95 @@ +// Functions for interacting with `io.Reader` & `io.Writer` objects. +module util + +import io +import os + +// reader_to_writer tries to consume the entire reader & write it to the writer. +pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { + mut buf := []u8{len: 10 * 1024} + + for { + bytes_read := reader.read(mut buf) or { break } + mut bytes_written := 0 + + for bytes_written < bytes_read { + c := writer.write(buf[bytes_written..bytes_read]) or { break } + + bytes_written += c + } + } +} + +// reader_to_file writes the contents of a BufferedReader to a file +pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { + mut file := os.create(path)? + defer { + file.close() + } + + mut buf := []u8{len: reader_buf_size} + mut bytes_left := length + + // Repeat as long as the stream still has data + for bytes_left > 0 { + // TODO check if just breaking here is safe + bytes_read := reader.read(mut buf) or { break } + bytes_left -= bytes_read + + mut to_write := bytes_read + + for to_write > 0 { + // TODO don't just loop infinitely here + bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } + // file.flush() + + to_write = to_write - bytes_written + } + } +} + +// match_array_in_array returns how many elements of a2 overlap with a1. For +// example, if a1 = "abcd" & a2 = "cd", the result will be 2. If the match is +// not at the end of a1, the result is 0. +pub fn match_array_in_array(a1 []T, a2 []T) int { + mut i := 0 + mut match_len := 0 + + for i + match_len < a1.len { + if a1[i + match_len] == a2[match_len] { + match_len += 1 + } else { + i += match_len + 1 + match_len = 0 + } + } + + return match_len +} + +// read_until_separator consumes an io.Reader until it encounters some +// separator array. The data read is stored inside the provided res array. +pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? { + mut buf := []u8{len: sep.len} + + for { + c := reader.read(mut buf)? + res << buf[..c] + + match_len := match_array_in_array(buf[..c], sep) + + if match_len == sep.len { + break + } + + if match_len > 0 { + match_left := sep.len - match_len + c2 := reader.read(mut buf[..match_left])? + res << buf[..c2] + + if buf[..c2] == sep[match_len..] { + break + } + } + } +} diff --git a/src/util/util.v b/src/util/util.v index 9cf3011..266bcb5 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -1,13 +1,13 @@ module util import os -import io import crypto.md5 import crypto.sha256 -const reader_buf_size = 1_000_000 - -const prefixes = ['B', 'KB', 'MB', 'GB'] +const ( + reader_buf_size = 1_000_000 + prefixes = ['B', 'KB', 'MB', 'GB'] +) // Dummy struct to work around the fact that you can only share structs, maps & // arrays @@ -23,50 +23,6 @@ pub fn exit_with_message(code int, msg string) { exit(code) } -// reader_to_writer tries to consume the entire reader & write it to the writer. -pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { - mut buf := []u8{len: 10 * 1024} - - for { - bytes_read := reader.read(mut buf) or { break } - mut bytes_written := 0 - - for bytes_written < bytes_read { - c := writer.write(buf[bytes_written..bytes_read]) or { break } - - bytes_written += c - } - } -} - -// reader_to_file writes the contents of a BufferedReader to a file -pub fn reader_to_file(mut reader io.BufferedReader, length int, path string) ? { - mut file := os.create(path)? - defer { - file.close() - } - - mut buf := []u8{len: util.reader_buf_size} - mut bytes_left := length - - // Repeat as long as the stream still has data - for bytes_left > 0 { - // TODO check if just breaking here is safe - bytes_read := reader.read(mut buf) or { break } - bytes_left -= bytes_read - - mut to_write := bytes_read - - for to_write > 0 { - // TODO don't just loop infinitely here - bytes_written := file.write(buf[bytes_read - to_write..bytes_read]) or { continue } - // file.flush() - - to_write = to_write - bytes_written - } - } -} - // hash_file returns the md5 & sha256 hash of a given file // TODO actually implement sha256 pub fn hash_file(path &string) ?(string, string) { @@ -108,49 +64,3 @@ pub fn pretty_bytes(bytes int) string { return '${n:.2}${util.prefixes[i]}' } - -// match_array_in_array returns how many elements of a2 overlap with a1. For -// example, if a1 = "abcd" & a2 = "cd", the result will be 2. If the match is -// not at the end of a1, the result is 0. -pub fn match_array_in_array(a1 []T, a2 []T) int { - mut i := 0 - mut match_len := 0 - - for i + match_len < a1.len { - if a1[i + match_len] == a2[match_len] { - match_len += 1 - } else { - i += match_len + 1 - match_len = 0 - } - } - - return match_len -} - -// read_until_separator consumes an io.Reader until it encounters some -// separator array. The data read is stored inside the provided res array. -pub fn read_until_separator(mut reader io.Reader, mut res []u8, sep []u8) ? { - mut buf := []u8{len: sep.len} - - for { - c := reader.read(mut buf)? - res << buf[..c] - - match_len := match_array_in_array(buf[..c], sep) - - if match_len == sep.len { - break - } - - if match_len > 0 { - match_left := sep.len - match_len - c2 := reader.read(mut buf[..match_left])? - res << buf[..c2] - - if buf[..c2] == sep[match_len..] { - break - } - } - } -} From d4c803c41c7d1322cd3a3e7975bc10c6adb69a59 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 16 May 2022 14:53:48 +0200 Subject: [PATCH 14/17] doc(env): added missing docstring & README --- src/env/README.md | 7 +++++++ src/env/env.v | 5 +++++ 2 files changed, 12 insertions(+) create mode 100644 src/env/README.md diff --git a/src/env/README.md b/src/env/README.md new file mode 100644 index 0000000..135e8fa --- /dev/null +++ b/src/env/README.md @@ -0,0 +1,7 @@ +This module provides a framework for parsing a configuration, defined as a +struct, from both a TOML configuration file & environment variables. Some +notable features are: + +* Overwrite values in config file using environment variables +* Allow default values in config struct +* Read environment variable value from file diff --git a/src/env/env.v b/src/env/env.v index 3d07d23..5ed1955 100644 --- a/src/env/env.v +++ b/src/env/env.v @@ -11,6 +11,11 @@ const ( file_suffix = '_FILE' ) +// get_env_var tries to read the contents of the given environment variable. It +// looks for either `${env.prefix}${field_name.to_upper()}` or +// `${env.prefix}${field_name.to_upper()}${env.file_suffix}`, returning the +// contents of the file instead if the latter. If both or neither exist, the +// function returns an error. fn get_env_var(field_name string) ?string { env_var_name := '$env.prefix$field_name.to_upper()' env_file_name := '$env.prefix$field_name.to_upper()$env.file_suffix' From 850cba6ab9b81b8bf2c28a9348c5ba7ed2f9181f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 16 May 2022 15:02:57 +0200 Subject: [PATCH 15/17] refactor(docker): use http.Method instead of strings --- src/docker/containers.v | 14 +++++++------- src/docker/docker.v | 6 +++--- src/docker/images.v | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/docker/containers.v b/src/docker/containers.v index 62c8031..a790243 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -3,7 +3,7 @@ module docker import json import net.urllib import time -import net.http +import net.http { Method } struct DockerError { message string @@ -16,7 +16,7 @@ struct Container { // containers returns a list of all containers. pub fn (mut d DockerConn) containers() ?[]Container { - d.send_request('GET', urllib.parse('/v1.41/containers/json')?)? + d.send_request(Method.get, urllib.parse('/v1.41/containers/json')?)? head, res := d.read_response()? if head.status_code != 200 { @@ -47,7 +47,7 @@ pub: // create_container creates a new container with the given config. pub fn (mut d DockerConn) create_container(c NewContainer) ?CreatedContainer { - d.send_request_with_json('POST', urllib.parse('/v1.41/containers/create')?, c)? + d.send_request_with_json(Method.post, urllib.parse('/v1.41/containers/create')?, c)? head, res := d.read_response()? if head.status_code != 201 { @@ -63,7 +63,7 @@ pub fn (mut d DockerConn) create_container(c NewContainer) ?CreatedContainer { // start_container starts the container with the given id. pub fn (mut d DockerConn) start_container(id string) ? { - d.send_request('POST', urllib.parse('/v1.41/containers/$id/start')?)? + d.send_request(Method.post, urllib.parse('/v1.41/containers/$id/start')?)? head, body := d.read_response()? if head.status_code != 204 { @@ -93,7 +93,7 @@ pub mut: // inspect_container returns detailed information for a given container. pub fn (mut d DockerConn) inspect_container(id string) ?ContainerInspect { - d.send_request('GET', urllib.parse('/v1.41/containers/$id/json')?)? + d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/json')?)? head, body := d.read_response()? if head.status_code != 200 { @@ -115,7 +115,7 @@ pub fn (mut d DockerConn) inspect_container(id string) ?ContainerInspect { // remove_container removes the container with the given id. pub fn (mut d DockerConn) remove_container(id string) ? { - d.send_request('DELETE', urllib.parse('/v1.41/containers/$id')?)? + d.send_request(Method.delete, urllib.parse('/v1.41/containers/$id')?)? head, body := d.read_response()? if head.status_code != 204 { @@ -128,7 +128,7 @@ pub fn (mut d DockerConn) remove_container(id string) ? { // get_container_logs returns a reader object allowing access to the // container's logs. pub fn (mut d DockerConn) get_container_logs(id string) ?&StreamFormatReader { - d.send_request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)? + d.send_request(Method.get, urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)? head := d.read_response_head()? if head.status_code != 200 { diff --git a/src/docker/docker.v b/src/docker/docker.v index 4ce1ea6..ccc6bed 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -39,7 +39,7 @@ pub fn (mut d DockerConn) close() ? { } // send_request sends an HTTP request without body. -pub fn (mut d DockerConn) send_request(method string, url urllib.URL) ? { +pub fn (mut d DockerConn) send_request(method http.Method, url urllib.URL) ? { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' d.socket.write_string(req)? @@ -49,7 +49,7 @@ pub fn (mut d DockerConn) send_request(method string, url urllib.URL) ? { } // send_request_with_body sends an HTTP request with the given body. -pub fn (mut d DockerConn) send_request_with_body(method string, url urllib.URL, content_type string, body string) ? { +pub fn (mut d DockerConn) send_request_with_body(method http.Method, url urllib.URL, content_type string, body string) ? { req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' d.socket.write_string(req)? @@ -60,7 +60,7 @@ pub fn (mut d DockerConn) send_request_with_body(method string, url urllib.URL, // send_request_with_json is a convenience wrapper around // send_request_with_body that encodes the input as JSON. -pub fn (mut d DockerConn) send_request_with_json(method string, url urllib.URL, data &T) ? { +pub fn (mut d DockerConn) send_request_with_json(method http.Method, url urllib.URL, data &T) ? { body := json.encode(data) return d.send_request_with_body(method, url, 'application/json', body) diff --git a/src/docker/images.v b/src/docker/images.v index cab7f34..6161565 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -1,6 +1,6 @@ module docker -import net.http +import net.http { Method } import net.urllib import json @@ -11,7 +11,7 @@ pub: // pull_image pulls the given image:tag. pub fn (mut d DockerConn) pull_image(image string, tag string) ? { - d.send_request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)? + d.send_request(Method.post, urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?)? head := d.read_response_head()? if head.status_code != 200 { @@ -34,7 +34,7 @@ pub fn (mut d DockerConn) pull_image(image string, tag string) ? { // create_image_from_container creates a new image from a container. pub fn (mut d DockerConn) create_image_from_container(id string, repo string, tag string) ?Image { - d.send_request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? + d.send_request(Method.post, urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? head, body := d.read_response()? if head.status_code != 201 { @@ -50,7 +50,7 @@ pub fn (mut d DockerConn) create_image_from_container(id string, repo string, ta // remove_image removes the image with the given id. pub fn (mut d DockerConn) remove_image(id string) ? { - d.send_request('DELETE', urllib.parse('/v1.41/images/$id')?)? + d.send_request(Method.delete, urllib.parse('/v1.41/images/$id')?)? head, body := d.read_response()? if head.status_code != 200 { From 3c87e60293810b37a1c574976261b1d7fc5a9df0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 16 May 2022 15:36:21 +0200 Subject: [PATCH 16/17] refactor(docker): more tightly integrate streams --- src/docker/stream.v | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/src/docker/stream.v b/src/docker/stream.v index 02fb972..001f4b3 100644 --- a/src/docker/stream.v +++ b/src/docker/stream.v @@ -9,15 +9,14 @@ import encoding.hex // it as if it was a continuous stream of data. struct ChunkedResponseReader { mut: - reader io.Reader + reader io.BufferedReader bytes_left_in_chunk u64 - end_of_stream bool started bool } // new_chunked_response_reader creates a new ChunkedResponseReader on the heap // with the provided reader. -pub fn new_chunked_response_reader(reader io.Reader) &ChunkedResponseReader { +pub fn new_chunked_response_reader(reader io.BufferedReader) &ChunkedResponseReader { r := &ChunkedResponseReader{ reader: reader } @@ -27,16 +26,10 @@ pub fn new_chunked_response_reader(reader io.Reader) &ChunkedResponseReader { // read satisfies the io.Reader interface. pub fn (mut r ChunkedResponseReader) read(mut buf []u8) ?int { - if r.end_of_stream { - return none - } - if r.bytes_left_in_chunk == 0 { + // An io.BufferedReader always returns none if its stream has + // ended. r.bytes_left_in_chunk = r.read_chunk_size()? - - if r.end_of_stream { - return none - } } mut c := 0 @@ -82,7 +75,7 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { // This only occurs for the very last chunk, which always reports a size of // 0. if num == 0 { - r.end_of_stream = true + return none } return num @@ -92,14 +85,13 @@ fn (mut r ChunkedResponseReader) read_chunk_size() ?u64 { // header bytes. struct StreamFormatReader { mut: - reader io.Reader + reader ChunkedResponseReader bytes_left_in_chunk u32 - end_of_stream bool } // new_stream_format_reader creates a new StreamFormatReader using the given // reader. -pub fn new_stream_format_reader(reader io.Reader) &StreamFormatReader { +pub fn new_stream_format_reader(reader ChunkedResponseReader) &StreamFormatReader { r := &StreamFormatReader{ reader: reader } @@ -109,16 +101,8 @@ pub fn new_stream_format_reader(reader io.Reader) &StreamFormatReader { // read satisfies the io.Reader interface. pub fn (mut r StreamFormatReader) read(mut buf []u8) ?int { - if r.end_of_stream { - return none - } - if r.bytes_left_in_chunk == 0 { r.bytes_left_in_chunk = r.read_chunk_size()? - - if r.end_of_stream { - return none - } } mut c := 0 @@ -143,5 +127,9 @@ fn (mut r StreamFormatReader) read_chunk_size() ?u32 { num := binary.big_endian_u32(buf[4..]) + if num == 0 { + return none + } + return num } From 889d5a08849a6ab08b41e6ff94b78f7a9624c460 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 16 May 2022 15:39:23 +0200 Subject: [PATCH 17/17] refactor(docker): removed unused function --- src/docker/containers.v | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/docker/containers.v b/src/docker/containers.v index a790243..0bc59bb 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -9,27 +9,6 @@ struct DockerError { message string } -struct Container { - id string [json: Id] - names []string [json: Names] -} - -// containers returns a list of all containers. -pub fn (mut d DockerConn) containers() ?[]Container { - d.send_request(Method.get, urllib.parse('/v1.41/containers/json')?)? - head, res := d.read_response()? - - if head.status_code != 200 { - data := json.decode(DockerError, res)? - - return error(data.message) - } - - data := json.decode([]Container, res)? - - return data -} - pub struct NewContainer { image string [json: Image] entrypoint []string [json: Entrypoint]