diff --git a/src/build/build.v b/src/build/build.v index 0de91a6..98c56f5 100644 --- a/src/build/build.v +++ b/src/build/build.v @@ -6,8 +6,6 @@ import time import os import db import client -import strings -import util const container_build_dir = '/build' @@ -21,10 +19,6 @@ const build_image_repo = 'vieter-build' pub fn create_build_image(base_image string) ?string { mut dd := docker.new_conn()? - defer { - dd.close() or {} - } - commands := [ // Update repos & install required packages 'pacman -Syu --needed --noconfirm base-devel git' @@ -54,7 +48,7 @@ pub fn create_build_image(base_image string) ?string { image_tag := if image_parts.len > 1 { image_parts[1] } else { 'latest' } // We pull the provided image - dd.pull_image(image_name, image_tag)? + docker.pull_image(image_name, image_tag)? id := dd.create_container(c)?.id // id := docker.create_container(c)? @@ -76,7 +70,7 @@ pub fn create_build_image(base_image string) ?string { // TODO also add the base image's name into the image name to prevent // conflicts. tag := time.sys_mono_now().str() - image := dd.create_image_from_container(id, 'vieter-build', tag)? + image := docker.create_image_from_container(id, 'vieter-build', tag)? dd.remove_container(id)? return image.id @@ -94,12 +88,6 @@ pub: // provided GitRepo. The base image ID should be of an image previously created // by create_build_image. It returns the logs of the container. pub fn build_repo(address string, api_key string, base_image_id string, repo &db.GitRepo) ?BuildResult { - mut dd := docker.new_conn()? - - defer { - dd.close() or {} - } - build_arch := os.uname().machine // TODO what to do with PKGBUILDs that build multiple packages? @@ -127,31 +115,27 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &db user: 'builder:builder' } - id := dd.create_container(c)?.id - dd.start_container(id)? + id := docker.create_container(c)? + docker.start_container(id)? - mut data := dd.inspect_container(id)? + mut data := docker.inspect_container(id)? // This loop waits until the container has stopped, so we can remove it after for data.state.running { time.sleep(1 * time.second) - data = dd.inspect_container(id)? + data = docker.inspect_container(id)? } - mut logs_stream := dd.get_container_logs(id)? + logs := docker.get_container_logs(id)? - // Read in the entire stream - mut logs_builder := strings.new_builder(10 * 1024) - util.reader_to_writer(mut logs_stream, mut logs_builder)? - - dd.remove_container(id)? + docker.remove_container(id)? return BuildResult{ start_time: data.state.start_time end_time: data.state.end_time exit_code: data.state.exit_code - logs: logs_builder.str() + logs: logs } } @@ -169,14 +153,7 @@ fn build(conf Config, repo_id int) ? { res := build_repo(conf.address, conf.api_key, image_id, repo)? println('Removing build image...') - - mut dd := docker.new_conn()? - - defer { - dd.close() or {} - } - - dd.remove_image(image_id)? + docker.remove_image(image_id)? println('Uploading logs to Vieter...') c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, diff --git a/src/cron/daemon/daemon.v b/src/cron/daemon/daemon.v index da3b46e..ade8fcb 100644 --- a/src/cron/daemon/daemon.v +++ b/src/cron/daemon/daemon.v @@ -253,21 +253,14 @@ fn (mut d Daemon) rebuild_base_image() bool { fn (mut d Daemon) clean_old_base_images() { mut i := 0 - mut dd := docker.new_conn() or { - d.lerror('Failed to connect to Docker socket.') - return - } - - defer { - dd.close() or {} - } - for i < d.builder_images.len - 1 { // For each builder image, we try to remove it by calling the Docker // API. If the function returns an error or false, that means the image // wasn't deleted. Therefore, we move the index over. If the function // returns true, the array's length has decreased by one so we don't // move the index. - dd.remove_image(d.builder_images[i]) or { i += 1 } + if !docker.remove_image(d.builder_images[i]) or { false } { + i += 1 + } } } diff --git a/src/docker/containers.v b/src/docker/containers.v index 7605452..05c9cc7 100644 --- a/src/docker/containers.v +++ b/src/docker/containers.v @@ -30,6 +30,7 @@ pub fn (mut d DockerDaemon) containers() ?[]Container { return data } +[params] pub struct NewContainer { image string [json: Image] entrypoint []string [json: Entrypoint] @@ -73,6 +74,26 @@ pub fn (mut d DockerDaemon) start_container(id string) ? { } } +// create_container creates a container defined by the given configuration. If +// successful, it returns the ID of the newly created container. +pub fn create_container(c &NewContainer) ?string { + res := request_with_json('POST', urllib.parse('/v1.41/containers/create')?, c)? + + if res.status_code != 201 { + return error('Failed to create container.') + } + + return json.decode(CreatedContainer, res.text)?.id +} + +// start_container starts a container with a given ID. It returns whether the +// container was started or not. +pub fn start_container(id string) ?bool { + res := request('POST', urllib.parse('/v1.41/containers/$id/start')?)? + + return res.status_code == 204 +} + struct ContainerInspect { pub mut: state ContainerState [json: State] @@ -113,6 +134,26 @@ pub fn (mut d DockerDaemon) inspect_container(id string) ?ContainerInspect { return data } +// inspect_container returns the result of inspecting a container with a given +// ID. +pub fn inspect_container(id string) ?ContainerInspect { + res := request('GET', urllib.parse('/v1.41/containers/$id/json')?)? + + if res.status_code != 200 { + return error('Failed to inspect container.') + } + + mut data := json.decode(ContainerInspect, res.text)? + + data.state.start_time = time.parse_rfc3339(data.state.start_time_str)? + + if data.state.status == 'exited' { + data.state.end_time = time.parse_rfc3339(data.state.end_time_str)? + } + + return data +} + // remove_container removes the container with the given id. pub fn (mut d DockerDaemon) remove_container(id string) ? { d.send_request('DELETE', urllib.parse('/v1.41/containers/$id')?)? @@ -125,6 +166,13 @@ pub fn (mut d DockerDaemon) remove_container(id string) ? { } } +// remove_container removes a container with a given ID. +pub fn remove_container(id string) ?bool { + res := request('DELETE', urllib.parse('/v1.41/containers/$id')?)? + + return res.status_code == 204 +} + // get_container_logs returns a reader object allowing access to the // container's logs. pub fn (mut d DockerDaemon) get_container_logs(id string) ?&StreamFormatReader { @@ -141,3 +189,25 @@ pub fn (mut d DockerDaemon) get_container_logs(id string) ?&StreamFormatReader { return d.get_stream_format_reader() } + +// get_container_logs retrieves the logs for a Docker container, both stdout & +// stderr. +pub fn get_container_logs(id string) ?string { + res := request('GET', urllib.parse('/v1.41/containers/$id/logs?stdout=true&stderr=true')?)? + mut res_bytes := res.text.bytes() + + // Docker uses a special "stream" format for their logs, so we have to + // clean up the data. + mut index := 0 + + for index < res_bytes.len { + // The reverse is required because V reads in the bytes differently + t := res_bytes[index + 4..index + 8].reverse() + len_length := unsafe { *(&u32(&t[0])) } + + res_bytes.delete_many(index, 8) + index += int(len_length) + } + + return res_bytes.bytestr() +} diff --git a/src/docker/docker.v b/src/docker/docker.v index 2ee9bff..fa83d89 100644 --- a/src/docker/docker.v +++ b/src/docker/docker.v @@ -1,158 +1,91 @@ module docker import net.unix -import io -import net.http -import strings import net.urllib +import net.http import json -import util -const ( - socket = '/var/run/docker.sock' - buf_len = 10 * 1024 - http_separator = [u8(`\r`), `\n`, `\r`, `\n`] - http_chunk_separator = [u8(`\r`), `\n`] -) - -pub struct DockerDaemon { -mut: - socket &unix.StreamConn - reader &io.BufferedReader -} - -// new_conn creates a new connection to the Docker daemon. -pub fn new_conn() ?&DockerDaemon { - s := unix.connect_stream(docker.socket)? - - d := &DockerDaemon{ - socket: s - reader: io.new_buffered_reader(reader: s) +// send writes a request to the Docker socket, waits for a response & returns +// it. +fn send(req &string) ?http.Response { + // Open a connection to the socket + mut s := unix.connect_stream(socket) or { + return error('Failed to connect to socket ${socket}.') } - return d -} + defer { + // This or is required because otherwise, the V compiler segfaults for + // some reason + // https://github.com/vlang/v/issues/13534 + s.close() or {} + } -// close closes the underlying socket connection. -pub fn (mut d DockerDaemon) close() ? { - d.socket.close()? -} + // Write the request to the socket + s.write_string(req) or { return error('Failed to write request to socket ${socket}.') } -// send_request sends an HTTP request without body. -pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' + s.wait_for_write()? - d.socket.write_string(req)? - - // When starting a new request, the reader needs to be reset. - d.reader = io.new_buffered_reader(reader: d.socket) -} - -// send_request_with_body sends an HTTP request with the given body. -pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL, content_type string, body string) ? { - req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' - - d.socket.write_string(req)? - - // When starting a new request, the reader needs to be reset. - d.reader = io.new_buffered_reader(reader: d.socket) -} - -// send_request_with_json is a convenience wrapper around -// send_request_with_body that encodes the input as JSON. -pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib.URL, data &T) ? { - body := json.encode(data) - - return d.send_request_with_body(method, url, 'application/json', body) -} - -// read_response_head consumes the socket's contents until it encounters -// '\r\n\r\n', after which it parses the response as an HTTP response. -// Importantly, this function never consumes the reader past the HTTP -// separator, so the body can be read fully later on. -pub fn (mut d DockerDaemon) read_response_head() ?http.Response { mut c := 0 - mut buf := []u8{len: 4} + mut buf := []u8{len: buf_len} mut res := []u8{} for { - c = d.reader.read(mut buf)? + c = s.read(mut buf) or { return error('Failed to read data from socket ${socket}.') } res << buf[..c] - match_len := util.match_array_in_array(buf[..c], docker.http_separator) - - if match_len == 4 { + if c < buf_len { break } + } - if match_len > 0 { - mut buf2 := []u8{len: 4 - match_len} - c2 := d.reader.read(mut buf2)? - res << buf2[..c2] + // After reading the first part of the response, we parse it into an HTTP + // response. If it isn't chunked, we return early with the data. + parsed := http.parse_response(res.bytestr()) or { + return error('Failed to parse HTTP response from socket ${socket}.') + } - if buf2 == docker.http_separator[match_len..] { + if parsed.header.get(http.CommonHeader.transfer_encoding) or { '' } != 'chunked' { + return parsed + } + + // We loop until we've encountered the end of the chunked response + // A chunked HTTP response always ends with '0\r\n\r\n'. + for res.len < 5 || res#[-5..] != [u8(`0`), `\r`, `\n`, `\r`, `\n`] { + // Wait for the server to respond + s.wait_for_write()? + + for { + c = s.read(mut buf) or { return error('Failed to read data from socket ${socket}.') } + res << buf[..c] + + if c < buf_len { break } } } + // Decode chunked response return http.parse_response(res.bytestr()) } -// read_response_body reads `length` bytes from the stream. It can be used when -// the response encoding isn't chunked to fully read it. -pub fn (mut d DockerDaemon) read_response_body(length int) ?string { - if length == 0 { - return '' - } +// request_with_body sends a request to the Docker socket with the given body. +fn request_with_body(method string, url urllib.URL, content_type string, body string) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' - mut buf := []u8{len: docker.buf_len} - mut c := 0 - mut builder := strings.new_builder(docker.buf_len) - - for builder.len < length { - c = d.reader.read(mut buf) or { break } - - builder.write(buf[..c])? - } - - return builder.str() + return send(req) } -// read_response is a convenience function which always consumes the entire -// response & returns it. It should only be used when we're certain that the -// result isn't too large. -pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { - head := d.read_response_head()? +// request sends a request to the Docker socket with an empty body. +fn request(method string, url urllib.URL) ?http.Response { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' - if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' { - mut builder := strings.new_builder(1024) - mut body := d.get_chunked_response_reader() - - util.reader_to_writer(mut body, mut builder)? - - return head, builder.str() - } - - content_length := head.header.get(http.CommonHeader.content_length)?.int() - res := d.read_response_body(content_length)? - - return head, res + return send(req) } -// get_chunked_response_reader returns a ChunkedResponseReader using the socket -// as reader. -pub fn (mut d DockerDaemon) get_chunked_response_reader() &ChunkedResponseReader { - r := new_chunked_response_reader(d.reader) +// request_with_json sends a request to the Docker socket with a given JSON +// payload +pub fn request_with_json(method string, url urllib.URL, data &T) ?http.Response { + body := json.encode(data) - return r -} - -// get_stream_format_reader returns a StreamFormatReader using the socket as -// reader. -pub fn (mut d DockerDaemon) get_stream_format_reader() &StreamFormatReader { - r := new_chunked_response_reader(d.reader) - r2 := new_stream_format_reader(r) - - return r2 + return request_with_body(method, url, 'application/json', body) } diff --git a/src/docker/images.v b/src/docker/images.v index 5866905..ab735f2 100644 --- a/src/docker/images.v +++ b/src/docker/images.v @@ -22,40 +22,37 @@ pub fn (mut d DockerDaemon) pull_image(image string, tag string) ? { return error(data.message) } - // Keep reading the body until the pull has completed mut body := d.get_chunked_response_reader() mut buf := []u8{len: 1024} for { - body.read(mut buf) or { break } + c := body.read(mut buf) or { break } + + print(buf[..c].bytestr()) } } -// create_image_from_container creates a new image from a container. -pub fn (mut d DockerDaemon) create_image_from_container(id string, repo string, tag string) ?Image { - d.send_request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? - head, body := d.read_response()? - - if head.status_code != 201 { - data := json.decode(DockerError, body)? - - return error(data.message) - } - - data := json.decode(Image, body)? - - return data +// pull_image pulls tries to pull the image for the given image & tag +pub fn pull_image(image string, tag string) ?http.Response { + return request('POST', urllib.parse('/v1.41/images/create?fromImage=$image&tag=$tag')?) } -// remove_image removes the image with the given id. -pub fn (mut d DockerDaemon) remove_image(id string) ? { - d.send_request('DELETE', urllib.parse('/v1.41/images/$id')?)? - head, body := d.read_response()? +// create_image_from_container creates a new image from a container with the +// given repo & tag, given the container's ID. +pub fn create_image_from_container(id string, repo string, tag string) ?Image { + res := request('POST', urllib.parse('/v1.41/commit?container=$id&repo=$repo&tag=$tag')?)? - if head.status_code != 200 { - data := json.decode(DockerError, body)? - - return error(data.message) + if res.status_code != 201 { + return error('Failed to create image from container.') } + + return json.decode(Image, res.text) or {} +} + +// remove_image removes the image with the given ID. +pub fn remove_image(id string) ?bool { + res := request('DELETE', urllib.parse('/v1.41/images/$id')?)? + + return res.status_code == 200 } diff --git a/src/docker/socket.v b/src/docker/socket.v new file mode 100644 index 0000000..dfa7ea7 --- /dev/null +++ b/src/docker/socket.v @@ -0,0 +1,153 @@ +module docker + +import net.unix +import io +import net.http +import strings +import net.urllib +import json +import util + +const ( + socket = '/var/run/docker.sock' + buf_len = 10 * 1024 + http_separator = [u8(`\r`), `\n`, `\r`, `\n`] + http_chunk_separator = [u8(`\r`), `\n`] +) + +pub struct DockerDaemon { +mut: + socket &unix.StreamConn + reader &io.BufferedReader +} + +// new_conn creates a new connection to the Docker daemon. +pub fn new_conn() ?&DockerDaemon { + s := unix.connect_stream(docker.socket)? + + d := &DockerDaemon{ + socket: s + reader: io.new_buffered_reader(reader: s) + } + + return d +} + +// send_request sends an HTTP request without body. +pub fn (mut d DockerDaemon) send_request(method string, url urllib.URL) ? { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\n\n' + + d.socket.write_string(req)? + + // When starting a new request, the reader needs to be reset. + d.reader = io.new_buffered_reader(reader: d.socket) +} + +// send_request_with_body sends an HTTP request with the given body. +pub fn (mut d DockerDaemon) send_request_with_body(method string, url urllib.URL, content_type string, body string) ? { + req := '$method $url.request_uri() HTTP/1.1\nHost: localhost\nContent-Type: $content_type\nContent-Length: $body.len\n\n$body\n\n' + + d.socket.write_string(req)? + + // When starting a new request, the reader needs to be reset. + d.reader = io.new_buffered_reader(reader: d.socket) +} + +// send_request_with_json is a convenience wrapper around +// send_request_with_body that encodes the input as JSON. +pub fn (mut d DockerDaemon) send_request_with_json(method string, url urllib.URL, data &T) ? { + body := json.encode(data) + + return d.send_request_with_body(method, url, 'application/json', body) +} + +// read_response_head consumes the socket's contents until it encounters +// '\r\n\r\n', after which it parses the response as an HTTP response. +// Importantly, this function never consumes the reader past the HTTP +// separator, so the body can be read fully later on. +pub fn (mut d DockerDaemon) read_response_head() ?http.Response { + mut c := 0 + mut buf := []u8{len: 4} + mut res := []u8{} + + for { + c = d.reader.read(mut buf)? + res << buf[..c] + + match_len := util.match_array_in_array(buf[..c], docker.http_separator) + + if match_len == 4 { + break + } + + if match_len > 0 { + mut buf2 := []u8{len: 4 - match_len} + c2 := d.reader.read(mut buf2)? + res << buf2[..c2] + + if buf2 == docker.http_separator[match_len..] { + break + } + } + } + + return http.parse_response(res.bytestr()) +} + +// read_response_body reads `length` bytes from the stream. It can be used when +// the response encoding isn't chunked to fully read it. +pub fn (mut d DockerDaemon) read_response_body(length int) ?string { + if length == 0 { + return '' + } + + mut buf := []u8{len: docker.buf_len} + mut c := 0 + mut builder := strings.new_builder(docker.buf_len) + + for builder.len < length { + c = d.reader.read(mut buf) or { break } + + builder.write(buf[..c])? + } + + return builder.str() +} + +// read_response is a convenience function which always consumes the entire +// response & returns it. It should only be used when we're certain that the +// result isn't too large. +pub fn (mut d DockerDaemon) read_response() ?(http.Response, string) { + head := d.read_response_head()? + + if head.header.get(http.CommonHeader.transfer_encoding) or { '' } == 'chunked' { + mut builder := strings.new_builder(1024) + mut body := d.get_chunked_response_reader() + + util.reader_to_writer(mut body, mut builder) ? + + return head, builder.str() + } + + content_length := head.header.get(http.CommonHeader.content_length)?.int() + res := d.read_response_body(content_length)? + + return head, res +} + +// get_chunked_response_reader returns a ChunkedResponseReader using the socket +// as reader. +pub fn (mut d DockerDaemon) get_chunked_response_reader() &ChunkedResponseReader { + r := new_chunked_response_reader(d.reader) + + return r +} + +// get_stream_format_reader returns a StreamFormatReader using the socket as +// reader. +pub fn (mut d DockerDaemon) get_stream_format_reader() &StreamFormatReader { + r := new_chunked_response_reader(d.reader) + r2 := new_stream_format_reader(r) + + return r2 +} diff --git a/src/main.v b/src/main.v index db6d5ef..6b1e7bc 100644 --- a/src/main.v +++ b/src/main.v @@ -7,6 +7,7 @@ import build import console.git import console.logs import cron +import docker fn main() { mut app := cli.Command{ diff --git a/src/util/util.v b/src/util/util.v index 324fb3d..7aabc1b 100644 --- a/src/util/util.v +++ b/src/util/util.v @@ -28,7 +28,7 @@ pub fn reader_to_writer(mut reader io.Reader, mut writer io.Writer) ? { mut buf := []u8{len: 10 * 1024} for { - reader.read(mut buf) or { break } + c := reader.read(mut buf) or { break } writer.write(buf) or { break } }