feat(agent): ensure images exist when starting build

src/agent/daemon.v

@@ -80,13 +80,24 @@ pub fn (mut d AgentDaemon) run() {
 			last_poll_time = time.now()
 
 			for config in new_configs {
-				// TODO handle this better than to just skip the config
 				// Make sure a recent build base image is available for
 				// building the config
-				d.images.refresh_image(config.base_image) or {
+				if !d.images.up_to_date(config.base_image) {
-					d.lerror(err.msg())
+					d.linfo('Building builder image from base image $config.base_image')
-					continue
+
+					// TODO handle this better than to just skip the config
+					d.images.refresh_image(config.base_image) or {
+						d.lerror(err.msg())
+						continue
+					}
 				}
+
+				// It's technically still possible that the build image is
+				// removed in the very short period between building the
+				// builder image and starting a build container with it. If
+				// this happens, faith really just didn't want you to do this
+				// build.
+
 				d.start_build(config)
 			}
 

src/agent/images.v

@@ -33,16 +33,42 @@ pub fn (m &ImageManager) get(base_image string) string {
 	return m.images[base_image].last()
 }
 
-// refresh_image builds a new builder image from the given base image if the
+// up_to_date returns whether the last known builder image is exists and is up
-// previous builder image is too old or non-existent. This function will do
+// to date. If this function returns true, the last builder image may be used
-// nothing if these conditions aren't met, so it's safe to call it every time
+// to perform a build.
-// you want to ensure an image is up to date.
+pub fn (mut m ImageManager) up_to_date(base_image string) bool {
-fn (mut m ImageManager) refresh_image(base_image string) ! {
+	if base_image !in m.timestamps
-	if base_image in m.timestamps
+		|| m.timestamps[base_image].add_seconds(m.max_image_age) <= time.now() {
-		&& m.timestamps[base_image].add_seconds(m.max_image_age) > time.now() {
+		return false
-		return
 	}
 
+	// It's possible the image has been removed by some external event, so we
+	// check whether it actually exists as well.
+	mut dd := docker.new_conn() or { return false }
+
+	defer {
+		dd.close() or {}
+	}
+
+	dd.image_inspect(m.images[base_image].last()) or {
+		// Image doesn't exist, so we stop tracking it
+		if err.code() == 404 {
+			m.images[base_image].delete_last()
+			m.timestamps.delete(base_image)
+		}
+
+		// If the inspect fails, it's either because the image doesn't exist or
+		// because of some other error. Either we can't know *for certain* that
+		// the image exists, so we return false.
+		return false
+	}
+
+	return true
+}
+
+// refresh_image builds a new builder image from the given base image. This
+// function should only be called if `up_to_date` return false.
+fn (mut m ImageManager) refresh_image(base_image string) ! {
 	// TODO use better image tags for built images
 	new_image := build.create_build_image(base_image) or {
 		return error('Failed to build builder image from base image $base_image')