From 73a1221bb5679ad8ef638844d7c94232c6002147 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 2 Feb 2023 16:14:26 +0100 Subject: [PATCH 1/3] wip --- include/vieter_job_queue.h | 14 +++++++++++--- src/job-queue/vieter_job_queue_internal.h | 11 +++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 src/job-queue/vieter_job_queue_internal.h diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h index 6c21da0..bc2f5e5 100644 --- a/include/vieter_job_queue.h +++ b/include/vieter_job_queue.h @@ -12,9 +12,9 @@ * other things. */ typedef enum vieter_job_state { - vieter_job_queued = 0, - vieter_job_dispatched = 1, - vieter_job_finished = 2 + vieter_job_state_queued = 0, + vieter_job_state_ready = 1, + vieter_job_state_build_finished = 2 } vieter_job_state; // This macro should be kept in sync with the above enum @@ -27,6 +27,14 @@ typedef struct vieter_job { bool single; vieter_job_state state; uint64_t state_transition_times[VIETER_JOB_STATES]; + bool dispatched; + void *build_config; } vieter_job; +typedef struct vieter_job_queue vieter_job_queue; + +vieter_job_queue *vieter_job_queue_init(); + +void vieter_job_queue_insert(int id); + #endif diff --git a/src/job-queue/vieter_job_queue_internal.h b/src/job-queue/vieter_job_queue_internal.h new file mode 100644 index 0000000..cbad8f8 --- /dev/null +++ b/src/job-queue/vieter_job_queue_internal.h @@ -0,0 +1,11 @@ +#ifndef VIETER_JOB_QUEUE_INTERNAL +#define VIETER_JOB_QUEUE_INTERNAL + +#include "vieter_tree.h" + +struct vieter_job_queue { + vieter_tree *tree; + +}; + +#endif From f2f69b47f850c511e3c8e1a37f14a8c2c6127d70 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 2 Feb 2023 16:14:26 +0100 Subject: [PATCH 2/3] chore: format internal header files --- .editorconfig | 5 ++++ Makefile | 5 ++-- include/vieter_job_queue.h | 30 ++++++++++++-------- src/heap/vieter_heap_tree.h | 23 +++++++-------- src/job-queue/vieter_job_queue_internal.h | 10 +++++++ src/tree/vieter_tree_internal.h | 6 ++-- src/tree/vieter_tree_node.h | 34 ++++++++++++++--------- 7 files changed, 73 insertions(+), 40 deletions(-) create mode 100644 .editorconfig create mode 100644 src/job-queue/vieter_job_queue_internal.h diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..a2eb55d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[*.{c,h}] +indent_style = space +indent_size = 2 diff --git a/Makefile b/Makefile index 665c411..217477c 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ LIB := $(BUILD_DIR)/$(LIB_FILENAME) SRCS != find '$(SRC_DIR)' -iname '*.c' SRCS_H != find $(INC_DIRS) -iname '*.h' +SRCS_H_INTERNAL != find $(SRC_DIR) -iname '*.h' SRCS_TEST != find '$(TEST_DIR)' -iname '*.c' OBJS := $(SRCS:%=$(BUILD_DIR)/%.o) @@ -92,11 +93,11 @@ $(BUILD_DIR)/$(TEST_DIR)/%.c.o: $(TEST_DIR)/%.c # =====MAINTENANCE===== .PHONY: lint lint: - clang-format -n --Werror $(SRCS) $(SRCS_H) + clang-format -n --Werror $(SRCS) $(SRCS_H) $(SRCS_H_INTERNAL) .PHONY: fmt fmt: - clang-format -i $(SRCS) $(SRCS_H) + clang-format -i $(SRCS) $(SRCS_H) $(SRCS_H_INTERNAL) .PHONY: clean clean: diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h index 6c21da0..fc02d7e 100644 --- a/include/vieter_job_queue.h +++ b/include/vieter_job_queue.h @@ -1,9 +1,9 @@ #ifndef VIETER_JOB_QUEUE #define VIETER_JOB_QUEUE -#include -#include #include "vieter_cron.h" +#include +#include /* * The order of these do not imply that they happen in this order. New states @@ -12,21 +12,29 @@ * other things. */ typedef enum vieter_job_state { - vieter_job_queued = 0, - vieter_job_dispatched = 1, - vieter_job_finished = 2 + vieter_job_state_queued = 0, + vieter_job_state_ready = 1, + vieter_job_state_build_finished = 2 } vieter_job_state; // This macro should be kept in sync with the above enum #define VIETER_JOB_STATES 3 typedef struct vieter_job { - int id; - uint64_t next_scheduled_time; - vieter_cron_expression *ce; - bool single; - vieter_job_state state; - uint64_t state_transition_times[VIETER_JOB_STATES]; + int id; + uint64_t next_scheduled_time; + vieter_cron_expression *ce; + bool single; + vieter_job_state state; + uint64_t state_transition_times[VIETER_JOB_STATES]; + bool dispatched; + void *build_config; } vieter_job; +typedef struct vieter_job_queue vieter_job_queue; + +vieter_job_queue *vieter_job_queue_init(); + +void vieter_job_queue_insert(int id); + #endif diff --git a/src/heap/vieter_heap_tree.h b/src/heap/vieter_heap_tree.h index 0a299db..615491a 100644 --- a/src/heap/vieter_heap_tree.h +++ b/src/heap/vieter_heap_tree.h @@ -5,16 +5,16 @@ #include typedef struct vieter_heap_node { - uint64_t key; - void *data; - struct vieter_heap_node *largest_order; - union { - // Roots point to next tree in the heap, other nodes point to their first - // neighbour. - struct vieter_heap_node *next_tree; - struct vieter_heap_node *next_largest_order; - } ptr; - uint8_t order; + uint64_t key; + void *data; + struct vieter_heap_node *largest_order; + union { + // Roots point to next tree in the heap, other nodes point to their first + // neighbour. + struct vieter_heap_node *next_tree; + struct vieter_heap_node *next_largest_order; + } ptr; + uint8_t order; } vieter_heap_node; /* @@ -36,7 +36,8 @@ void vieter_heap_tree_free(vieter_heap_node *root); * Given the roots of the smallest trees in two heaps, merge them into a single * large heap. */ -vieter_heap_node *vieter_heap_tree_merge(vieter_heap_node *root_a, vieter_heap_node *root_b); +vieter_heap_node *vieter_heap_tree_merge(vieter_heap_node *root_a, + vieter_heap_node *root_b); /* * Given the roots of two trees of the same order, merge them into a heap of one diff --git a/src/job-queue/vieter_job_queue_internal.h b/src/job-queue/vieter_job_queue_internal.h new file mode 100644 index 0000000..8d8ffc4 --- /dev/null +++ b/src/job-queue/vieter_job_queue_internal.h @@ -0,0 +1,10 @@ +#ifndef VIETER_JOB_QUEUE_INTERNAL +#define VIETER_JOB_QUEUE_INTERNAL + +#include "vieter_tree.h" + +struct vieter_job_queue { + vieter_tree *tree; +}; + +#endif diff --git a/src/tree/vieter_tree_internal.h b/src/tree/vieter_tree_internal.h index 855139f..1214572 100644 --- a/src/tree/vieter_tree_internal.h +++ b/src/tree/vieter_tree_internal.h @@ -16,7 +16,7 @@ struct vieter_tree { bool vieter_tree_validate(vieter_tree *tree); struct vieter_tree_iterator { - vieter_tree_node *current_node; - bool started; - bool done; + vieter_tree_node *current_node; + bool started; + bool done; }; diff --git a/src/tree/vieter_tree_node.h b/src/tree/vieter_tree_node.h index 3cf8e56..4db3cfe 100644 --- a/src/tree/vieter_tree_node.h +++ b/src/tree/vieter_tree_node.h @@ -2,8 +2,8 @@ #define VIETER_TREE_NODE #include "vieter_tree.h" -#include #include +#include typedef enum vieter_tree_node_flag { vieter_tree_node_black = ((uint8_t)1) << 0, @@ -13,11 +13,11 @@ typedef enum vieter_tree_node_flag { } vieter_tree_node_flag; typedef struct vieter_tree_node { - uint64_t key; - void *data; - struct vieter_tree_node *parent; - struct vieter_tree_node *children[2]; - uint8_t flags; + uint64_t key; + void *data; + struct vieter_tree_node *parent; + struct vieter_tree_node *children[2]; + uint8_t flags; } vieter_tree_node; /* @@ -33,27 +33,33 @@ void vieter_tree_node_free(vieter_tree_node *node); /* * Insert a new key into the given tree. */ -vieter_tree_error vieter_tree_node_insert(vieter_tree_node *root, uint64_t key, void *data); +vieter_tree_error vieter_tree_node_insert(vieter_tree_node *root, uint64_t key, + void *data); /* * Return the node representing the requested value. */ -vieter_tree_error vieter_tree_node_search_node(vieter_tree_node **out, vieter_tree_node *root, uint64_t key); +vieter_tree_error vieter_tree_node_search_node(vieter_tree_node **out, + vieter_tree_node *root, + uint64_t key); /* * Search for the data represented by the given key. */ -vieter_tree_error vieter_tree_node_search(void **out, vieter_tree_node *root, uint64_t key); +vieter_tree_error vieter_tree_node_search(void **out, vieter_tree_node *root, + uint64_t key); /* * Remove the data associated with the given key. */ -vieter_tree_error vieter_tree_node_remove(void **out, vieter_tree_node **root_ptr, uint64_t key); +vieter_tree_error +vieter_tree_node_remove(void **out, vieter_tree_node **root_ptr, uint64_t key); /* * Set a node's bit flag to the given value. */ -void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag, bool value); +void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag, + bool value); /* * Get whether a node's bit flag is set. @@ -70,12 +76,14 @@ void vieter_tree_node_add_child(vieter_tree_node *parent, uint64_t key, /* * Replace a node's children array. */ -void vieter_tree_node_set_children(vieter_tree_node *parent, vieter_tree_node **children); +void vieter_tree_node_set_children(vieter_tree_node *parent, + vieter_tree_node **children); /* * Set a node's left or right child to the given node. */ -void vieter_tree_node_set_child(vieter_tree_node *parent, vieter_tree_node *child, bool right); +void vieter_tree_node_set_child(vieter_tree_node *parent, + vieter_tree_node *child, bool right); /* * Return the in-order successor of the given node, or NULL if it's the last From 5159b9aead1f404b5cdf2000b664305467771915 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 23 Feb 2023 09:47:47 +0100 Subject: [PATCH 3/3] feat(job-queue): deesigning methods --- include/vieter_job_queue.h | 73 +++++++++++++++++++++++++++++++++----- src/job-queue/README.md | 27 ++++++++++++++ 2 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 src/job-queue/README.md diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h index fc02d7e..aeb7e1e 100644 --- a/include/vieter_job_queue.h +++ b/include/vieter_job_queue.h @@ -14,27 +14,82 @@ typedef enum vieter_job_state { vieter_job_state_queued = 0, vieter_job_state_ready = 1, - vieter_job_state_build_finished = 2 + vieter_job_state_build_finished = 2, + vieter_job_state_failed = 3 } vieter_job_state; // This macro should be kept in sync with the above enum -#define VIETER_JOB_STATES 3 +#define VIETER_JOB_STATES 4 +/* + * Struct storing a report for why a certain job failed to be processed in the + * given state. + */ +typedef struct vieter_job_failure_report { + vieter_job_state failed_state; + char *msg; +} vieter_job_failure_report; + +vieter_job_failure_report *vieter_job_failure_report_init(); + +void vieter_job_failure_report_free(vieter_job_failure_report **ptp); + +/* + * Represents a job currently being processed in the system. A job migrates + * between different states before finally being removed from the queue. + */ typedef struct vieter_job { - int id; + uint64_t id; uint64_t next_scheduled_time; - vieter_cron_expression *ce; - bool single; - vieter_job_state state; - uint64_t state_transition_times[VIETER_JOB_STATES]; - bool dispatched; + vieter_cron_expression *schedule; void *build_config; + vieter_job_failure_report *failure_report; + uint64_t state_transition_times[VIETER_JOB_STATES]; + vieter_job_state current_state; + bool single; + bool dispatched; } vieter_job; +/* + * Allocate a new vieter_job object. + */ +vieter_job *vieter_job_init(); + +void vieter_job_free(vieter_job **ptp); + +/* + * Represents the actual queue managing the list of jobs. + */ typedef struct vieter_job_queue vieter_job_queue; +typedef enum vieter_job_queue_error { + vieter_job_queue_ok = 0, + vieter_job_queue_not_found = 1 +} vieter_job_queue_error; + +/* + * Allocate and initialize a new job queue. + */ vieter_job_queue *vieter_job_queue_init(); -void vieter_job_queue_insert(int id); +void vieter_job_queue_free(vieter_job_queue **ptp); + +/* + * Insert the given job into the system. + */ +vieter_job_queue_error vieter_job_queue_insert(vieter_job *job); + +/* + * Dispatch the job with the given id, returning the pointer to the job. + * Dispatching a job removes it from its respective state's queue. + */ +vieter_job_queue_error vieter_job_queue_dispatch(vieter_job **out, uint64_t id); + +/* + * Transition the job with the given id to the new state. This sets the + * job's dispatch flag to false, and adds it to the new state's queue. + */ +vieter_job_queue_error vieter_job_queue_transition(uint64_t id, + vieter_job_state new_state); #endif diff --git a/src/job-queue/README.md b/src/job-queue/README.md new file mode 100644 index 0000000..8fa9c94 --- /dev/null +++ b/src/job-queue/README.md @@ -0,0 +1,27 @@ +The goal of this job queue design is to process jobs in order, with each job +moving through a pipeline of tasks that need to be completed. + +At any given time, a job is in one of a few given states, e.g. "queued". These +states are explained below. Along with this, each job also has a "dispatched" +flag. If this flag is set to true, it means this job is currently being +processed. "Being processed" could mean anything; it depends entirely on the +state a job's in. While a job is dispatched, it is no longer present in the +priority queue of its respective state. + +## Job + +A job describes a scheduled build as it moves through the pipeline of states. +The job queue datastructure keeps track of all jobs in a central red-black +binary tree. For each state, a priority queue tracks in what order jobs should +be processed. + +## States + +* `queued`: a job that's in the job queue but does not yet need to be executed + (as defined by its timestamp) +* `ready`: a job that's scheduled for building, with all preprocessing tasks + fulfilled. +* `build_finished`: a job whose build has finished, and is waiting for any + post-build tasks. +* `failed`: a job whose processing failed at some point. Jobs in this state + include a failure report that describes in what state they failed, and why.