From 75a09cc0f99bda2e72f3035b00ee1a2453ac9f00 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 2 Feb 2023 10:10:03 +0100 Subject: [PATCH 01/14] feat(job-queue): first draft of job struct --- include/vieter_job_queue.h | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 include/vieter_job_queue.h diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h new file mode 100644 index 0000000..6c21da0 --- /dev/null +++ b/include/vieter_job_queue.h @@ -0,0 +1,32 @@ +#ifndef VIETER_JOB_QUEUE +#define VIETER_JOB_QUEUE + +#include +#include +#include "vieter_cron.h" + +/* + * The order of these do not imply that they happen in this order. New states + * will just get added as consecutive numbers. Their values should be + * monotonically increasing values, as these will be used to index arrays, among + * other things. + */ +typedef enum vieter_job_state { + vieter_job_queued = 0, + vieter_job_dispatched = 1, + vieter_job_finished = 2 +} vieter_job_state; + +// This macro should be kept in sync with the above enum +#define VIETER_JOB_STATES 3 + +typedef struct vieter_job { + int id; + uint64_t next_scheduled_time; + vieter_cron_expression *ce; + bool single; + vieter_job_state state; + uint64_t state_transition_times[VIETER_JOB_STATES]; +} vieter_job; + +#endif From a6ed89d8cd9a99174a7d4cc28264c30f55aa2538 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 2 Feb 2023 14:10:35 +0100 Subject: [PATCH 02/14] feat(tree): add tree iterator --- include/vieter_tree.h | 24 +++++++++++++++- src/tree/vieter_tree.c | 8 +----- src/tree/vieter_tree_internal.h | 12 ++++++++ src/tree/vieter_tree_iterator.c | 49 +++++++++++++++++++++++++++++++++ src/tree/vieter_tree_node.c | 19 +++++++++++++ src/tree/vieter_tree_node.h | 6 ++++ test/tree/test_tree.c | 15 ++++++++-- 7 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 src/tree/vieter_tree_iterator.c diff --git a/include/vieter_tree.h b/include/vieter_tree.h index d3abd2f..2079eb9 100644 --- a/include/vieter_tree.h +++ b/include/vieter_tree.h @@ -9,7 +9,8 @@ typedef struct vieter_tree vieter_tree; typedef enum vieter_tree_error { vieter_tree_ok = 0, vieter_tree_already_present = 1, - vieter_tree_not_present = 2 + vieter_tree_not_present = 2, + vieter_tree_iterator_done = 3 } vieter_tree_error; /* @@ -45,4 +46,25 @@ vieter_tree_error vieter_tree_remove(void **out, vieter_tree *tree, */ uint64_t vieter_tree_size(vieter_tree *tree); +/* + * An iterator that can be used to traverse a tree in-order. + */ +typedef struct vieter_tree_iterator vieter_tree_iterator; + +/* + * Allocate and initialize a new iterator from the given tree. + */ +vieter_tree_iterator *vieter_tree_iterator_from(vieter_tree *tree); + +/* + * Free an iterator. + */ +void vieter_tree_iterator_free(vieter_tree_iterator **ptp); + +/* + * Return the iterator's next element. + */ +vieter_tree_error vieter_tree_iterator_next(void **out, + vieter_tree_iterator *iter); + #endif diff --git a/src/tree/vieter_tree.c b/src/tree/vieter_tree.c index eef2aef..84a1a26 100644 --- a/src/tree/vieter_tree.c +++ b/src/tree/vieter_tree.c @@ -1,11 +1,5 @@ -#include "vieter_tree_internal.h" -#include "vieter_tree_node.h" #include "vieter_tree_balancing.h" - -struct vieter_tree { - uint64_t size; - vieter_tree_node *root; -}; +#include "vieter_tree_internal.h" vieter_tree *vieter_tree_init() { return calloc(1, sizeof(vieter_tree)); } diff --git a/src/tree/vieter_tree_internal.h b/src/tree/vieter_tree_internal.h index 670d45b..855139f 100644 --- a/src/tree/vieter_tree_internal.h +++ b/src/tree/vieter_tree_internal.h @@ -1,10 +1,22 @@ #include "vieter_tree.h" +#include "vieter_tree_node.h" #include +struct vieter_tree { + uint64_t size; + vieter_tree_node *root; +}; + /* * Inspect whether the tree is still a valid red-black-tree. * * @return true if valid, false otherwise. */ bool vieter_tree_validate(vieter_tree *tree); + +struct vieter_tree_iterator { + vieter_tree_node *current_node; + bool started; + bool done; +}; diff --git a/src/tree/vieter_tree_iterator.c b/src/tree/vieter_tree_iterator.c new file mode 100644 index 0000000..4eefe02 --- /dev/null +++ b/src/tree/vieter_tree_iterator.c @@ -0,0 +1,49 @@ +#include "vieter_tree.h" +#include "vieter_tree_internal.h" + +vieter_tree_iterator *vieter_tree_iterator_from(vieter_tree *tree) { + vieter_tree_iterator *iter = calloc(1, sizeof(vieter_tree_iterator)); + + // An empty tree's iterator will be done immediately + if (tree->size == 0) { + iter->started = true; + iter->done = true; + + return iter; + } + + iter->current_node = tree->root; + + return iter; +} + +vieter_tree_error vieter_tree_iterator_next(void **out, + vieter_tree_iterator *iter) { + if (iter->done) { + return vieter_tree_iterator_done; + } + + if (!iter->started) { + while (iter->current_node->children[0] != NULL) { + iter->current_node = iter->current_node->children[0]; + } + + iter->started = true; + } + + *out = iter->current_node->data; + + iter->current_node = vieter_tree_node_next(iter->current_node); + + if (iter->current_node == NULL) { + iter->done = true; + } + + return vieter_tree_ok; +} + +void vieter_tree_iterator_free(vieter_tree_iterator **ptp) { + free(*ptp); + + *ptp = NULL; +} diff --git a/src/tree/vieter_tree_node.c b/src/tree/vieter_tree_node.c index e58092e..0a8e658 100644 --- a/src/tree/vieter_tree_node.c +++ b/src/tree/vieter_tree_node.c @@ -150,3 +150,22 @@ void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag, bool vieter_tree_node_get(vieter_tree_node *node, vieter_tree_node_flag flag) { return (node->flags & flag) != 0; } + +vieter_tree_node *vieter_tree_node_next(vieter_tree_node *node) { + if (node->children[1] != NULL) { + node = node->children[1]; + + while (node->children[0] != NULL) { + node = node->children[0]; + } + + return node; + } + + while (node->parent != NULL && + vieter_tree_node_get(node, vieter_tree_node_right)) { + node = node->parent; + } + + return node->parent; +} diff --git a/src/tree/vieter_tree_node.h b/src/tree/vieter_tree_node.h index 3b1c169..3cf8e56 100644 --- a/src/tree/vieter_tree_node.h +++ b/src/tree/vieter_tree_node.h @@ -77,4 +77,10 @@ void vieter_tree_node_set_children(vieter_tree_node *parent, vieter_tree_node ** */ void vieter_tree_node_set_child(vieter_tree_node *parent, vieter_tree_node *child, bool right); +/* + * Return the in-order successor of the given node, or NULL if it's the last + * node in the tree. + */ +vieter_tree_node *vieter_tree_node_next(vieter_tree_node *node); + #endif diff --git a/test/tree/test_tree.c b/test/tree/test_tree.c index 2ab6870..10235cb 100644 --- a/test/tree/test_tree.c +++ b/test/tree/test_tree.c @@ -16,19 +16,30 @@ void test_insert() { vieter_tree *tree = vieter_tree_init(); for (uint64_t i = 0; i < 250; i++) { - TEST_CHECK(vieter_tree_insert(tree, i, NULL) == vieter_tree_ok); + TEST_CHECK(vieter_tree_insert(tree, i, (void *)i) == vieter_tree_ok); TEST_SIZE(tree, i + 1); TEST_CHECK(vieter_tree_validate(tree)); } - void *out; + vieter_tree_iterator *iter = vieter_tree_iterator_from(tree); + + void *out = NULL; for (uint64_t i = 0; i < 250; i++) { TEST_CHECK(vieter_tree_search(&out, tree, i) == vieter_tree_ok); + TEST_CHECK(out == (void *)i); TEST_CHECK(vieter_tree_insert(tree, i, NULL) == vieter_tree_already_present); TEST_CHECK(vieter_tree_search(&out, tree, i) == vieter_tree_ok); + + out = NULL; + + TEST_CHECK(vieter_tree_iterator_next(&out, iter) == vieter_tree_ok); + TEST_CHECK(out == (void *)i); } + TEST_CHECK(vieter_tree_iterator_next(&out, iter) == vieter_tree_iterator_done); + + vieter_tree_iterator_free(&iter); vieter_tree_free(tree); } From d9402ced54c44961b70e6559321e6d42fb9b14f3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 2 Feb 2023 15:30:57 +0100 Subject: [PATCH 03/14] test(tree): add random insert test --- test/tree/test_tree_random.c | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 test/tree/test_tree_random.c diff --git a/test/tree/test_tree_random.c b/test/tree/test_tree_random.c new file mode 100644 index 0000000..d1b9ab0 --- /dev/null +++ b/test/tree/test_tree_random.c @@ -0,0 +1,59 @@ +#include "acutest.h" +#include "vieter_tree_internal.h" + +#define TEST_SIZE(tree, size) \ + TEST_CHECK(vieter_tree_size(tree) == size); \ + TEST_MSG("Size: %zu", vieter_tree_size(tree)) + +int uint64_t_compare(const void *a, const void *b) { + if ((*(uint64_t *)a) < (*(uint64_t *)b)) { + return -1; + } else if ((*(uint64_t *)a) > (*(uint64_t *)b)) { + return 1; + } else { + return 0; + } +} + +void test_insert_random() { + const uint64_t n = 1000; + srand(0); + + vieter_tree *tree = vieter_tree_init(); + + uint64_t *numbers = malloc(n * sizeof(uint64_t)); + uint64_t num; + void *out = NULL; + + for (uint64_t i = 0; i < n; i++) { + num = rand(); + vieter_tree_insert(tree, num, (void *)num); + TEST_SIZE(tree, i + 1); + TEST_CHECK(vieter_tree_validate(tree)); + TEST_CHECK(vieter_tree_search(&out, tree, num) == vieter_tree_ok); + TEST_CHECK(out == (void *)num); + + out = NULL; + + numbers[i] = num; + } + + qsort(numbers, n, sizeof(uint64_t), uint64_t_compare); + + vieter_tree_iterator *iter = vieter_tree_iterator_from(tree); + out = NULL; + + for (uint64_t i = 0; i < n; i++) { + TEST_CHECK(vieter_tree_iterator_next(&out, iter) == vieter_tree_ok); + TEST_CHECK(out == (void *)numbers[i]); + } + + free(numbers); + vieter_tree_iterator_free(&iter); + vieter_tree_free(tree); +} + +TEST_LIST = { + {"tree insert random", test_insert_random}, + {NULL, NULL} +}; From 274e493fd32e3b1934aa255307ce6181d32c5bbf Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 2 Feb 2023 14:10:35 +0100 Subject: [PATCH 04/14] feat(tree): add tree iterator --- include/vieter_tree.h | 24 +++++++++++++++- src/tree/vieter_tree.c | 8 +----- src/tree/vieter_tree_internal.h | 12 ++++++++ src/tree/vieter_tree_iterator.c | 49 +++++++++++++++++++++++++++++++++ src/tree/vieter_tree_node.c | 19 +++++++++++++ src/tree/vieter_tree_node.h | 6 ++++ test/tree/test_tree.c | 15 ++++++++-- 7 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 src/tree/vieter_tree_iterator.c diff --git a/include/vieter_tree.h b/include/vieter_tree.h index d3abd2f..2079eb9 100644 --- a/include/vieter_tree.h +++ b/include/vieter_tree.h @@ -9,7 +9,8 @@ typedef struct vieter_tree vieter_tree; typedef enum vieter_tree_error { vieter_tree_ok = 0, vieter_tree_already_present = 1, - vieter_tree_not_present = 2 + vieter_tree_not_present = 2, + vieter_tree_iterator_done = 3 } vieter_tree_error; /* @@ -45,4 +46,25 @@ vieter_tree_error vieter_tree_remove(void **out, vieter_tree *tree, */ uint64_t vieter_tree_size(vieter_tree *tree); +/* + * An iterator that can be used to traverse a tree in-order. + */ +typedef struct vieter_tree_iterator vieter_tree_iterator; + +/* + * Allocate and initialize a new iterator from the given tree. + */ +vieter_tree_iterator *vieter_tree_iterator_from(vieter_tree *tree); + +/* + * Free an iterator. + */ +void vieter_tree_iterator_free(vieter_tree_iterator **ptp); + +/* + * Return the iterator's next element. + */ +vieter_tree_error vieter_tree_iterator_next(void **out, + vieter_tree_iterator *iter); + #endif diff --git a/src/tree/vieter_tree.c b/src/tree/vieter_tree.c index eef2aef..84a1a26 100644 --- a/src/tree/vieter_tree.c +++ b/src/tree/vieter_tree.c @@ -1,11 +1,5 @@ -#include "vieter_tree_internal.h" -#include "vieter_tree_node.h" #include "vieter_tree_balancing.h" - -struct vieter_tree { - uint64_t size; - vieter_tree_node *root; -}; +#include "vieter_tree_internal.h" vieter_tree *vieter_tree_init() { return calloc(1, sizeof(vieter_tree)); } diff --git a/src/tree/vieter_tree_internal.h b/src/tree/vieter_tree_internal.h index 670d45b..855139f 100644 --- a/src/tree/vieter_tree_internal.h +++ b/src/tree/vieter_tree_internal.h @@ -1,10 +1,22 @@ #include "vieter_tree.h" +#include "vieter_tree_node.h" #include +struct vieter_tree { + uint64_t size; + vieter_tree_node *root; +}; + /* * Inspect whether the tree is still a valid red-black-tree. * * @return true if valid, false otherwise. */ bool vieter_tree_validate(vieter_tree *tree); + +struct vieter_tree_iterator { + vieter_tree_node *current_node; + bool started; + bool done; +}; diff --git a/src/tree/vieter_tree_iterator.c b/src/tree/vieter_tree_iterator.c new file mode 100644 index 0000000..4eefe02 --- /dev/null +++ b/src/tree/vieter_tree_iterator.c @@ -0,0 +1,49 @@ +#include "vieter_tree.h" +#include "vieter_tree_internal.h" + +vieter_tree_iterator *vieter_tree_iterator_from(vieter_tree *tree) { + vieter_tree_iterator *iter = calloc(1, sizeof(vieter_tree_iterator)); + + // An empty tree's iterator will be done immediately + if (tree->size == 0) { + iter->started = true; + iter->done = true; + + return iter; + } + + iter->current_node = tree->root; + + return iter; +} + +vieter_tree_error vieter_tree_iterator_next(void **out, + vieter_tree_iterator *iter) { + if (iter->done) { + return vieter_tree_iterator_done; + } + + if (!iter->started) { + while (iter->current_node->children[0] != NULL) { + iter->current_node = iter->current_node->children[0]; + } + + iter->started = true; + } + + *out = iter->current_node->data; + + iter->current_node = vieter_tree_node_next(iter->current_node); + + if (iter->current_node == NULL) { + iter->done = true; + } + + return vieter_tree_ok; +} + +void vieter_tree_iterator_free(vieter_tree_iterator **ptp) { + free(*ptp); + + *ptp = NULL; +} diff --git a/src/tree/vieter_tree_node.c b/src/tree/vieter_tree_node.c index e58092e..0a8e658 100644 --- a/src/tree/vieter_tree_node.c +++ b/src/tree/vieter_tree_node.c @@ -150,3 +150,22 @@ void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag, bool vieter_tree_node_get(vieter_tree_node *node, vieter_tree_node_flag flag) { return (node->flags & flag) != 0; } + +vieter_tree_node *vieter_tree_node_next(vieter_tree_node *node) { + if (node->children[1] != NULL) { + node = node->children[1]; + + while (node->children[0] != NULL) { + node = node->children[0]; + } + + return node; + } + + while (node->parent != NULL && + vieter_tree_node_get(node, vieter_tree_node_right)) { + node = node->parent; + } + + return node->parent; +} diff --git a/src/tree/vieter_tree_node.h b/src/tree/vieter_tree_node.h index 3b1c169..3cf8e56 100644 --- a/src/tree/vieter_tree_node.h +++ b/src/tree/vieter_tree_node.h @@ -77,4 +77,10 @@ void vieter_tree_node_set_children(vieter_tree_node *parent, vieter_tree_node ** */ void vieter_tree_node_set_child(vieter_tree_node *parent, vieter_tree_node *child, bool right); +/* + * Return the in-order successor of the given node, or NULL if it's the last + * node in the tree. + */ +vieter_tree_node *vieter_tree_node_next(vieter_tree_node *node); + #endif diff --git a/test/tree/test_tree.c b/test/tree/test_tree.c index 2ab6870..10235cb 100644 --- a/test/tree/test_tree.c +++ b/test/tree/test_tree.c @@ -16,19 +16,30 @@ void test_insert() { vieter_tree *tree = vieter_tree_init(); for (uint64_t i = 0; i < 250; i++) { - TEST_CHECK(vieter_tree_insert(tree, i, NULL) == vieter_tree_ok); + TEST_CHECK(vieter_tree_insert(tree, i, (void *)i) == vieter_tree_ok); TEST_SIZE(tree, i + 1); TEST_CHECK(vieter_tree_validate(tree)); } - void *out; + vieter_tree_iterator *iter = vieter_tree_iterator_from(tree); + + void *out = NULL; for (uint64_t i = 0; i < 250; i++) { TEST_CHECK(vieter_tree_search(&out, tree, i) == vieter_tree_ok); + TEST_CHECK(out == (void *)i); TEST_CHECK(vieter_tree_insert(tree, i, NULL) == vieter_tree_already_present); TEST_CHECK(vieter_tree_search(&out, tree, i) == vieter_tree_ok); + + out = NULL; + + TEST_CHECK(vieter_tree_iterator_next(&out, iter) == vieter_tree_ok); + TEST_CHECK(out == (void *)i); } + TEST_CHECK(vieter_tree_iterator_next(&out, iter) == vieter_tree_iterator_done); + + vieter_tree_iterator_free(&iter); vieter_tree_free(tree); } From f3d5ad1c68578e4f7fd00b7956146b96ddc746d2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 2 Feb 2023 15:30:57 +0100 Subject: [PATCH 05/14] test(tree): add random insert test --- test/tree/test_tree_random.c | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 test/tree/test_tree_random.c diff --git a/test/tree/test_tree_random.c b/test/tree/test_tree_random.c new file mode 100644 index 0000000..d1b9ab0 --- /dev/null +++ b/test/tree/test_tree_random.c @@ -0,0 +1,59 @@ +#include "acutest.h" +#include "vieter_tree_internal.h" + +#define TEST_SIZE(tree, size) \ + TEST_CHECK(vieter_tree_size(tree) == size); \ + TEST_MSG("Size: %zu", vieter_tree_size(tree)) + +int uint64_t_compare(const void *a, const void *b) { + if ((*(uint64_t *)a) < (*(uint64_t *)b)) { + return -1; + } else if ((*(uint64_t *)a) > (*(uint64_t *)b)) { + return 1; + } else { + return 0; + } +} + +void test_insert_random() { + const uint64_t n = 1000; + srand(0); + + vieter_tree *tree = vieter_tree_init(); + + uint64_t *numbers = malloc(n * sizeof(uint64_t)); + uint64_t num; + void *out = NULL; + + for (uint64_t i = 0; i < n; i++) { + num = rand(); + vieter_tree_insert(tree, num, (void *)num); + TEST_SIZE(tree, i + 1); + TEST_CHECK(vieter_tree_validate(tree)); + TEST_CHECK(vieter_tree_search(&out, tree, num) == vieter_tree_ok); + TEST_CHECK(out == (void *)num); + + out = NULL; + + numbers[i] = num; + } + + qsort(numbers, n, sizeof(uint64_t), uint64_t_compare); + + vieter_tree_iterator *iter = vieter_tree_iterator_from(tree); + out = NULL; + + for (uint64_t i = 0; i < n; i++) { + TEST_CHECK(vieter_tree_iterator_next(&out, iter) == vieter_tree_ok); + TEST_CHECK(out == (void *)numbers[i]); + } + + free(numbers); + vieter_tree_iterator_free(&iter); + vieter_tree_free(tree); +} + +TEST_LIST = { + {"tree insert random", test_insert_random}, + {NULL, NULL} +}; From 38e84afd1d10606aec18711f66464140ae2e5aba Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 23 Feb 2023 10:13:58 +0100 Subject: [PATCH 06/14] fix: also lint internal header files --- Makefile | 8 +------- src/heap/vieter_heap_tree.h | 23 +++++++++++----------- src/tree/vieter_tree_internal.h | 6 +++--- src/tree/vieter_tree_node.h | 34 ++++++++++++++++++++------------- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index 665c411..fadf89d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ INC_DIRS ?= include LIB := $(BUILD_DIR)/$(LIB_FILENAME) SRCS != find '$(SRC_DIR)' -iname '*.c' -SRCS_H != find $(INC_DIRS) -iname '*.h' +SRCS_H != find $(INC_DIRS) '$(SRC_DIR)' -iname '*.h' SRCS_TEST != find '$(TEST_DIR)' -iname '*.c' OBJS := $(SRCS:%=$(BUILD_DIR)/%.o) @@ -66,15 +66,9 @@ $(TARGETS_TEST): test-%: % $(TARGETS_MEM_TEST): test-mem-%: % valgrind --tool=memcheck --error-exitcode=1 --track-origins=yes --leak-check=full ./$^ -test-mem: build-test - @ $(foreach bin,$(BINS_TEST),valgrind --tool=memcheck --error-exitcode=1 \ - --track-origins=yes --leak-check=full ./$(bin);) - .PHONY: build-test build-test: $(BINS_TEST) -# For simplicity, we link every object file to each of the test files. This -# might be changed later if this starts to become too slow. $(BINS_TEST): %: %.c.o $(LIB) $(CC) \ $^ -o $@ diff --git a/src/heap/vieter_heap_tree.h b/src/heap/vieter_heap_tree.h index 0a299db..615491a 100644 --- a/src/heap/vieter_heap_tree.h +++ b/src/heap/vieter_heap_tree.h @@ -5,16 +5,16 @@ #include typedef struct vieter_heap_node { - uint64_t key; - void *data; - struct vieter_heap_node *largest_order; - union { - // Roots point to next tree in the heap, other nodes point to their first - // neighbour. - struct vieter_heap_node *next_tree; - struct vieter_heap_node *next_largest_order; - } ptr; - uint8_t order; + uint64_t key; + void *data; + struct vieter_heap_node *largest_order; + union { + // Roots point to next tree in the heap, other nodes point to their first + // neighbour. + struct vieter_heap_node *next_tree; + struct vieter_heap_node *next_largest_order; + } ptr; + uint8_t order; } vieter_heap_node; /* @@ -36,7 +36,8 @@ void vieter_heap_tree_free(vieter_heap_node *root); * Given the roots of the smallest trees in two heaps, merge them into a single * large heap. */ -vieter_heap_node *vieter_heap_tree_merge(vieter_heap_node *root_a, vieter_heap_node *root_b); +vieter_heap_node *vieter_heap_tree_merge(vieter_heap_node *root_a, + vieter_heap_node *root_b); /* * Given the roots of two trees of the same order, merge them into a heap of one diff --git a/src/tree/vieter_tree_internal.h b/src/tree/vieter_tree_internal.h index 855139f..1214572 100644 --- a/src/tree/vieter_tree_internal.h +++ b/src/tree/vieter_tree_internal.h @@ -16,7 +16,7 @@ struct vieter_tree { bool vieter_tree_validate(vieter_tree *tree); struct vieter_tree_iterator { - vieter_tree_node *current_node; - bool started; - bool done; + vieter_tree_node *current_node; + bool started; + bool done; }; diff --git a/src/tree/vieter_tree_node.h b/src/tree/vieter_tree_node.h index 3cf8e56..4db3cfe 100644 --- a/src/tree/vieter_tree_node.h +++ b/src/tree/vieter_tree_node.h @@ -2,8 +2,8 @@ #define VIETER_TREE_NODE #include "vieter_tree.h" -#include #include +#include typedef enum vieter_tree_node_flag { vieter_tree_node_black = ((uint8_t)1) << 0, @@ -13,11 +13,11 @@ typedef enum vieter_tree_node_flag { } vieter_tree_node_flag; typedef struct vieter_tree_node { - uint64_t key; - void *data; - struct vieter_tree_node *parent; - struct vieter_tree_node *children[2]; - uint8_t flags; + uint64_t key; + void *data; + struct vieter_tree_node *parent; + struct vieter_tree_node *children[2]; + uint8_t flags; } vieter_tree_node; /* @@ -33,27 +33,33 @@ void vieter_tree_node_free(vieter_tree_node *node); /* * Insert a new key into the given tree. */ -vieter_tree_error vieter_tree_node_insert(vieter_tree_node *root, uint64_t key, void *data); +vieter_tree_error vieter_tree_node_insert(vieter_tree_node *root, uint64_t key, + void *data); /* * Return the node representing the requested value. */ -vieter_tree_error vieter_tree_node_search_node(vieter_tree_node **out, vieter_tree_node *root, uint64_t key); +vieter_tree_error vieter_tree_node_search_node(vieter_tree_node **out, + vieter_tree_node *root, + uint64_t key); /* * Search for the data represented by the given key. */ -vieter_tree_error vieter_tree_node_search(void **out, vieter_tree_node *root, uint64_t key); +vieter_tree_error vieter_tree_node_search(void **out, vieter_tree_node *root, + uint64_t key); /* * Remove the data associated with the given key. */ -vieter_tree_error vieter_tree_node_remove(void **out, vieter_tree_node **root_ptr, uint64_t key); +vieter_tree_error +vieter_tree_node_remove(void **out, vieter_tree_node **root_ptr, uint64_t key); /* * Set a node's bit flag to the given value. */ -void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag, bool value); +void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag, + bool value); /* * Get whether a node's bit flag is set. @@ -70,12 +76,14 @@ void vieter_tree_node_add_child(vieter_tree_node *parent, uint64_t key, /* * Replace a node's children array. */ -void vieter_tree_node_set_children(vieter_tree_node *parent, vieter_tree_node **children); +void vieter_tree_node_set_children(vieter_tree_node *parent, + vieter_tree_node **children); /* * Set a node's left or right child to the given node. */ -void vieter_tree_node_set_child(vieter_tree_node *parent, vieter_tree_node *child, bool right); +void vieter_tree_node_set_child(vieter_tree_node *parent, + vieter_tree_node *child, bool right); /* * Return the in-order successor of the given node, or NULL if it's the last From f2f69b47f850c511e3c8e1a37f14a8c2c6127d70 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 2 Feb 2023 16:14:26 +0100 Subject: [PATCH 07/14] chore: format internal header files --- .editorconfig | 5 ++++ Makefile | 5 ++-- include/vieter_job_queue.h | 30 ++++++++++++-------- src/heap/vieter_heap_tree.h | 23 +++++++-------- src/job-queue/vieter_job_queue_internal.h | 10 +++++++ src/tree/vieter_tree_internal.h | 6 ++-- src/tree/vieter_tree_node.h | 34 ++++++++++++++--------- 7 files changed, 73 insertions(+), 40 deletions(-) create mode 100644 .editorconfig create mode 100644 src/job-queue/vieter_job_queue_internal.h diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..a2eb55d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[*.{c,h}] +indent_style = space +indent_size = 2 diff --git a/Makefile b/Makefile index 665c411..217477c 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ LIB := $(BUILD_DIR)/$(LIB_FILENAME) SRCS != find '$(SRC_DIR)' -iname '*.c' SRCS_H != find $(INC_DIRS) -iname '*.h' +SRCS_H_INTERNAL != find $(SRC_DIR) -iname '*.h' SRCS_TEST != find '$(TEST_DIR)' -iname '*.c' OBJS := $(SRCS:%=$(BUILD_DIR)/%.o) @@ -92,11 +93,11 @@ $(BUILD_DIR)/$(TEST_DIR)/%.c.o: $(TEST_DIR)/%.c # =====MAINTENANCE===== .PHONY: lint lint: - clang-format -n --Werror $(SRCS) $(SRCS_H) + clang-format -n --Werror $(SRCS) $(SRCS_H) $(SRCS_H_INTERNAL) .PHONY: fmt fmt: - clang-format -i $(SRCS) $(SRCS_H) + clang-format -i $(SRCS) $(SRCS_H) $(SRCS_H_INTERNAL) .PHONY: clean clean: diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h index 6c21da0..fc02d7e 100644 --- a/include/vieter_job_queue.h +++ b/include/vieter_job_queue.h @@ -1,9 +1,9 @@ #ifndef VIETER_JOB_QUEUE #define VIETER_JOB_QUEUE -#include -#include #include "vieter_cron.h" +#include +#include /* * The order of these do not imply that they happen in this order. New states @@ -12,21 +12,29 @@ * other things. */ typedef enum vieter_job_state { - vieter_job_queued = 0, - vieter_job_dispatched = 1, - vieter_job_finished = 2 + vieter_job_state_queued = 0, + vieter_job_state_ready = 1, + vieter_job_state_build_finished = 2 } vieter_job_state; // This macro should be kept in sync with the above enum #define VIETER_JOB_STATES 3 typedef struct vieter_job { - int id; - uint64_t next_scheduled_time; - vieter_cron_expression *ce; - bool single; - vieter_job_state state; - uint64_t state_transition_times[VIETER_JOB_STATES]; + int id; + uint64_t next_scheduled_time; + vieter_cron_expression *ce; + bool single; + vieter_job_state state; + uint64_t state_transition_times[VIETER_JOB_STATES]; + bool dispatched; + void *build_config; } vieter_job; +typedef struct vieter_job_queue vieter_job_queue; + +vieter_job_queue *vieter_job_queue_init(); + +void vieter_job_queue_insert(int id); + #endif diff --git a/src/heap/vieter_heap_tree.h b/src/heap/vieter_heap_tree.h index 0a299db..615491a 100644 --- a/src/heap/vieter_heap_tree.h +++ b/src/heap/vieter_heap_tree.h @@ -5,16 +5,16 @@ #include typedef struct vieter_heap_node { - uint64_t key; - void *data; - struct vieter_heap_node *largest_order; - union { - // Roots point to next tree in the heap, other nodes point to their first - // neighbour. - struct vieter_heap_node *next_tree; - struct vieter_heap_node *next_largest_order; - } ptr; - uint8_t order; + uint64_t key; + void *data; + struct vieter_heap_node *largest_order; + union { + // Roots point to next tree in the heap, other nodes point to their first + // neighbour. + struct vieter_heap_node *next_tree; + struct vieter_heap_node *next_largest_order; + } ptr; + uint8_t order; } vieter_heap_node; /* @@ -36,7 +36,8 @@ void vieter_heap_tree_free(vieter_heap_node *root); * Given the roots of the smallest trees in two heaps, merge them into a single * large heap. */ -vieter_heap_node *vieter_heap_tree_merge(vieter_heap_node *root_a, vieter_heap_node *root_b); +vieter_heap_node *vieter_heap_tree_merge(vieter_heap_node *root_a, + vieter_heap_node *root_b); /* * Given the roots of two trees of the same order, merge them into a heap of one diff --git a/src/job-queue/vieter_job_queue_internal.h b/src/job-queue/vieter_job_queue_internal.h new file mode 100644 index 0000000..8d8ffc4 --- /dev/null +++ b/src/job-queue/vieter_job_queue_internal.h @@ -0,0 +1,10 @@ +#ifndef VIETER_JOB_QUEUE_INTERNAL +#define VIETER_JOB_QUEUE_INTERNAL + +#include "vieter_tree.h" + +struct vieter_job_queue { + vieter_tree *tree; +}; + +#endif diff --git a/src/tree/vieter_tree_internal.h b/src/tree/vieter_tree_internal.h index 855139f..1214572 100644 --- a/src/tree/vieter_tree_internal.h +++ b/src/tree/vieter_tree_internal.h @@ -16,7 +16,7 @@ struct vieter_tree { bool vieter_tree_validate(vieter_tree *tree); struct vieter_tree_iterator { - vieter_tree_node *current_node; - bool started; - bool done; + vieter_tree_node *current_node; + bool started; + bool done; }; diff --git a/src/tree/vieter_tree_node.h b/src/tree/vieter_tree_node.h index 3cf8e56..4db3cfe 100644 --- a/src/tree/vieter_tree_node.h +++ b/src/tree/vieter_tree_node.h @@ -2,8 +2,8 @@ #define VIETER_TREE_NODE #include "vieter_tree.h" -#include #include +#include typedef enum vieter_tree_node_flag { vieter_tree_node_black = ((uint8_t)1) << 0, @@ -13,11 +13,11 @@ typedef enum vieter_tree_node_flag { } vieter_tree_node_flag; typedef struct vieter_tree_node { - uint64_t key; - void *data; - struct vieter_tree_node *parent; - struct vieter_tree_node *children[2]; - uint8_t flags; + uint64_t key; + void *data; + struct vieter_tree_node *parent; + struct vieter_tree_node *children[2]; + uint8_t flags; } vieter_tree_node; /* @@ -33,27 +33,33 @@ void vieter_tree_node_free(vieter_tree_node *node); /* * Insert a new key into the given tree. */ -vieter_tree_error vieter_tree_node_insert(vieter_tree_node *root, uint64_t key, void *data); +vieter_tree_error vieter_tree_node_insert(vieter_tree_node *root, uint64_t key, + void *data); /* * Return the node representing the requested value. */ -vieter_tree_error vieter_tree_node_search_node(vieter_tree_node **out, vieter_tree_node *root, uint64_t key); +vieter_tree_error vieter_tree_node_search_node(vieter_tree_node **out, + vieter_tree_node *root, + uint64_t key); /* * Search for the data represented by the given key. */ -vieter_tree_error vieter_tree_node_search(void **out, vieter_tree_node *root, uint64_t key); +vieter_tree_error vieter_tree_node_search(void **out, vieter_tree_node *root, + uint64_t key); /* * Remove the data associated with the given key. */ -vieter_tree_error vieter_tree_node_remove(void **out, vieter_tree_node **root_ptr, uint64_t key); +vieter_tree_error +vieter_tree_node_remove(void **out, vieter_tree_node **root_ptr, uint64_t key); /* * Set a node's bit flag to the given value. */ -void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag, bool value); +void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag, + bool value); /* * Get whether a node's bit flag is set. @@ -70,12 +76,14 @@ void vieter_tree_node_add_child(vieter_tree_node *parent, uint64_t key, /* * Replace a node's children array. */ -void vieter_tree_node_set_children(vieter_tree_node *parent, vieter_tree_node **children); +void vieter_tree_node_set_children(vieter_tree_node *parent, + vieter_tree_node **children); /* * Set a node's left or right child to the given node. */ -void vieter_tree_node_set_child(vieter_tree_node *parent, vieter_tree_node *child, bool right); +void vieter_tree_node_set_child(vieter_tree_node *parent, + vieter_tree_node *child, bool right); /* * Return the in-order successor of the given node, or NULL if it's the last From 5159b9aead1f404b5cdf2000b664305467771915 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 23 Feb 2023 09:47:47 +0100 Subject: [PATCH 08/14] feat(job-queue): deesigning methods --- include/vieter_job_queue.h | 73 +++++++++++++++++++++++++++++++++----- src/job-queue/README.md | 27 ++++++++++++++ 2 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 src/job-queue/README.md diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h index fc02d7e..aeb7e1e 100644 --- a/include/vieter_job_queue.h +++ b/include/vieter_job_queue.h @@ -14,27 +14,82 @@ typedef enum vieter_job_state { vieter_job_state_queued = 0, vieter_job_state_ready = 1, - vieter_job_state_build_finished = 2 + vieter_job_state_build_finished = 2, + vieter_job_state_failed = 3 } vieter_job_state; // This macro should be kept in sync with the above enum -#define VIETER_JOB_STATES 3 +#define VIETER_JOB_STATES 4 +/* + * Struct storing a report for why a certain job failed to be processed in the + * given state. + */ +typedef struct vieter_job_failure_report { + vieter_job_state failed_state; + char *msg; +} vieter_job_failure_report; + +vieter_job_failure_report *vieter_job_failure_report_init(); + +void vieter_job_failure_report_free(vieter_job_failure_report **ptp); + +/* + * Represents a job currently being processed in the system. A job migrates + * between different states before finally being removed from the queue. + */ typedef struct vieter_job { - int id; + uint64_t id; uint64_t next_scheduled_time; - vieter_cron_expression *ce; - bool single; - vieter_job_state state; - uint64_t state_transition_times[VIETER_JOB_STATES]; - bool dispatched; + vieter_cron_expression *schedule; void *build_config; + vieter_job_failure_report *failure_report; + uint64_t state_transition_times[VIETER_JOB_STATES]; + vieter_job_state current_state; + bool single; + bool dispatched; } vieter_job; +/* + * Allocate a new vieter_job object. + */ +vieter_job *vieter_job_init(); + +void vieter_job_free(vieter_job **ptp); + +/* + * Represents the actual queue managing the list of jobs. + */ typedef struct vieter_job_queue vieter_job_queue; +typedef enum vieter_job_queue_error { + vieter_job_queue_ok = 0, + vieter_job_queue_not_found = 1 +} vieter_job_queue_error; + +/* + * Allocate and initialize a new job queue. + */ vieter_job_queue *vieter_job_queue_init(); -void vieter_job_queue_insert(int id); +void vieter_job_queue_free(vieter_job_queue **ptp); + +/* + * Insert the given job into the system. + */ +vieter_job_queue_error vieter_job_queue_insert(vieter_job *job); + +/* + * Dispatch the job with the given id, returning the pointer to the job. + * Dispatching a job removes it from its respective state's queue. + */ +vieter_job_queue_error vieter_job_queue_dispatch(vieter_job **out, uint64_t id); + +/* + * Transition the job with the given id to the new state. This sets the + * job's dispatch flag to false, and adds it to the new state's queue. + */ +vieter_job_queue_error vieter_job_queue_transition(uint64_t id, + vieter_job_state new_state); #endif diff --git a/src/job-queue/README.md b/src/job-queue/README.md new file mode 100644 index 0000000..8fa9c94 --- /dev/null +++ b/src/job-queue/README.md @@ -0,0 +1,27 @@ +The goal of this job queue design is to process jobs in order, with each job +moving through a pipeline of tasks that need to be completed. + +At any given time, a job is in one of a few given states, e.g. "queued". These +states are explained below. Along with this, each job also has a "dispatched" +flag. If this flag is set to true, it means this job is currently being +processed. "Being processed" could mean anything; it depends entirely on the +state a job's in. While a job is dispatched, it is no longer present in the +priority queue of its respective state. + +## Job + +A job describes a scheduled build as it moves through the pipeline of states. +The job queue datastructure keeps track of all jobs in a central red-black +binary tree. For each state, a priority queue tracks in what order jobs should +be processed. + +## States + +* `queued`: a job that's in the job queue but does not yet need to be executed + (as defined by its timestamp) +* `ready`: a job that's scheduled for building, with all preprocessing tasks + fulfilled. +* `build_finished`: a job whose build has finished, and is waiting for any + post-build tasks. +* `failed`: a job whose processing failed at some point. Jobs in this state + include a failure report that describes in what state they failed, and why. From 0e63714de3943e388ba553a0b09b9a22660444a6 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 7 Mar 2023 11:55:12 +0100 Subject: [PATCH 09/14] feat(job-queue): initial implementation of functionality --- include/vieter_job_queue.h | 44 ++++++- src/job-queue/vieter_job_queue.c | 153 ++++++++++++++++++++++ src/job-queue/vieter_job_queue_internal.h | 3 + 3 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 src/job-queue/vieter_job_queue.c diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h index aeb7e1e..6cd748f 100644 --- a/include/vieter_job_queue.h +++ b/include/vieter_job_queue.h @@ -20,6 +20,8 @@ typedef enum vieter_job_state { // This macro should be kept in sync with the above enum #define VIETER_JOB_STATES 4 +#define VIETER_JOB_INITIAL_STATE vieter_job_state_queued +#define VIETER_JOB_FAILURE_STATE vieter_job_state_failed /* * Struct storing a report for why a certain job failed to be processed in the @@ -64,7 +66,10 @@ typedef struct vieter_job_queue vieter_job_queue; typedef enum vieter_job_queue_error { vieter_job_queue_ok = 0, - vieter_job_queue_not_found = 1 + vieter_job_queue_not_present = 1, + vieter_job_queue_already_present = 2, + vieter_job_queue_state_empty = 3, + vieter_job_queue_not_dispatched = 4 } vieter_job_queue_error; /* @@ -72,24 +77,51 @@ typedef enum vieter_job_queue_error { */ vieter_job_queue *vieter_job_queue_init(); +/* + * Free a job queue. + */ void vieter_job_queue_free(vieter_job_queue **ptp); /* * Insert the given job into the system. */ -vieter_job_queue_error vieter_job_queue_insert(vieter_job *job); +vieter_job_queue_error vieter_job_queue_insert(vieter_job_queue *queue, + vieter_job *job); /* - * Dispatch the job with the given id, returning the pointer to the job. - * Dispatching a job removes it from its respective state's queue. + * Pop a job from the given state's queue. The job will then be marked as + * dispatched. */ -vieter_job_queue_error vieter_job_queue_dispatch(vieter_job **out, uint64_t id); +vieter_job_queue_error vieter_job_queue_pop(vieter_job **out, + vieter_job_queue *queue, + vieter_job_state state); /* * Transition the job with the given id to the new state. This sets the * job's dispatch flag to false, and adds it to the new state's queue. + * + * NOTE: this can only be done with dispatched jobs. */ -vieter_job_queue_error vieter_job_queue_transition(uint64_t id, +vieter_job_queue_error vieter_job_queue_transition(vieter_job_queue *queue, + uint64_t id, vieter_job_state new_state); +/* + * Remove the given job from the job queue, returning its pointer to the caller. + * + * NOTE: this can only be done with dispatched jobs. + */ +vieter_job_queue_error +vieter_job_queue_remove(vieter_job **out, vieter_job_queue *queue, uint64_t id); + +/* + * Transition a job into the failure state, and attach a failure report with the + * provided message. The message is copied, so the caller is responsible for + * freeing the provided string. + * + * NOTE: this can only be done with dispatched jobs. + */ +vieter_job_queue_error vieter_job_queue_fail(vieter_job_queue *queue, + uint64_t id, char *report_message); + #endif diff --git a/src/job-queue/vieter_job_queue.c b/src/job-queue/vieter_job_queue.c new file mode 100644 index 0000000..14ae1dc --- /dev/null +++ b/src/job-queue/vieter_job_queue.c @@ -0,0 +1,153 @@ +#include "vieter_job_queue_internal.h" + +vieter_job_queue *vieter_job_queue_init() { + vieter_job_queue *queue = malloc(sizeof(vieter_job_queue)); + + queue->tree = vieter_tree_init(); + + for (int i = 0; i < VIETER_JOB_STATES; i++) { + queue->heaps[i] = vieter_heap_init(); + } + + return queue; +} + +void vieter_job_queue_free(vieter_job_queue **ptp) { + vieter_job_queue *queue = *ptp; + + vieter_tree_free(queue->tree); + + for (int i = 0; i < VIETER_JOB_STATES; i++) { + vieter_heap_free(queue->heaps[i]); + } + + free(queue); + *ptp = NULL; +} + +vieter_job *vieter_job_init() { return calloc(1, sizeof(vieter_job)); } + +void vieter_job_free(vieter_job **ptp) { + vieter_job *job = *ptp; + + if (job->schedule != NULL) { + vieter_cron_expr_free(job->schedule); + } + + if (job->failure_report != NULL) { + vieter_job_failure_report_free(&job->failure_report); + } + + free(job); + + *ptp = NULL; +} + +vieter_job_queue_error vieter_job_queue_insert(vieter_job_queue *queue, + vieter_job *job) { + vieter_tree_error tree_res = vieter_tree_insert(queue->tree, job->id, job); + + if (tree_res != vieter_tree_ok) { + return vieter_job_queue_already_present; + } + + vieter_heap_insert(queue->heaps[VIETER_JOB_INITIAL_STATE], job->id, job); + + job->current_state = VIETER_JOB_INITIAL_STATE; + job->dispatched = false; + job->state_transition_times[VIETER_JOB_INITIAL_STATE] = time(NULL); + + return vieter_job_queue_ok; +} + +vieter_job_queue_error vieter_job_queue_pop(vieter_job **out, + vieter_job_queue *queue, + vieter_job_state state) { + vieter_heap_error res = vieter_heap_pop((void **)out, queue->heaps[state]); + + if (res != vieter_heap_ok) { + return vieter_job_queue_state_empty; + } + + (*out)->dispatched = true; + + return vieter_job_queue_ok; +} + +vieter_job_queue_error vieter_job_queue_transition(vieter_job_queue *queue, + uint64_t id, + vieter_job_state new_state) { + vieter_job *job; + + vieter_tree_error res = vieter_tree_search((void **)&job, queue->tree, id); + + if (res != vieter_tree_ok) { + return vieter_job_queue_not_present; + } + + if (!job->dispatched) { + return vieter_job_queue_not_dispatched; + } + + vieter_heap_insert(queue->heaps[new_state], job->id, job); + + job->current_state = new_state; + job->dispatched = false; + job->state_transition_times[new_state] = time(NULL); + + return vieter_job_queue_ok; +} + +vieter_job_queue_error vieter_job_queue_remove(vieter_job **out, + vieter_job_queue *queue, + uint64_t id) { + vieter_tree_error res = vieter_tree_search((void **)out, queue->tree, id); + + if (res != vieter_tree_ok) { + return vieter_job_queue_not_present; + } + + vieter_job *job = *out; + + if (!job->dispatched) { + return vieter_job_queue_not_dispatched; + } + + // This can't fail if the search succeeded + vieter_tree_remove((void **)out, queue->tree, job->id); + + return vieter_job_queue_ok; +} + +vieter_job_failure_report *vieter_job_failure_report_init() { + return calloc(1, sizeof(vieter_job_failure_report)); +} + +vieter_job_queue_error vieter_job_queue_fail(vieter_job_queue *queue, + uint64_t id, + char *report_message) { + vieter_job *job; + + vieter_tree_error res = vieter_tree_search((void **)&job, queue->tree, id); + + if (res != vieter_tree_ok) { + return vieter_job_queue_not_present; + } + + if (!job->dispatched) { + return vieter_job_queue_not_dispatched; + } + + vieter_heap_insert(queue->heaps[VIETER_JOB_FAILURE_STATE], job->id, job); + + job->dispatched = false; + job->state_transition_times[VIETER_JOB_FAILURE_STATE] = time(NULL); + + job->failure_report = vieter_job_failure_report_init(); + job->failure_report->failed_state = job->current_state; + job->failure_report->msg = strdup(report_message); + + job->current_state = VIETER_JOB_FAILURE_STATE; + + return vieter_job_queue_ok; +} diff --git a/src/job-queue/vieter_job_queue_internal.h b/src/job-queue/vieter_job_queue_internal.h index 8d8ffc4..dac8b97 100644 --- a/src/job-queue/vieter_job_queue_internal.h +++ b/src/job-queue/vieter_job_queue_internal.h @@ -1,10 +1,13 @@ #ifndef VIETER_JOB_QUEUE_INTERNAL #define VIETER_JOB_QUEUE_INTERNAL +#include "vieter_heap.h" +#include "vieter_job_queue.h" #include "vieter_tree.h" struct vieter_job_queue { vieter_tree *tree; + vieter_heap *heaps[VIETER_JOB_STATES]; }; #endif From 53eb5c1784c76efc3b0fde46427afb75ab4fe5d1 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 2 Feb 2023 10:10:03 +0100 Subject: [PATCH 10/14] feat(job-queue): first draft of job struct --- include/vieter_job_queue.h | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 include/vieter_job_queue.h diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h new file mode 100644 index 0000000..6c21da0 --- /dev/null +++ b/include/vieter_job_queue.h @@ -0,0 +1,32 @@ +#ifndef VIETER_JOB_QUEUE +#define VIETER_JOB_QUEUE + +#include +#include +#include "vieter_cron.h" + +/* + * The order of these do not imply that they happen in this order. New states + * will just get added as consecutive numbers. Their values should be + * monotonically increasing values, as these will be used to index arrays, among + * other things. + */ +typedef enum vieter_job_state { + vieter_job_queued = 0, + vieter_job_dispatched = 1, + vieter_job_finished = 2 +} vieter_job_state; + +// This macro should be kept in sync with the above enum +#define VIETER_JOB_STATES 3 + +typedef struct vieter_job { + int id; + uint64_t next_scheduled_time; + vieter_cron_expression *ce; + bool single; + vieter_job_state state; + uint64_t state_transition_times[VIETER_JOB_STATES]; +} vieter_job; + +#endif From 8668148e96188540c0e547c51461756f230a9445 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 2 Feb 2023 16:14:26 +0100 Subject: [PATCH 11/14] chore: format internal header files --- .editorconfig | 5 ++++ Makefile | 7 +++--- include/vieter_job_queue.h | 30 ++++++++++++++--------- src/job-queue/vieter_job_queue_internal.h | 10 ++++++++ 4 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 .editorconfig create mode 100644 src/job-queue/vieter_job_queue_internal.h diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..a2eb55d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[*.{c,h}] +indent_style = space +indent_size = 2 diff --git a/Makefile b/Makefile index fadf89d..9db32d4 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ INC_DIRS ?= include LIB := $(BUILD_DIR)/$(LIB_FILENAME) SRCS != find '$(SRC_DIR)' -iname '*.c' -SRCS_H != find $(INC_DIRS) '$(SRC_DIR)' -iname '*.h' +SRCS_H != find $(INC_DIRS) -iname '*.h' +SRCS_H_INTERNAL != find $(SRC_DIR) -iname '*.h' SRCS_TEST != find '$(TEST_DIR)' -iname '*.c' OBJS := $(SRCS:%=$(BUILD_DIR)/%.o) @@ -86,11 +87,11 @@ $(BUILD_DIR)/$(TEST_DIR)/%.c.o: $(TEST_DIR)/%.c # =====MAINTENANCE===== .PHONY: lint lint: - clang-format -n --Werror $(SRCS) $(SRCS_H) + clang-format -n --Werror $(SRCS) $(SRCS_H) $(SRCS_H_INTERNAL) .PHONY: fmt fmt: - clang-format -i $(SRCS) $(SRCS_H) + clang-format -i $(SRCS) $(SRCS_H) $(SRCS_H_INTERNAL) .PHONY: clean clean: diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h index 6c21da0..fc02d7e 100644 --- a/include/vieter_job_queue.h +++ b/include/vieter_job_queue.h @@ -1,9 +1,9 @@ #ifndef VIETER_JOB_QUEUE #define VIETER_JOB_QUEUE -#include -#include #include "vieter_cron.h" +#include +#include /* * The order of these do not imply that they happen in this order. New states @@ -12,21 +12,29 @@ * other things. */ typedef enum vieter_job_state { - vieter_job_queued = 0, - vieter_job_dispatched = 1, - vieter_job_finished = 2 + vieter_job_state_queued = 0, + vieter_job_state_ready = 1, + vieter_job_state_build_finished = 2 } vieter_job_state; // This macro should be kept in sync with the above enum #define VIETER_JOB_STATES 3 typedef struct vieter_job { - int id; - uint64_t next_scheduled_time; - vieter_cron_expression *ce; - bool single; - vieter_job_state state; - uint64_t state_transition_times[VIETER_JOB_STATES]; + int id; + uint64_t next_scheduled_time; + vieter_cron_expression *ce; + bool single; + vieter_job_state state; + uint64_t state_transition_times[VIETER_JOB_STATES]; + bool dispatched; + void *build_config; } vieter_job; +typedef struct vieter_job_queue vieter_job_queue; + +vieter_job_queue *vieter_job_queue_init(); + +void vieter_job_queue_insert(int id); + #endif diff --git a/src/job-queue/vieter_job_queue_internal.h b/src/job-queue/vieter_job_queue_internal.h new file mode 100644 index 0000000..8d8ffc4 --- /dev/null +++ b/src/job-queue/vieter_job_queue_internal.h @@ -0,0 +1,10 @@ +#ifndef VIETER_JOB_QUEUE_INTERNAL +#define VIETER_JOB_QUEUE_INTERNAL + +#include "vieter_tree.h" + +struct vieter_job_queue { + vieter_tree *tree; +}; + +#endif From 02bd2c24b72b91eb3317d846d28669d8b6f595d9 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 23 Feb 2023 09:47:47 +0100 Subject: [PATCH 12/14] feat(job-queue): deesigning methods --- include/vieter_job_queue.h | 73 +++++++++++++++++++++++++++++++++----- src/job-queue/README.md | 27 ++++++++++++++ 2 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 src/job-queue/README.md diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h index fc02d7e..aeb7e1e 100644 --- a/include/vieter_job_queue.h +++ b/include/vieter_job_queue.h @@ -14,27 +14,82 @@ typedef enum vieter_job_state { vieter_job_state_queued = 0, vieter_job_state_ready = 1, - vieter_job_state_build_finished = 2 + vieter_job_state_build_finished = 2, + vieter_job_state_failed = 3 } vieter_job_state; // This macro should be kept in sync with the above enum -#define VIETER_JOB_STATES 3 +#define VIETER_JOB_STATES 4 +/* + * Struct storing a report for why a certain job failed to be processed in the + * given state. + */ +typedef struct vieter_job_failure_report { + vieter_job_state failed_state; + char *msg; +} vieter_job_failure_report; + +vieter_job_failure_report *vieter_job_failure_report_init(); + +void vieter_job_failure_report_free(vieter_job_failure_report **ptp); + +/* + * Represents a job currently being processed in the system. A job migrates + * between different states before finally being removed from the queue. + */ typedef struct vieter_job { - int id; + uint64_t id; uint64_t next_scheduled_time; - vieter_cron_expression *ce; - bool single; - vieter_job_state state; - uint64_t state_transition_times[VIETER_JOB_STATES]; - bool dispatched; + vieter_cron_expression *schedule; void *build_config; + vieter_job_failure_report *failure_report; + uint64_t state_transition_times[VIETER_JOB_STATES]; + vieter_job_state current_state; + bool single; + bool dispatched; } vieter_job; +/* + * Allocate a new vieter_job object. + */ +vieter_job *vieter_job_init(); + +void vieter_job_free(vieter_job **ptp); + +/* + * Represents the actual queue managing the list of jobs. + */ typedef struct vieter_job_queue vieter_job_queue; +typedef enum vieter_job_queue_error { + vieter_job_queue_ok = 0, + vieter_job_queue_not_found = 1 +} vieter_job_queue_error; + +/* + * Allocate and initialize a new job queue. + */ vieter_job_queue *vieter_job_queue_init(); -void vieter_job_queue_insert(int id); +void vieter_job_queue_free(vieter_job_queue **ptp); + +/* + * Insert the given job into the system. + */ +vieter_job_queue_error vieter_job_queue_insert(vieter_job *job); + +/* + * Dispatch the job with the given id, returning the pointer to the job. + * Dispatching a job removes it from its respective state's queue. + */ +vieter_job_queue_error vieter_job_queue_dispatch(vieter_job **out, uint64_t id); + +/* + * Transition the job with the given id to the new state. This sets the + * job's dispatch flag to false, and adds it to the new state's queue. + */ +vieter_job_queue_error vieter_job_queue_transition(uint64_t id, + vieter_job_state new_state); #endif diff --git a/src/job-queue/README.md b/src/job-queue/README.md new file mode 100644 index 0000000..8fa9c94 --- /dev/null +++ b/src/job-queue/README.md @@ -0,0 +1,27 @@ +The goal of this job queue design is to process jobs in order, with each job +moving through a pipeline of tasks that need to be completed. + +At any given time, a job is in one of a few given states, e.g. "queued". These +states are explained below. Along with this, each job also has a "dispatched" +flag. If this flag is set to true, it means this job is currently being +processed. "Being processed" could mean anything; it depends entirely on the +state a job's in. While a job is dispatched, it is no longer present in the +priority queue of its respective state. + +## Job + +A job describes a scheduled build as it moves through the pipeline of states. +The job queue datastructure keeps track of all jobs in a central red-black +binary tree. For each state, a priority queue tracks in what order jobs should +be processed. + +## States + +* `queued`: a job that's in the job queue but does not yet need to be executed + (as defined by its timestamp) +* `ready`: a job that's scheduled for building, with all preprocessing tasks + fulfilled. +* `build_finished`: a job whose build has finished, and is waiting for any + post-build tasks. +* `failed`: a job whose processing failed at some point. Jobs in this state + include a failure report that describes in what state they failed, and why. From 83b3198a49750de041e401db230ec14cdf90b3fc Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 7 Mar 2023 11:55:12 +0100 Subject: [PATCH 13/14] feat(job-queue): initial implementation of functionality --- include/vieter_job_queue.h | 44 ++++++- src/job-queue/vieter_job_queue.c | 153 ++++++++++++++++++++++ src/job-queue/vieter_job_queue_internal.h | 3 + 3 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 src/job-queue/vieter_job_queue.c diff --git a/include/vieter_job_queue.h b/include/vieter_job_queue.h index aeb7e1e..6cd748f 100644 --- a/include/vieter_job_queue.h +++ b/include/vieter_job_queue.h @@ -20,6 +20,8 @@ typedef enum vieter_job_state { // This macro should be kept in sync with the above enum #define VIETER_JOB_STATES 4 +#define VIETER_JOB_INITIAL_STATE vieter_job_state_queued +#define VIETER_JOB_FAILURE_STATE vieter_job_state_failed /* * Struct storing a report for why a certain job failed to be processed in the @@ -64,7 +66,10 @@ typedef struct vieter_job_queue vieter_job_queue; typedef enum vieter_job_queue_error { vieter_job_queue_ok = 0, - vieter_job_queue_not_found = 1 + vieter_job_queue_not_present = 1, + vieter_job_queue_already_present = 2, + vieter_job_queue_state_empty = 3, + vieter_job_queue_not_dispatched = 4 } vieter_job_queue_error; /* @@ -72,24 +77,51 @@ typedef enum vieter_job_queue_error { */ vieter_job_queue *vieter_job_queue_init(); +/* + * Free a job queue. + */ void vieter_job_queue_free(vieter_job_queue **ptp); /* * Insert the given job into the system. */ -vieter_job_queue_error vieter_job_queue_insert(vieter_job *job); +vieter_job_queue_error vieter_job_queue_insert(vieter_job_queue *queue, + vieter_job *job); /* - * Dispatch the job with the given id, returning the pointer to the job. - * Dispatching a job removes it from its respective state's queue. + * Pop a job from the given state's queue. The job will then be marked as + * dispatched. */ -vieter_job_queue_error vieter_job_queue_dispatch(vieter_job **out, uint64_t id); +vieter_job_queue_error vieter_job_queue_pop(vieter_job **out, + vieter_job_queue *queue, + vieter_job_state state); /* * Transition the job with the given id to the new state. This sets the * job's dispatch flag to false, and adds it to the new state's queue. + * + * NOTE: this can only be done with dispatched jobs. */ -vieter_job_queue_error vieter_job_queue_transition(uint64_t id, +vieter_job_queue_error vieter_job_queue_transition(vieter_job_queue *queue, + uint64_t id, vieter_job_state new_state); +/* + * Remove the given job from the job queue, returning its pointer to the caller. + * + * NOTE: this can only be done with dispatched jobs. + */ +vieter_job_queue_error +vieter_job_queue_remove(vieter_job **out, vieter_job_queue *queue, uint64_t id); + +/* + * Transition a job into the failure state, and attach a failure report with the + * provided message. The message is copied, so the caller is responsible for + * freeing the provided string. + * + * NOTE: this can only be done with dispatched jobs. + */ +vieter_job_queue_error vieter_job_queue_fail(vieter_job_queue *queue, + uint64_t id, char *report_message); + #endif diff --git a/src/job-queue/vieter_job_queue.c b/src/job-queue/vieter_job_queue.c new file mode 100644 index 0000000..14ae1dc --- /dev/null +++ b/src/job-queue/vieter_job_queue.c @@ -0,0 +1,153 @@ +#include "vieter_job_queue_internal.h" + +vieter_job_queue *vieter_job_queue_init() { + vieter_job_queue *queue = malloc(sizeof(vieter_job_queue)); + + queue->tree = vieter_tree_init(); + + for (int i = 0; i < VIETER_JOB_STATES; i++) { + queue->heaps[i] = vieter_heap_init(); + } + + return queue; +} + +void vieter_job_queue_free(vieter_job_queue **ptp) { + vieter_job_queue *queue = *ptp; + + vieter_tree_free(queue->tree); + + for (int i = 0; i < VIETER_JOB_STATES; i++) { + vieter_heap_free(queue->heaps[i]); + } + + free(queue); + *ptp = NULL; +} + +vieter_job *vieter_job_init() { return calloc(1, sizeof(vieter_job)); } + +void vieter_job_free(vieter_job **ptp) { + vieter_job *job = *ptp; + + if (job->schedule != NULL) { + vieter_cron_expr_free(job->schedule); + } + + if (job->failure_report != NULL) { + vieter_job_failure_report_free(&job->failure_report); + } + + free(job); + + *ptp = NULL; +} + +vieter_job_queue_error vieter_job_queue_insert(vieter_job_queue *queue, + vieter_job *job) { + vieter_tree_error tree_res = vieter_tree_insert(queue->tree, job->id, job); + + if (tree_res != vieter_tree_ok) { + return vieter_job_queue_already_present; + } + + vieter_heap_insert(queue->heaps[VIETER_JOB_INITIAL_STATE], job->id, job); + + job->current_state = VIETER_JOB_INITIAL_STATE; + job->dispatched = false; + job->state_transition_times[VIETER_JOB_INITIAL_STATE] = time(NULL); + + return vieter_job_queue_ok; +} + +vieter_job_queue_error vieter_job_queue_pop(vieter_job **out, + vieter_job_queue *queue, + vieter_job_state state) { + vieter_heap_error res = vieter_heap_pop((void **)out, queue->heaps[state]); + + if (res != vieter_heap_ok) { + return vieter_job_queue_state_empty; + } + + (*out)->dispatched = true; + + return vieter_job_queue_ok; +} + +vieter_job_queue_error vieter_job_queue_transition(vieter_job_queue *queue, + uint64_t id, + vieter_job_state new_state) { + vieter_job *job; + + vieter_tree_error res = vieter_tree_search((void **)&job, queue->tree, id); + + if (res != vieter_tree_ok) { + return vieter_job_queue_not_present; + } + + if (!job->dispatched) { + return vieter_job_queue_not_dispatched; + } + + vieter_heap_insert(queue->heaps[new_state], job->id, job); + + job->current_state = new_state; + job->dispatched = false; + job->state_transition_times[new_state] = time(NULL); + + return vieter_job_queue_ok; +} + +vieter_job_queue_error vieter_job_queue_remove(vieter_job **out, + vieter_job_queue *queue, + uint64_t id) { + vieter_tree_error res = vieter_tree_search((void **)out, queue->tree, id); + + if (res != vieter_tree_ok) { + return vieter_job_queue_not_present; + } + + vieter_job *job = *out; + + if (!job->dispatched) { + return vieter_job_queue_not_dispatched; + } + + // This can't fail if the search succeeded + vieter_tree_remove((void **)out, queue->tree, job->id); + + return vieter_job_queue_ok; +} + +vieter_job_failure_report *vieter_job_failure_report_init() { + return calloc(1, sizeof(vieter_job_failure_report)); +} + +vieter_job_queue_error vieter_job_queue_fail(vieter_job_queue *queue, + uint64_t id, + char *report_message) { + vieter_job *job; + + vieter_tree_error res = vieter_tree_search((void **)&job, queue->tree, id); + + if (res != vieter_tree_ok) { + return vieter_job_queue_not_present; + } + + if (!job->dispatched) { + return vieter_job_queue_not_dispatched; + } + + vieter_heap_insert(queue->heaps[VIETER_JOB_FAILURE_STATE], job->id, job); + + job->dispatched = false; + job->state_transition_times[VIETER_JOB_FAILURE_STATE] = time(NULL); + + job->failure_report = vieter_job_failure_report_init(); + job->failure_report->failed_state = job->current_state; + job->failure_report->msg = strdup(report_message); + + job->current_state = VIETER_JOB_FAILURE_STATE; + + return vieter_job_queue_ok; +} diff --git a/src/job-queue/vieter_job_queue_internal.h b/src/job-queue/vieter_job_queue_internal.h index 8d8ffc4..dac8b97 100644 --- a/src/job-queue/vieter_job_queue_internal.h +++ b/src/job-queue/vieter_job_queue_internal.h @@ -1,10 +1,13 @@ #ifndef VIETER_JOB_QUEUE_INTERNAL #define VIETER_JOB_QUEUE_INTERNAL +#include "vieter_heap.h" +#include "vieter_job_queue.h" #include "vieter_tree.h" struct vieter_job_queue { vieter_tree *tree; + vieter_heap *heaps[VIETER_JOB_STATES]; }; #endif From 1c563d84215aa021312463aed8793fb37bb4a4ad Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 7 Mar 2023 12:12:46 +0100 Subject: [PATCH 14/14] feat(tree): add rwlock --- include/vieter_tree.h | 18 ++++++++++++++++++ src/tree/vieter_tree.c | 12 ++++++++++++ src/tree/vieter_tree_internal.h | 2 ++ 3 files changed, 32 insertions(+) diff --git a/include/vieter_tree.h b/include/vieter_tree.h index 2079eb9..4da500f 100644 --- a/include/vieter_tree.h +++ b/include/vieter_tree.h @@ -67,4 +67,22 @@ void vieter_tree_iterator_free(vieter_tree_iterator **ptp); vieter_tree_error vieter_tree_iterator_next(void **out, vieter_tree_iterator *iter); +/* + * Acquire a read lock on the tree. Return value is the result of + * pthread_rwlock_rdlock. + */ +int vieter_tree_rlock(vieter_tree *tree); + +/* + * Acquire a write lock on the tree. Return value is the result of + * pthread_rwlock_wrlock. + */ +int vieter_tree_wlock(vieter_tree *tree); + +/* + * Unlock the lock after having acquired it. Return value is the result of + * pthread_rwlock_unlock. + */ +int vieter_tree_unlock(vieter_tree *tree); + #endif diff --git a/src/tree/vieter_tree.c b/src/tree/vieter_tree.c index 84a1a26..ca6016e 100644 --- a/src/tree/vieter_tree.c +++ b/src/tree/vieter_tree.c @@ -114,3 +114,15 @@ bool vieter_tree_validate(vieter_tree *tree) { return vieter_tree_node_get(tree->root, vieter_tree_node_black) && vieter_tree_node_validate(tree->root, 0, expected_black_nodes); } + +int vieter_tree_rlock(vieter_tree *tree) { + return pthread_rwlock_rdlock(&tree->lock); +} + +int vieter_tree_wlock(vieter_tree *tree) { + return pthread_rwlock_wrlock(&tree->lock); +} + +int vieter_tree_unlock(vieter_tree *tree) { + return pthread_rwlock_unlock(&tree->lock); +} diff --git a/src/tree/vieter_tree_internal.h b/src/tree/vieter_tree_internal.h index 1214572..037345c 100644 --- a/src/tree/vieter_tree_internal.h +++ b/src/tree/vieter_tree_internal.h @@ -1,11 +1,13 @@ #include "vieter_tree.h" #include "vieter_tree_node.h" +#include #include struct vieter_tree { uint64_t size; vieter_tree_node *root; + pthread_rwlock_t lock; }; /*