Compare commits

...

9 Commits

15 changed files with 575 additions and 42 deletions

5
.editorconfig 100644
View File

@ -0,0 +1,5 @@
root = true
[*.{c,h}]
indent_style = space
indent_size = 2

View File

@ -12,6 +12,7 @@ LIB := $(BUILD_DIR)/$(LIB_FILENAME)
SRCS != find '$(SRC_DIR)' -iname '*.c'
SRCS_H != find $(INC_DIRS) -iname '*.h'
SRCS_H_INTERNAL != find $(SRC_DIR) -iname '*.h'
SRCS_TEST != find '$(TEST_DIR)' -iname '*.c'
OBJS := $(SRCS:%=$(BUILD_DIR)/%.o)
@ -66,15 +67,9 @@ $(TARGETS_TEST): test-%: %
$(TARGETS_MEM_TEST): test-mem-%: %
valgrind --tool=memcheck --error-exitcode=1 --track-origins=yes --leak-check=full ./$^
test-mem: build-test
@ $(foreach bin,$(BINS_TEST),valgrind --tool=memcheck --error-exitcode=1 \
--track-origins=yes --leak-check=full ./$(bin);)
.PHONY: build-test
build-test: $(BINS_TEST)
# For simplicity, we link every object file to each of the test files. This
# might be changed later if this starts to become too slow.
$(BINS_TEST): %: %.c.o $(LIB)
$(CC) \
$^ -o $@
@ -92,11 +87,11 @@ $(BUILD_DIR)/$(TEST_DIR)/%.c.o: $(TEST_DIR)/%.c
# =====MAINTENANCE=====
.PHONY: lint
lint:
clang-format -n --Werror $(SRCS) $(SRCS_H)
clang-format -n --Werror $(SRCS) $(SRCS_H) $(SRCS_H_INTERNAL)
.PHONY: fmt
fmt:
clang-format -i $(SRCS) $(SRCS_H)
clang-format -i $(SRCS) $(SRCS_H) $(SRCS_H_INTERNAL)
.PHONY: clean
clean:

View File

@ -0,0 +1,127 @@
#ifndef VIETER_JOB_QUEUE
#define VIETER_JOB_QUEUE
#include "vieter_cron.h"
#include <stdbool.h>
#include <stdint.h>
/*
* The order of these do not imply that they happen in this order. New states
* will just get added as consecutive numbers. Their values should be
* monotonically increasing values, as these will be used to index arrays, among
* other things.
*/
typedef enum vieter_job_state {
vieter_job_state_queued = 0,
vieter_job_state_ready = 1,
vieter_job_state_build_finished = 2,
vieter_job_state_failed = 3
} vieter_job_state;
// This macro should be kept in sync with the above enum
#define VIETER_JOB_STATES 4
#define VIETER_JOB_INITIAL_STATE vieter_job_state_queued
#define VIETER_JOB_FAILURE_STATE vieter_job_state_failed
/*
* Struct storing a report for why a certain job failed to be processed in the
* given state.
*/
typedef struct vieter_job_failure_report {
vieter_job_state failed_state;
char *msg;
} vieter_job_failure_report;
vieter_job_failure_report *vieter_job_failure_report_init();
void vieter_job_failure_report_free(vieter_job_failure_report **ptp);
/*
* Represents a job currently being processed in the system. A job migrates
* between different states before finally being removed from the queue.
*/
typedef struct vieter_job {
uint64_t id;
uint64_t next_scheduled_time;
vieter_cron_expression *schedule;
void *build_config;
vieter_job_failure_report *failure_report;
uint64_t state_transition_times[VIETER_JOB_STATES];
vieter_job_state current_state;
bool single;
bool dispatched;
} vieter_job;
/*
* Allocate a new vieter_job object.
*/
vieter_job *vieter_job_init();
void vieter_job_free(vieter_job **ptp);
/*
* Represents the actual queue managing the list of jobs.
*/
typedef struct vieter_job_queue vieter_job_queue;
typedef enum vieter_job_queue_error {
vieter_job_queue_ok = 0,
vieter_job_queue_not_present = 1,
vieter_job_queue_already_present = 2,
vieter_job_queue_state_empty = 3,
vieter_job_queue_not_dispatched = 4
} vieter_job_queue_error;
/*
* Allocate and initialize a new job queue.
*/
vieter_job_queue *vieter_job_queue_init();
/*
* Free a job queue.
*/
void vieter_job_queue_free(vieter_job_queue **ptp);
/*
* Insert the given job into the system.
*/
vieter_job_queue_error vieter_job_queue_insert(vieter_job_queue *queue,
vieter_job *job);
/*
* Pop a job from the given state's queue. The job will then be marked as
* dispatched.
*/
vieter_job_queue_error vieter_job_queue_pop(vieter_job **out,
vieter_job_queue *queue,
vieter_job_state state);
/*
* Transition the job with the given id to the new state. This sets the
* job's dispatch flag to false, and adds it to the new state's queue.
*
* NOTE: this can only be done with dispatched jobs.
*/
vieter_job_queue_error vieter_job_queue_transition(vieter_job_queue *queue,
uint64_t id,
vieter_job_state new_state);
/*
* Remove the given job from the job queue, returning its pointer to the caller.
*
* NOTE: this can only be done with dispatched jobs.
*/
vieter_job_queue_error
vieter_job_queue_remove(vieter_job **out, vieter_job_queue *queue, uint64_t id);
/*
* Transition a job into the failure state, and attach a failure report with the
* provided message. The message is copied, so the caller is responsible for
* freeing the provided string.
*
* NOTE: this can only be done with dispatched jobs.
*/
vieter_job_queue_error vieter_job_queue_fail(vieter_job_queue *queue,
uint64_t id, char *report_message);
#endif

View File

@ -9,7 +9,8 @@ typedef struct vieter_tree vieter_tree;
typedef enum vieter_tree_error {
vieter_tree_ok = 0,
vieter_tree_already_present = 1,
vieter_tree_not_present = 2
vieter_tree_not_present = 2,
vieter_tree_iterator_done = 3
} vieter_tree_error;
/*
@ -45,4 +46,43 @@ vieter_tree_error vieter_tree_remove(void **out, vieter_tree *tree,
*/
uint64_t vieter_tree_size(vieter_tree *tree);
/*
* An iterator that can be used to traverse a tree in-order.
*/
typedef struct vieter_tree_iterator vieter_tree_iterator;
/*
* Allocate and initialize a new iterator from the given tree.
*/
vieter_tree_iterator *vieter_tree_iterator_from(vieter_tree *tree);
/*
* Free an iterator.
*/
void vieter_tree_iterator_free(vieter_tree_iterator **ptp);
/*
* Return the iterator's next element.
*/
vieter_tree_error vieter_tree_iterator_next(void **out,
vieter_tree_iterator *iter);
/*
* Acquire a read lock on the tree. Return value is the result of
* pthread_rwlock_rdlock.
*/
int vieter_tree_rlock(vieter_tree *tree);
/*
* Acquire a write lock on the tree. Return value is the result of
* pthread_rwlock_wrlock.
*/
int vieter_tree_wlock(vieter_tree *tree);
/*
* Unlock the lock after having acquired it. Return value is the result of
* pthread_rwlock_unlock.
*/
int vieter_tree_unlock(vieter_tree *tree);
#endif

View File

@ -5,16 +5,16 @@
#include <stdlib.h>
typedef struct vieter_heap_node {
uint64_t key;
void *data;
struct vieter_heap_node *largest_order;
union {
// Roots point to next tree in the heap, other nodes point to their first
// neighbour.
struct vieter_heap_node *next_tree;
struct vieter_heap_node *next_largest_order;
} ptr;
uint8_t order;
uint64_t key;
void *data;
struct vieter_heap_node *largest_order;
union {
// Roots point to next tree in the heap, other nodes point to their first
// neighbour.
struct vieter_heap_node *next_tree;
struct vieter_heap_node *next_largest_order;
} ptr;
uint8_t order;
} vieter_heap_node;
/*
@ -36,7 +36,8 @@ void vieter_heap_tree_free(vieter_heap_node *root);
* Given the roots of the smallest trees in two heaps, merge them into a single
* large heap.
*/
vieter_heap_node *vieter_heap_tree_merge(vieter_heap_node *root_a, vieter_heap_node *root_b);
vieter_heap_node *vieter_heap_tree_merge(vieter_heap_node *root_a,
vieter_heap_node *root_b);
/*
* Given the roots of two trees of the same order, merge them into a heap of one

View File

@ -0,0 +1,27 @@
The goal of this job queue design is to process jobs in order, with each job
moving through a pipeline of tasks that need to be completed.
At any given time, a job is in one of a few given states, e.g. "queued". These
states are explained below. Along with this, each job also has a "dispatched"
flag. If this flag is set to true, it means this job is currently being
processed. "Being processed" could mean anything; it depends entirely on the
state a job's in. While a job is dispatched, it is no longer present in the
priority queue of its respective state.
## Job
A job describes a scheduled build as it moves through the pipeline of states.
The job queue datastructure keeps track of all jobs in a central red-black
binary tree. For each state, a priority queue tracks in what order jobs should
be processed.
## States
* `queued`: a job that's in the job queue but does not yet need to be executed
(as defined by its timestamp)
* `ready`: a job that's scheduled for building, with all preprocessing tasks
fulfilled.
* `build_finished`: a job whose build has finished, and is waiting for any
post-build tasks.
* `failed`: a job whose processing failed at some point. Jobs in this state
include a failure report that describes in what state they failed, and why.

View File

@ -0,0 +1,153 @@
#include "vieter_job_queue_internal.h"
vieter_job_queue *vieter_job_queue_init() {
vieter_job_queue *queue = malloc(sizeof(vieter_job_queue));
queue->tree = vieter_tree_init();
for (int i = 0; i < VIETER_JOB_STATES; i++) {
queue->heaps[i] = vieter_heap_init();
}
return queue;
}
void vieter_job_queue_free(vieter_job_queue **ptp) {
vieter_job_queue *queue = *ptp;
vieter_tree_free(queue->tree);
for (int i = 0; i < VIETER_JOB_STATES; i++) {
vieter_heap_free(queue->heaps[i]);
}
free(queue);
*ptp = NULL;
}
vieter_job *vieter_job_init() { return calloc(1, sizeof(vieter_job)); }
void vieter_job_free(vieter_job **ptp) {
vieter_job *job = *ptp;
if (job->schedule != NULL) {
vieter_cron_expr_free(job->schedule);
}
if (job->failure_report != NULL) {
vieter_job_failure_report_free(&job->failure_report);
}
free(job);
*ptp = NULL;
}
vieter_job_queue_error vieter_job_queue_insert(vieter_job_queue *queue,
vieter_job *job) {
vieter_tree_error tree_res = vieter_tree_insert(queue->tree, job->id, job);
if (tree_res != vieter_tree_ok) {
return vieter_job_queue_already_present;
}
vieter_heap_insert(queue->heaps[VIETER_JOB_INITIAL_STATE], job->id, job);
job->current_state = VIETER_JOB_INITIAL_STATE;
job->dispatched = false;
job->state_transition_times[VIETER_JOB_INITIAL_STATE] = time(NULL);
return vieter_job_queue_ok;
}
vieter_job_queue_error vieter_job_queue_pop(vieter_job **out,
vieter_job_queue *queue,
vieter_job_state state) {
vieter_heap_error res = vieter_heap_pop((void **)out, queue->heaps[state]);
if (res != vieter_heap_ok) {
return vieter_job_queue_state_empty;
}
(*out)->dispatched = true;
return vieter_job_queue_ok;
}
vieter_job_queue_error vieter_job_queue_transition(vieter_job_queue *queue,
uint64_t id,
vieter_job_state new_state) {
vieter_job *job;
vieter_tree_error res = vieter_tree_search((void **)&job, queue->tree, id);
if (res != vieter_tree_ok) {
return vieter_job_queue_not_present;
}
if (!job->dispatched) {
return vieter_job_queue_not_dispatched;
}
vieter_heap_insert(queue->heaps[new_state], job->id, job);
job->current_state = new_state;
job->dispatched = false;
job->state_transition_times[new_state] = time(NULL);
return vieter_job_queue_ok;
}
vieter_job_queue_error vieter_job_queue_remove(vieter_job **out,
vieter_job_queue *queue,
uint64_t id) {
vieter_tree_error res = vieter_tree_search((void **)out, queue->tree, id);
if (res != vieter_tree_ok) {
return vieter_job_queue_not_present;
}
vieter_job *job = *out;
if (!job->dispatched) {
return vieter_job_queue_not_dispatched;
}
// This can't fail if the search succeeded
vieter_tree_remove((void **)out, queue->tree, job->id);
return vieter_job_queue_ok;
}
vieter_job_failure_report *vieter_job_failure_report_init() {
return calloc(1, sizeof(vieter_job_failure_report));
}
vieter_job_queue_error vieter_job_queue_fail(vieter_job_queue *queue,
uint64_t id,
char *report_message) {
vieter_job *job;
vieter_tree_error res = vieter_tree_search((void **)&job, queue->tree, id);
if (res != vieter_tree_ok) {
return vieter_job_queue_not_present;
}
if (!job->dispatched) {
return vieter_job_queue_not_dispatched;
}
vieter_heap_insert(queue->heaps[VIETER_JOB_FAILURE_STATE], job->id, job);
job->dispatched = false;
job->state_transition_times[VIETER_JOB_FAILURE_STATE] = time(NULL);
job->failure_report = vieter_job_failure_report_init();
job->failure_report->failed_state = job->current_state;
job->failure_report->msg = strdup(report_message);
job->current_state = VIETER_JOB_FAILURE_STATE;
return vieter_job_queue_ok;
}

View File

@ -0,0 +1,13 @@
#ifndef VIETER_JOB_QUEUE_INTERNAL
#define VIETER_JOB_QUEUE_INTERNAL
#include "vieter_heap.h"
#include "vieter_job_queue.h"
#include "vieter_tree.h"
struct vieter_job_queue {
vieter_tree *tree;
vieter_heap *heaps[VIETER_JOB_STATES];
};
#endif

View File

@ -1,11 +1,5 @@
#include "vieter_tree_internal.h"
#include "vieter_tree_node.h"
#include "vieter_tree_balancing.h"
struct vieter_tree {
uint64_t size;
vieter_tree_node *root;
};
#include "vieter_tree_internal.h"
vieter_tree *vieter_tree_init() { return calloc(1, sizeof(vieter_tree)); }
@ -120,3 +114,15 @@ bool vieter_tree_validate(vieter_tree *tree) {
return vieter_tree_node_get(tree->root, vieter_tree_node_black) &&
vieter_tree_node_validate(tree->root, 0, expected_black_nodes);
}
int vieter_tree_rlock(vieter_tree *tree) {
return pthread_rwlock_rdlock(&tree->lock);
}
int vieter_tree_wlock(vieter_tree *tree) {
return pthread_rwlock_wrlock(&tree->lock);
}
int vieter_tree_unlock(vieter_tree *tree) {
return pthread_rwlock_unlock(&tree->lock);
}

View File

@ -1,10 +1,24 @@
#include "vieter_tree.h"
#include "vieter_tree_node.h"
#include <pthread.h>
#include <stdbool.h>
struct vieter_tree {
uint64_t size;
vieter_tree_node *root;
pthread_rwlock_t lock;
};
/*
* Inspect whether the tree is still a valid red-black-tree.
*
* @return true if valid, false otherwise.
*/
bool vieter_tree_validate(vieter_tree *tree);
struct vieter_tree_iterator {
vieter_tree_node *current_node;
bool started;
bool done;
};

View File

@ -0,0 +1,49 @@
#include "vieter_tree.h"
#include "vieter_tree_internal.h"
vieter_tree_iterator *vieter_tree_iterator_from(vieter_tree *tree) {
vieter_tree_iterator *iter = calloc(1, sizeof(vieter_tree_iterator));
// An empty tree's iterator will be done immediately
if (tree->size == 0) {
iter->started = true;
iter->done = true;
return iter;
}
iter->current_node = tree->root;
return iter;
}
vieter_tree_error vieter_tree_iterator_next(void **out,
vieter_tree_iterator *iter) {
if (iter->done) {
return vieter_tree_iterator_done;
}
if (!iter->started) {
while (iter->current_node->children[0] != NULL) {
iter->current_node = iter->current_node->children[0];
}
iter->started = true;
}
*out = iter->current_node->data;
iter->current_node = vieter_tree_node_next(iter->current_node);
if (iter->current_node == NULL) {
iter->done = true;
}
return vieter_tree_ok;
}
void vieter_tree_iterator_free(vieter_tree_iterator **ptp) {
free(*ptp);
*ptp = NULL;
}

View File

@ -150,3 +150,22 @@ void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag,
bool vieter_tree_node_get(vieter_tree_node *node, vieter_tree_node_flag flag) {
return (node->flags & flag) != 0;
}
vieter_tree_node *vieter_tree_node_next(vieter_tree_node *node) {
if (node->children[1] != NULL) {
node = node->children[1];
while (node->children[0] != NULL) {
node = node->children[0];
}
return node;
}
while (node->parent != NULL &&
vieter_tree_node_get(node, vieter_tree_node_right)) {
node = node->parent;
}
return node->parent;
}

View File

@ -2,8 +2,8 @@
#define VIETER_TREE_NODE
#include "vieter_tree.h"
#include <stdint.h>
#include <stdbool.h>
#include <stdint.h>
typedef enum vieter_tree_node_flag {
vieter_tree_node_black = ((uint8_t)1) << 0,
@ -13,11 +13,11 @@ typedef enum vieter_tree_node_flag {
} vieter_tree_node_flag;
typedef struct vieter_tree_node {
uint64_t key;
void *data;
struct vieter_tree_node *parent;
struct vieter_tree_node *children[2];
uint8_t flags;
uint64_t key;
void *data;
struct vieter_tree_node *parent;
struct vieter_tree_node *children[2];
uint8_t flags;
} vieter_tree_node;
/*
@ -33,27 +33,33 @@ void vieter_tree_node_free(vieter_tree_node *node);
/*
* Insert a new key into the given tree.
*/
vieter_tree_error vieter_tree_node_insert(vieter_tree_node *root, uint64_t key, void *data);
vieter_tree_error vieter_tree_node_insert(vieter_tree_node *root, uint64_t key,
void *data);
/*
* Return the node representing the requested value.
*/
vieter_tree_error vieter_tree_node_search_node(vieter_tree_node **out, vieter_tree_node *root, uint64_t key);
vieter_tree_error vieter_tree_node_search_node(vieter_tree_node **out,
vieter_tree_node *root,
uint64_t key);
/*
* Search for the data represented by the given key.
*/
vieter_tree_error vieter_tree_node_search(void **out, vieter_tree_node *root, uint64_t key);
vieter_tree_error vieter_tree_node_search(void **out, vieter_tree_node *root,
uint64_t key);
/*
* Remove the data associated with the given key.
*/
vieter_tree_error vieter_tree_node_remove(void **out, vieter_tree_node **root_ptr, uint64_t key);
vieter_tree_error
vieter_tree_node_remove(void **out, vieter_tree_node **root_ptr, uint64_t key);
/*
* Set a node's bit flag to the given value.
*/
void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag, bool value);
void vieter_tree_node_set(vieter_tree_node *node, vieter_tree_node_flag flag,
bool value);
/*
* Get whether a node's bit flag is set.
@ -70,11 +76,19 @@ void vieter_tree_node_add_child(vieter_tree_node *parent, uint64_t key,
/*
* Replace a node's children array.
*/
void vieter_tree_node_set_children(vieter_tree_node *parent, vieter_tree_node **children);
void vieter_tree_node_set_children(vieter_tree_node *parent,
vieter_tree_node **children);
/*
* Set a node's left or right child to the given node.
*/
void vieter_tree_node_set_child(vieter_tree_node *parent, vieter_tree_node *child, bool right);
void vieter_tree_node_set_child(vieter_tree_node *parent,
vieter_tree_node *child, bool right);
/*
* Return the in-order successor of the given node, or NULL if it's the last
* node in the tree.
*/
vieter_tree_node *vieter_tree_node_next(vieter_tree_node *node);
#endif

View File

@ -16,19 +16,30 @@ void test_insert() {
vieter_tree *tree = vieter_tree_init();
for (uint64_t i = 0; i < 250; i++) {
TEST_CHECK(vieter_tree_insert(tree, i, NULL) == vieter_tree_ok);
TEST_CHECK(vieter_tree_insert(tree, i, (void *)i) == vieter_tree_ok);
TEST_SIZE(tree, i + 1);
TEST_CHECK(vieter_tree_validate(tree));
}
void *out;
vieter_tree_iterator *iter = vieter_tree_iterator_from(tree);
void *out = NULL;
for (uint64_t i = 0; i < 250; i++) {
TEST_CHECK(vieter_tree_search(&out, tree, i) == vieter_tree_ok);
TEST_CHECK(out == (void *)i);
TEST_CHECK(vieter_tree_insert(tree, i, NULL) == vieter_tree_already_present);
TEST_CHECK(vieter_tree_search(&out, tree, i) == vieter_tree_ok);
out = NULL;
TEST_CHECK(vieter_tree_iterator_next(&out, iter) == vieter_tree_ok);
TEST_CHECK(out == (void *)i);
}
TEST_CHECK(vieter_tree_iterator_next(&out, iter) == vieter_tree_iterator_done);
vieter_tree_iterator_free(&iter);
vieter_tree_free(tree);
}

View File

@ -0,0 +1,59 @@
#include "acutest.h"
#include "vieter_tree_internal.h"
#define TEST_SIZE(tree, size) \
TEST_CHECK(vieter_tree_size(tree) == size); \
TEST_MSG("Size: %zu", vieter_tree_size(tree))
int uint64_t_compare(const void *a, const void *b) {
if ((*(uint64_t *)a) < (*(uint64_t *)b)) {
return -1;
} else if ((*(uint64_t *)a) > (*(uint64_t *)b)) {
return 1;
} else {
return 0;
}
}
void test_insert_random() {
const uint64_t n = 1000;
srand(0);
vieter_tree *tree = vieter_tree_init();
uint64_t *numbers = malloc(n * sizeof(uint64_t));
uint64_t num;
void *out = NULL;
for (uint64_t i = 0; i < n; i++) {
num = rand();
vieter_tree_insert(tree, num, (void *)num);
TEST_SIZE(tree, i + 1);
TEST_CHECK(vieter_tree_validate(tree));
TEST_CHECK(vieter_tree_search(&out, tree, num) == vieter_tree_ok);
TEST_CHECK(out == (void *)num);
out = NULL;
numbers[i] = num;
}
qsort(numbers, n, sizeof(uint64_t), uint64_t_compare);
vieter_tree_iterator *iter = vieter_tree_iterator_from(tree);
out = NULL;
for (uint64_t i = 0; i < n; i++) {
TEST_CHECK(vieter_tree_iterator_next(&out, iter) == vieter_tree_ok);
TEST_CHECK(out == (void *)numbers[i]);
}
free(numbers);
vieter_tree_iterator_free(&iter);
vieter_tree_free(tree);
}
TEST_LIST = {
{"tree insert random", test_insert_random},
{NULL, NULL}
};