From b40389bbe2d5eda10e751ea9ab557316ea9738d0 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 12 Nov 2023 12:48:36 +0100 Subject: [PATCH 1/4] feat(lsm): implement basic remove --- lsm/example/test.c | 17 ++++++++++------- lsm/include/lsm/store.h | 7 +++++++ lsm/src/_include/lsm/store_internal.h | 7 +++++++ lsm/src/store/lsm_store.c | 4 ++++ lsm/src/store/lsm_store_disk_write.c | 25 +++++++++++++++++++++++++ lsm/src/store/lsm_store_entry.c | 13 ++++++++++++- 6 files changed, 65 insertions(+), 8 deletions(-) diff --git a/lsm/example/test.c b/lsm/example/test.c index bd78f2e..660cd04 100644 --- a/lsm/example/test.c +++ b/lsm/example/test.c @@ -20,18 +20,13 @@ int main() { lsm_str *attr; lsm_str_init_copy(&attr, "some attribute value"); - lsm_entry_attr_insert(handle, lsm_attr_type_content_type, attr); + lsm_entry_attr_insert(handle, 1, attr); lsm_str *data; lsm_str_init_copy(&data, "hello"); for (int i = 0; i < 50; i++) { - lsm_entry_data_append(store, handle, data); - } - - if (lsm_entry_sync(store, handle) != lsm_error_ok) { - printf("godver"); - return 1; + lsm_entry_data_append(handle, data); } lsm_entry_close(handle); @@ -50,4 +45,12 @@ int main() { total += read; } printf("\n%lu", total); + + lsm_entry_close(handle); + + assert(lsm_store_open_write(&handle, store, key) == lsm_error_ok); + lsm_entry_remove(handle); + lsm_entry_close(handle); + + assert(lsm_store_open_read(&handle, store, key) == lsm_error_not_found); } diff --git a/lsm/include/lsm/store.h b/lsm/include/lsm/store.h index 31eb19b..45eda7e 100644 --- a/lsm/include/lsm/store.h +++ b/lsm/include/lsm/store.h @@ -176,6 +176,13 @@ void lsm_entry_close(lsm_entry_handle *handle); lsm_error lsm_store_insert(lsm_entry_handle **out, lsm_store *store, lsm_str *key); +/** + * Mark the entry as removed. + * + * @param handle handle to entry to remove + */ +void lsm_entry_remove(lsm_entry_handle *handle); + /** * Append new data to the given entry, which is expected to be in the store. * diff --git a/lsm/src/_include/lsm/store_internal.h b/lsm/src/_include/lsm/store_internal.h index fb50838..afbd4f3 100644 --- a/lsm/src/_include/lsm/store_internal.h +++ b/lsm/src/_include/lsm/store_internal.h @@ -40,6 +40,13 @@ typedef struct lsm_entry { */ lsm_error lsm_entry_init(lsm_entry **ptr); +/** + * Deallocate an existing entry + * + * @param entry pointer to entry + */ +void lsm_entry_free(lsm_entry *entry); + /** * Deallocate an existing lsm_entry object. * diff --git a/lsm/src/store/lsm_store.c b/lsm/src/store/lsm_store.c index 2185418..57eacb7 100644 --- a/lsm/src/store/lsm_store.c +++ b/lsm/src/store/lsm_store.c @@ -158,6 +158,10 @@ lsm_error lsm_store_insert(lsm_entry_handle **out, lsm_store *store, return lsm_error_ok; } +void lsm_entry_remove(lsm_entry_handle *handle) { + handle->states |= lsm_entry_handle_state_removed; +} + lsm_error lsm_entry_data_append(lsm_entry_handle *handle, lsm_str *data) { if (lsm_str_len(data) == 0) { return lsm_error_ok; diff --git a/lsm/src/store/lsm_store_disk_write.c b/lsm/src/store/lsm_store_disk_write.c index acb0015..b5081af 100644 --- a/lsm/src/store/lsm_store_disk_write.c +++ b/lsm/src/store/lsm_store_disk_write.c @@ -124,3 +124,28 @@ lsm_error lsm_entry_disk_insert(lsm_entry_handle *handle) { return res; } + +// Marking an entry as removed in the idx file is simply setting the length of +// its entry to zero +lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { + lsm_store *store = handle->store; + lsm_entry *entry = handle->wrapper->entry; + + pthread_mutex_lock(&store->idx.lock); + + lsm_error res = + lsm_fseek(store->idx.f, entry->idx_file_offset + sizeof(uint64_t)); + + if (res != lsm_error_ok) { + pthread_mutex_unlock(&store->idx.lock); + + return res; + } + + uint64_t val = 0; + res = lsm_fwrite(NULL, store->idx.f, sizeof(uint64_t), 1, &val); + + pthread_mutex_unlock(&store->idx.lock); + + return res; +} diff --git a/lsm/src/store/lsm_store_entry.c b/lsm/src/store/lsm_store_entry.c index 8212ba6..fd55624 100644 --- a/lsm/src/store/lsm_store_entry.c +++ b/lsm/src/store/lsm_store_entry.c @@ -19,6 +19,14 @@ lsm_error lsm_entry_init(lsm_entry **ptr) { return lsm_error_ok; } +void lsm_entry_free(lsm_entry *entry) { + if (entry->attrs.count > 0) { + free(entry->attrs.items); + } + + free(entry); +} + lsm_error lsm_entry_wrapper_init(lsm_entry_wrapper **ptr) { lsm_entry_wrapper *wrap = calloc(1, sizeof(lsm_entry_wrapper)); @@ -58,7 +66,10 @@ void lsm_entry_close(lsm_entry_handle *handle) { lsm_entry_disk_insert(handle); } else if ((handle->states & lsm_entry_handle_state_removed) && !(handle->states & lsm_entry_handle_state_new)) { - /* lsm_entry_disk_remove(handle); */ + lsm_entry_disk_remove(handle); + + lsm_entry_free(handle->wrapper->entry); + handle->wrapper->entry = NULL; } else if (handle->states & lsm_entry_handle_state_updated) { /* lsm_entry_disk_update(handle); */ } From c8728f2371980c5a5898c4ad68b484d289de1694 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 12 Nov 2023 13:19:30 +0100 Subject: [PATCH 2/4] refactor(lsm): abstract determining entry data path --- lsm/src/_include/lsm/store_internal.h | 12 ++++++++++++ lsm/src/store/lsm_store.c | 14 ++++++-------- lsm/src/store/lsm_store_entry.c | 22 ++++++++++++++++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/lsm/src/_include/lsm/store_internal.h b/lsm/src/_include/lsm/store_internal.h index afbd4f3..f356bd4 100644 --- a/lsm/src/_include/lsm/store_internal.h +++ b/lsm/src/_include/lsm/store_internal.h @@ -10,6 +10,7 @@ #define LSM_DB_FILE_NAME "lsm.db" #define LSM_IDX_FILE_NAME "lsm.idx" +#define LSM_DATA_FILE_SUFFIX ".data" typedef struct lsm_attr { uint8_t type; @@ -129,4 +130,15 @@ lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle); */ lsm_error lsm_entry_disk_update(lsm_entry_handle *handle); +/** + * Return the length of the path to this entry's data file + */ +uint64_t lsm_entry_data_path_len(lsm_entry_handle *handle); + +/** + * Fill in the entry's data file path in the provided buffer. Use + * `lsm_entry_data_path_len` to allocate an appropriately-sized buffer + */ +void lsm_entry_data_path(char *buf, lsm_entry_handle *handle); + #endif diff --git a/lsm/src/store/lsm_store.c b/lsm/src/store/lsm_store.c index 57eacb7..f93e289 100644 --- a/lsm/src/store/lsm_store.c +++ b/lsm/src/store/lsm_store.c @@ -174,11 +174,10 @@ lsm_error lsm_entry_data_append(lsm_entry_handle *handle, lsm_str *data) { // Entries don't open their file unless needed if (handle->f == NULL) { - char path[handle->store->data_path->len + entry->key->len + 2]; - sprintf(path, "%s/%s", lsm_str_ptr(handle->store->data_path), - lsm_str_ptr(entry->key)); + char data_path[lsm_entry_data_path_len(handle) + 1]; + lsm_entry_data_path(data_path, handle); - FILE *f = fopen(path, "ab"); + FILE *f = fopen(data_path, "ab"); if (f == NULL) { return lsm_error_failed_io; @@ -213,11 +212,10 @@ lsm_error lsm_entry_data_read(uint64_t *out, char *buf, // Entries don't open their file unless needed if (handle->f == NULL) { - char path[handle->store->data_path->len + entry->key->len + 2]; - sprintf(path, "%s/%s", lsm_str_ptr(handle->store->data_path), - lsm_str_ptr(entry->key)); + char data_path[lsm_entry_data_path_len(handle) + 1]; + lsm_entry_data_path(data_path, handle); - FILE *f = fopen(path, "rb"); + FILE *f = fopen(data_path, "rb"); if (f == NULL) { return lsm_error_failed_io; diff --git a/lsm/src/store/lsm_store_entry.c b/lsm/src/store/lsm_store_entry.c index fd55624..a9c0ade 100644 --- a/lsm/src/store/lsm_store_entry.c +++ b/lsm/src/store/lsm_store_entry.c @@ -227,3 +227,25 @@ lsm_error lsm_entry_attr_insert_uint8_t(lsm_entry_handle *handle, uint8_t type, uint64_t lsm_entry_data_len(lsm_entry_handle *handle) { return handle->wrapper->entry->data_len; } + +uint64_t lsm_entry_data_path_len(lsm_entry_handle *handle) { + // [data path]/[entry key][data file suffix] + return lsm_str_len(handle->store->data_path) + + lsm_str_len(handle->wrapper->entry->key) + + strlen(LSM_DATA_FILE_SUFFIX) + 1; +} +void lsm_entry_data_path(char *buf, lsm_entry_handle *handle) { + lsm_str *data_path = handle->store->data_path; + lsm_str *key = handle->wrapper->entry->key; + + memcpy(buf, lsm_str_ptr(data_path), lsm_str_len(data_path)); + + uint64_t index = lsm_str_len(data_path); + buf[index] = '/'; + + index += 1; + memcpy(&buf[index], lsm_str_ptr(key), lsm_str_len(key)); + + index += lsm_str_len(key); + strcpy(&buf[index], LSM_DATA_FILE_SUFFIX); +} From a4ad8c246e2d59940fdbabb9db4a7e79638204dc Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 12 Nov 2023 13:21:04 +0100 Subject: [PATCH 3/4] feat(lsm): remove data file when removing entry --- lsm/src/store/lsm_store_disk_write.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lsm/src/store/lsm_store_disk_write.c b/lsm/src/store/lsm_store_disk_write.c index b5081af..243c8f5 100644 --- a/lsm/src/store/lsm_store_disk_write.c +++ b/lsm/src/store/lsm_store_disk_write.c @@ -147,5 +147,24 @@ lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { pthread_mutex_unlock(&store->idx.lock); - return res; + if (res != lsm_error_ok) { + return res; + } + + // Remove data file if present + if (entry->data_len > 0) { + if (handle->f != NULL) { + fclose(handle->f); + handle->f = NULL; + } + + char data_path[lsm_entry_data_path_len(handle) + 1]; + lsm_entry_data_path(data_path, handle); + + if (remove(data_path) != 0) { + return lsm_error_failed_io; + } + } + + return lsm_error_ok; } From 3d48ee8019e9a84e7dd34527b19c4102d1e469b2 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 12 Nov 2023 13:43:21 +0100 Subject: [PATCH 4/4] feat(lander): support DELETE requests --- include/lander.h | 4 +++- lsm/src/store/lsm_store_disk_write.c | 2 ++ src/lander/lander.c | 8 ++++++++ src/lander/lander_delete.c | 29 ++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 src/lander/lander_delete.c diff --git a/include/lander.h b/include/lander.h index 88bfab9..61c4297 100644 --- a/include/lander.h +++ b/include/lander.h @@ -4,7 +4,7 @@ #include "http_loop.h" #include "lsm/store.h" -extern http_route lander_routes[4]; +extern http_route lander_routes[5]; typedef struct lander_gctx { const char *data_dir; @@ -57,4 +57,6 @@ bool lander_get_entry_lsm(event_loop_conn *conn); bool lander_post_redirect_body_to_attr(event_loop_conn *conn); +bool lander_remove_entry(event_loop_conn *conn); + #endif diff --git a/lsm/src/store/lsm_store_disk_write.c b/lsm/src/store/lsm_store_disk_write.c index 243c8f5..b6906e6 100644 --- a/lsm/src/store/lsm_store_disk_write.c +++ b/lsm/src/store/lsm_store_disk_write.c @@ -151,6 +151,8 @@ lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { return res; } + fflush(store->idx.f); + // Remove data file if present if (entry->data_len > 0) { if (handle->f != NULL) { diff --git a/src/lander/lander.c b/src/lander/lander.c index 57f5c5e..a045428 100644 --- a/src/lander/lander.c +++ b/src/lander/lander.c @@ -19,6 +19,14 @@ http_route lander_routes[] = { .steps_res = {http_loop_step_write_header, lander_stream_body_to_client, NULL}, }, + { + .type = http_route_regex, + .method = http_delete, + .path = "^/([^/]+)$", + .steps = {http_loop_step_auth, lander_remove_entry, NULL}, + .steps_res = {http_loop_step_write_header, http_loop_step_write_body, + NULL}, + }, { .type = http_route_regex, .method = http_post, diff --git a/src/lander/lander_delete.c b/src/lander/lander_delete.c new file mode 100644 index 0000000..e91b6c9 --- /dev/null +++ b/src/lander/lander_delete.c @@ -0,0 +1,29 @@ +#include "lander.h" + +bool lander_remove_entry(event_loop_conn *conn) { + http_loop_ctx *ctx = conn->ctx; + lander_ctx *c_ctx = ctx->c; + http_loop_gctx *gctx = ctx->g; + lander_gctx *c_gctx = gctx->c; + + const char *key_s = &ctx->req.path[ctx->req.regex_groups[1].rm_so]; + int key_len = ctx->req.regex_groups[1].rm_eo - ctx->req.regex_groups[1].rm_so; + + lsm_str *key; + lsm_str_init_copy_n(&key, (char *)key_s, key_len); + + switch (lsm_store_open_write(&c_ctx->entry, c_gctx->store, key)) { + case lsm_error_ok: + break; + case lsm_error_not_found: + ctx->res.status = http_not_found; + return true; + default: + ctx->res.status = http_internal_server_error; + return true; + } + + lsm_entry_remove(c_ctx->entry); + + return true; +}