From 719a65beff9ef6a6e23d30180b9c9420cd18231d Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 8 Nov 2023 08:47:24 +0100 Subject: [PATCH 1/4] chore(lsm): format code --- lsm/example/test.c | 2 +- lsm/include/lsm/store.h | 9 ++++++--- lsm/src/store/lsm_store.c | 10 ++++++---- lsm/src/store/lsm_store_entry.c | 9 ++++++--- lsm/src/store/lsm_store_sync.c | 25 ++++++++++++++++--------- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/lsm/example/test.c b/lsm/example/test.c index e1e3b69..bd78f2e 100644 --- a/lsm/example/test.c +++ b/lsm/example/test.c @@ -17,7 +17,7 @@ int main() { lsm_entry_handle *handle; assert(lsm_store_insert(&handle, store, key) == lsm_error_ok); - + lsm_str *attr; lsm_str_init_copy(&attr, "some attribute value"); lsm_entry_attr_insert(handle, lsm_attr_type_content_type, attr); diff --git a/lsm/include/lsm/store.h b/lsm/include/lsm/store.h index 7518059..72334c6 100644 --- a/lsm/include/lsm/store.h +++ b/lsm/include/lsm/store.h @@ -52,7 +52,8 @@ lsm_error lsm_entry_attr_get(lsm_str **out, lsm_entry_handle *handle, * @param entry entry to search for * @param type type of attribute to return */ -lsm_error lsm_entry_attr_get_num(uint64_t *out, lsm_entry_handle *handle, lsm_attr_type type); +lsm_error lsm_entry_attr_get_num(uint64_t *out, lsm_entry_handle *handle, + lsm_attr_type type); /** * Add a new attribute to the entry. @@ -72,7 +73,8 @@ lsm_error lsm_entry_attr_insert(lsm_entry_handle *handle, lsm_attr_type type, * @param type type of attribute to add * @param data data of attribute */ -lsm_error lsm_entry_attr_insert_num(lsm_entry_handle *handle, lsm_attr_type type, uint64_t data); +lsm_error lsm_entry_attr_insert_num(lsm_entry_handle *handle, + lsm_attr_type type, uint64_t data); /** * Remove an atribute from the given entry, if present. @@ -178,7 +180,8 @@ lsm_error lsm_entry_data_append(lsm_store *store, lsm_entry_handle *handle, * @param data data to append * @param len length of data array */ -lsm_error lsm_entry_data_append_raw(lsm_store *store, lsm_entry_handle *handle, char *data, uint64_t len); +lsm_error lsm_entry_data_append_raw(lsm_store *store, lsm_entry_handle *handle, + char *data, uint64_t len); /** * Read a number of bytes from the entry's data field. The position from which diff --git a/lsm/src/store/lsm_store.c b/lsm/src/store/lsm_store.c index 172c6d2..9c0ab83 100644 --- a/lsm/src/store/lsm_store.c +++ b/lsm/src/store/lsm_store.c @@ -37,7 +37,8 @@ lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) { // Try to open an existing db file or create a new one otherwise // This shit is why I need to improve the str library char db_file_path[lsm_str_len(data_path) + strlen(LSM_DB_FILE_NAME) + 2]; - memcpy(db_file_path, lsm_str_ptr(data_path), lsm_str_len(data_path) * sizeof(char)); + memcpy(db_file_path, lsm_str_ptr(data_path), + lsm_str_len(data_path) * sizeof(char)); sprintf(&db_file_path[lsm_str_len(data_path)], "/%s", LSM_DB_FILE_NAME); FILE *db_file = fopen(db_file_path, "r+b"); @@ -52,7 +53,8 @@ lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) { // Same for idx file char idx_file_path[lsm_str_len(data_path) + strlen(LSM_IDX_FILE_NAME) + 2]; - memcpy(idx_file_path, lsm_str_ptr(data_path), lsm_str_len(data_path) * sizeof(char)); + memcpy(idx_file_path, lsm_str_ptr(data_path), + lsm_str_len(data_path) * sizeof(char)); sprintf(&idx_file_path[lsm_str_len(data_path)], "/%s", LSM_IDX_FILE_NAME); FILE *idx_file = fopen(idx_file_path, "r+b"); @@ -200,7 +202,8 @@ lsm_error lsm_store_insert(lsm_entry_handle **out, lsm_store *store, // If a key was previously removed from the trie, the wrapper will already be // present in the trie - if (lsm_trie_search((void **)&wrapper, store->trie, key) == lsm_error_not_found) { + if (lsm_trie_search((void **)&wrapper, store->trie, key) == + lsm_error_not_found) { LSM_RES(lsm_entry_wrapper_init(&wrapper)); pthread_rwlock_wrlock(&wrapper->lock); @@ -222,7 +225,6 @@ lsm_error lsm_store_insert(lsm_entry_handle **out, lsm_store *store, } } - lsm_entry *entry; LSM_RES(lsm_entry_init(&entry)); diff --git a/lsm/src/store/lsm_store_entry.c b/lsm/src/store/lsm_store_entry.c index d7bbc40..2878996 100644 --- a/lsm/src/store/lsm_store_entry.c +++ b/lsm/src/store/lsm_store_entry.c @@ -78,7 +78,8 @@ lsm_error lsm_entry_attr_get(lsm_str **out, lsm_entry_handle *handle, return lsm_error_ok; } -lsm_error lsm_entry_attr_get_num(uint64_t *out, lsm_entry_handle *handle, lsm_attr_type type) { +lsm_error lsm_entry_attr_get_num(uint64_t *out, lsm_entry_handle *handle, + lsm_attr_type type) { lsm_str *s; LSM_RES(lsm_entry_attr_get(&s, handle, type)); @@ -167,9 +168,11 @@ lsm_error lsm_entry_attr_insert(lsm_entry_handle *handle, lsm_attr_type type, return lsm_error_ok; } -lsm_error lsm_entry_attr_insert_num(lsm_entry_handle *handle, lsm_attr_type type, uint64_t data) { +lsm_error lsm_entry_attr_insert_num(lsm_entry_handle *handle, + lsm_attr_type type, uint64_t data) { lsm_str *s; - LSM_RES(lsm_str_init_copy_n(&s, (char *)&data, sizeof(uint64_t) / sizeof(char))); + LSM_RES( + lsm_str_init_copy_n(&s, (char *)&data, sizeof(uint64_t) / sizeof(char))); return lsm_entry_attr_insert(handle, type, s); } diff --git a/lsm/src/store/lsm_store_sync.c b/lsm/src/store/lsm_store_sync.c index 39e7658..caaa42b 100644 --- a/lsm/src/store/lsm_store_sync.c +++ b/lsm/src/store/lsm_store_sync.c @@ -35,16 +35,19 @@ lsm_error lsm_entry_write_db(uint64_t *size, FILE *db_file, lsm_entry *entry) { for (uint64_t i = 0; i < entry->attrs.count; i++) { // Write attribute type, length & value LSM_RES(lsm_entry_write_uint64_t(db_file, entry->attrs.items[i].type)); - LSM_RES(lsm_entry_write_uint64_t(db_file, lsm_str_len(entry->attrs.items[i].str))); + LSM_RES(lsm_entry_write_uint64_t(db_file, + lsm_str_len(entry->attrs.items[i].str))); LSM_RES(lsm_entry_write_str(db_file, entry->attrs.items[i].str)); - *size += 2 * sizeof(uint64_t) + lsm_str_len(entry->attrs.items[i].str) * sizeof(char); + *size += 2 * sizeof(uint64_t) + + lsm_str_len(entry->attrs.items[i].str) * sizeof(char); } return lsm_error_ok; } -lsm_error lsm_entry_write_idx(uint64_t *size, FILE *idx_file, lsm_entry *entry, uint64_t offset, uint64_t len) { +lsm_error lsm_entry_write_idx(uint64_t *size, FILE *idx_file, lsm_entry *entry, + uint64_t offset, uint64_t len) { LSM_RES(lsm_entry_write_uint64_t(idx_file, lsm_str_len(entry->key))); LSM_RES(lsm_entry_write_str(idx_file, entry->key)); LSM_RES(lsm_entry_write_uint64_t(idx_file, offset)); @@ -66,7 +69,8 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { } uint64_t entry_size; - lsm_error res = lsm_entry_write_db(&entry_size, store->db_file, handle->wrapper->entry); + lsm_error res = + lsm_entry_write_db(&entry_size, store->db_file, handle->wrapper->entry); fflush(store->db_file); // TODO fsync db file? @@ -76,7 +80,7 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { return res; } - + uint64_t entry_index = store->db_file_size; store->db_file_size += entry_size; @@ -92,7 +96,8 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { return lsm_error_failed_io; } - res = lsm_entry_write_idx(&entry_size, store->idx_file, handle->wrapper->entry, entry_index, entry_size); + res = lsm_entry_write_idx(&entry_size, store->idx_file, + handle->wrapper->entry, entry_index, entry_size); if (res == lsm_error_ok) { // Update the counter at the beginning of the file @@ -151,14 +156,15 @@ static lsm_error lsm_entry_read_attrs(lsm_entry_handle *handle, FILE *db_file) { if (val_s == NULL) { return lsm_error_failed_alloc; } - + uint64_t read = 0; while (read < nums[1]) { read += fread(&val_s[read], 1, nums[1] - read, db_file); } - LSM_RES(lsm_str_init(&val, val_s));; + LSM_RES(lsm_str_init(&val, val_s)); + ; lsm_entry_attr_insert(handle, nums[0], val); } @@ -172,7 +178,8 @@ lsm_error lsm_store_load_db(lsm_store *store) { lsm_entry_handle *handle; // idx file starts with block count - size_t res = fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file); + size_t res = + fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file); if (res == 0) { return lsm_error_failed_io; From e10c43dfd6e1cc8b8ede0eb7e2a72f5295835fd9 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 8 Nov 2023 09:05:38 +0100 Subject: [PATCH 2/4] fix(lsm): work when first creating db --- lsm/src/store/lsm_store.c | 18 ++++++++++++++++++ lsm/src/store/lsm_store_sync.c | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/lsm/src/store/lsm_store.c b/lsm/src/store/lsm_store.c index 9c0ab83..43eb475 100644 --- a/lsm/src/store/lsm_store.c +++ b/lsm/src/store/lsm_store.c @@ -44,11 +44,20 @@ lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) { FILE *db_file = fopen(db_file_path, "r+b"); if (db_file == NULL) { + // Create the file first, then reopen it in extended read db_file = fopen(db_file_path, "wb"); if (db_file == NULL) { return lsm_error_failed_io; } + + fclose(db_file); + + FILE *db_file = fopen(db_file_path, "r+b"); + + if (db_file == NULL) { + return lsm_error_failed_io; + } } // Same for idx file @@ -60,6 +69,7 @@ lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) { FILE *idx_file = fopen(idx_file_path, "r+b"); if (idx_file == NULL) { + // Create the file first idx_file = fopen(idx_file_path, "wb"); if (idx_file == NULL) { @@ -75,6 +85,14 @@ lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) { } fflush(idx_file); + fclose(idx_file); + + // If opening it in extended read mode still fails now, there's a problem + FILE *idx_file = fopen(idx_file_path, "r+b"); + + if (idx_file == NULL) { + return lsm_error_failed_io; + } } store->data_path = data_path; diff --git a/lsm/src/store/lsm_store_sync.c b/lsm/src/store/lsm_store_sync.c index caaa42b..3a668d4 100644 --- a/lsm/src/store/lsm_store_sync.c +++ b/lsm/src/store/lsm_store_sync.c @@ -73,8 +73,6 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { lsm_entry_write_db(&entry_size, store->db_file, handle->wrapper->entry); fflush(store->db_file); - // TODO fsync db file? - if (res != lsm_error_ok) { pthread_mutex_unlock(&store->db_lock); @@ -177,6 +175,8 @@ lsm_error lsm_store_load_db(lsm_store *store) { lsm_str *key; lsm_entry_handle *handle; + rewind(store->idx_file); + // idx file starts with block count size_t res = fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file); From 9c249d40c749d99fd2e785d3dc3da0fd0f41b503 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 8 Nov 2023 09:11:07 +0100 Subject: [PATCH 3/4] refactor(lsm): better separate store disk functions --- lsm/src/store/lsm_store.c | 76 -------- lsm/src/store/lsm_store_disk_read.c | 183 ++++++++++++++++++ ...sm_store_sync.c => lsm_store_disk_write.c} | 111 ----------- 3 files changed, 183 insertions(+), 187 deletions(-) create mode 100644 lsm/src/store/lsm_store_disk_read.c rename lsm/src/store/{lsm_store_sync.c => lsm_store_disk_write.c} (58%) diff --git a/lsm/src/store/lsm_store.c b/lsm/src/store/lsm_store.c index 43eb475..2345cb8 100644 --- a/lsm/src/store/lsm_store.c +++ b/lsm/src/store/lsm_store.c @@ -30,82 +30,6 @@ lsm_error lsm_store_init(lsm_store **ptr) { return lsm_error_ok; } -lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) { - lsm_store *store; - LSM_RES(lsm_store_init(&store)); - - // Try to open an existing db file or create a new one otherwise - // This shit is why I need to improve the str library - char db_file_path[lsm_str_len(data_path) + strlen(LSM_DB_FILE_NAME) + 2]; - memcpy(db_file_path, lsm_str_ptr(data_path), - lsm_str_len(data_path) * sizeof(char)); - sprintf(&db_file_path[lsm_str_len(data_path)], "/%s", LSM_DB_FILE_NAME); - - FILE *db_file = fopen(db_file_path, "r+b"); - - if (db_file == NULL) { - // Create the file first, then reopen it in extended read - db_file = fopen(db_file_path, "wb"); - - if (db_file == NULL) { - return lsm_error_failed_io; - } - - fclose(db_file); - - FILE *db_file = fopen(db_file_path, "r+b"); - - if (db_file == NULL) { - return lsm_error_failed_io; - } - } - - // Same for idx file - char idx_file_path[lsm_str_len(data_path) + strlen(LSM_IDX_FILE_NAME) + 2]; - memcpy(idx_file_path, lsm_str_ptr(data_path), - lsm_str_len(data_path) * sizeof(char)); - sprintf(&idx_file_path[lsm_str_len(data_path)], "/%s", LSM_IDX_FILE_NAME); - - FILE *idx_file = fopen(idx_file_path, "r+b"); - - if (idx_file == NULL) { - // Create the file first - idx_file = fopen(idx_file_path, "wb"); - - if (idx_file == NULL) { - return lsm_error_failed_io; - } - - // The database code expects the idx file to start with how many blocks it - // contains, so we write that here - uint64_t num = 0; - - if (fwrite(&num, sizeof(uint64_t), 1, idx_file) == 0) { - return lsm_error_failed_io; - } - - fflush(idx_file); - fclose(idx_file); - - // If opening it in extended read mode still fails now, there's a problem - FILE *idx_file = fopen(idx_file_path, "r+b"); - - if (idx_file == NULL) { - return lsm_error_failed_io; - } - } - - store->data_path = data_path; - store->db_file = db_file; - store->idx_file = idx_file; - - LSM_RES(lsm_store_load_db(store)); - - *ptr = store; - - return lsm_error_ok; -} - lsm_error lsm_store_open_read(lsm_entry_handle **out, lsm_store *store, lsm_str *key) { lsm_entry_wrapper *wrapper; diff --git a/lsm/src/store/lsm_store_disk_read.c b/lsm/src/store/lsm_store_disk_read.c new file mode 100644 index 0000000..4c76b76 --- /dev/null +++ b/lsm/src/store/lsm_store_disk_read.c @@ -0,0 +1,183 @@ +#include +#include + +#include "lsm/store_internal.h" + +lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) { + lsm_store *store; + LSM_RES(lsm_store_init(&store)); + + // Try to open an existing db file or create a new one otherwise + // This shit is why I need to improve the str library + char db_file_path[lsm_str_len(data_path) + strlen(LSM_DB_FILE_NAME) + 2]; + memcpy(db_file_path, lsm_str_ptr(data_path), + lsm_str_len(data_path) * sizeof(char)); + sprintf(&db_file_path[lsm_str_len(data_path)], "/%s", LSM_DB_FILE_NAME); + + FILE *db_file = fopen(db_file_path, "r+b"); + + if (db_file == NULL) { + // Create the file first, then reopen it in extended read + db_file = fopen(db_file_path, "wb"); + + if (db_file == NULL) { + return lsm_error_failed_io; + } + + fclose(db_file); + + FILE *db_file = fopen(db_file_path, "r+b"); + + if (db_file == NULL) { + return lsm_error_failed_io; + } + } + + // Same for idx file + char idx_file_path[lsm_str_len(data_path) + strlen(LSM_IDX_FILE_NAME) + 2]; + memcpy(idx_file_path, lsm_str_ptr(data_path), + lsm_str_len(data_path) * sizeof(char)); + sprintf(&idx_file_path[lsm_str_len(data_path)], "/%s", LSM_IDX_FILE_NAME); + + FILE *idx_file = fopen(idx_file_path, "r+b"); + + if (idx_file == NULL) { + // Create the file first + idx_file = fopen(idx_file_path, "wb"); + + if (idx_file == NULL) { + return lsm_error_failed_io; + } + + // The database code expects the idx file to start with how many blocks it + // contains, so we write that here + uint64_t num = 0; + + if (fwrite(&num, sizeof(uint64_t), 1, idx_file) == 0) { + return lsm_error_failed_io; + } + + fflush(idx_file); + fclose(idx_file); + + // If opening it in extended read mode still fails now, there's a problem + FILE *idx_file = fopen(idx_file_path, "r+b"); + + if (idx_file == NULL) { + return lsm_error_failed_io; + } + } + + store->data_path = data_path; + store->db_file = db_file; + store->idx_file = idx_file; + + LSM_RES(lsm_store_load_db(store)); + + *ptr = store; + + return lsm_error_ok; +} + +static lsm_error lsm_entry_read_attrs(lsm_entry_handle *handle, FILE *db_file) { + uint64_t attr_count; + size_t res = fread(&attr_count, sizeof(uint64_t), 1, db_file); + + if (res == 0) { + return lsm_error_failed_io; + } + + // attr_type, val_len + uint64_t nums[2]; + lsm_str *val; + + for (uint64_t i = 0; i < attr_count; i++) { + res = fread(nums, sizeof(uint64_t), 2, db_file); + + if (res < 2) { + return lsm_error_failed_io; + } + + char *val_s = malloc(nums[1] + 1); + val_s[nums[1]] = '\0'; + + if (val_s == NULL) { + return lsm_error_failed_alloc; + } + + uint64_t read = 0; + + while (read < nums[1]) { + read += fread(&val_s[read], 1, nums[1] - read, db_file); + } + + LSM_RES(lsm_str_init(&val, val_s)); + ; + lsm_entry_attr_insert(handle, nums[0], val); + } + + return lsm_error_ok; +} + +lsm_error lsm_store_load_db(lsm_store *store) { + uint64_t key_len; + uint64_t db_dim[2]; + lsm_str *key; + lsm_entry_handle *handle; + + rewind(store->idx_file); + + // idx file starts with block count + size_t res = + fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file); + + if (res == 0) { + return lsm_error_failed_io; + } + + store->idx_file_size += sizeof(uint64_t); + + for (uint64_t i = 0; i < store->idx_file_block_count; i++) { + // Read in idx metadata + res = fread(&key_len, sizeof(uint64_t), 1, store->idx_file); + + if (res == 0) { + return lsm_error_failed_io; + } + + char *key_s = malloc(key_len + 1); + key_s[key_len] = '\0'; + + if (key_s == NULL) { + return lsm_error_failed_alloc; + } + + res = fread(key_s, 1, key_len, store->idx_file); + + if (res < key_len) { + return lsm_error_failed_io; + } + + res = fread(db_dim, sizeof(uint64_t), 2, store->idx_file); + + if (res < 2) { + return lsm_error_failed_io; + } + + LSM_RES(lsm_str_init(&key, key_s)); + LSM_RES(lsm_store_insert(&handle, store, key)); + + // Read attributes from database file + if (fseek(store->db_file, db_dim[0], SEEK_SET) != 0) { + return lsm_error_failed_io; + } + + LSM_RES(lsm_entry_read_attrs(handle, store->db_file)); + lsm_entry_close(handle); + + store->idx_file_size += 3 * sizeof(uint64_t) + key_len; + store->db_file_size += db_dim[1]; + } + + return lsm_error_ok; +} diff --git a/lsm/src/store/lsm_store_sync.c b/lsm/src/store/lsm_store_disk_write.c similarity index 58% rename from lsm/src/store/lsm_store_sync.c rename to lsm/src/store/lsm_store_disk_write.c index 3a668d4..8b02319 100644 --- a/lsm/src/store/lsm_store_sync.c +++ b/lsm/src/store/lsm_store_disk_write.c @@ -1,8 +1,3 @@ -#include -#include -#include - -#include "lsm/store.h" #include "lsm/store_internal.h" static lsm_error lsm_entry_write_uint64_t(FILE *f, uint64_t num) { @@ -110,7 +105,6 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { size_t r = fwrite(&new_block_count, sizeof(uint64_t), 1, store->idx_file); if (r != lsm_error_ok) { - printf("wuck\n"); pthread_mutex_unlock(&store->idx_lock); return res; @@ -118,8 +112,6 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { store->idx_file_size += entry_size; store->idx_file_block_count = new_block_count; - } else { - printf("failed write\n"); } fflush(store->idx_file); @@ -128,106 +120,3 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { return res; } - -static lsm_error lsm_entry_read_attrs(lsm_entry_handle *handle, FILE *db_file) { - uint64_t attr_count; - size_t res = fread(&attr_count, sizeof(uint64_t), 1, db_file); - - if (res == 0) { - return lsm_error_failed_io; - } - - // attr_type, val_len - uint64_t nums[2]; - lsm_str *val; - - for (uint64_t i = 0; i < attr_count; i++) { - res = fread(nums, sizeof(uint64_t), 2, db_file); - - if (res < 2) { - return lsm_error_failed_io; - } - - char *val_s = malloc(nums[1] + 1); - val_s[nums[1]] = '\0'; - - if (val_s == NULL) { - return lsm_error_failed_alloc; - } - - uint64_t read = 0; - - while (read < nums[1]) { - read += fread(&val_s[read], 1, nums[1] - read, db_file); - } - - LSM_RES(lsm_str_init(&val, val_s)); - ; - lsm_entry_attr_insert(handle, nums[0], val); - } - - return lsm_error_ok; -} - -lsm_error lsm_store_load_db(lsm_store *store) { - uint64_t key_len; - uint64_t db_dim[2]; - lsm_str *key; - lsm_entry_handle *handle; - - rewind(store->idx_file); - - // idx file starts with block count - size_t res = - fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file); - - if (res == 0) { - return lsm_error_failed_io; - } - - store->idx_file_size += sizeof(uint64_t); - - for (uint64_t i = 0; i < store->idx_file_block_count; i++) { - // Read in idx metadata - res = fread(&key_len, sizeof(uint64_t), 1, store->idx_file); - - if (res == 0) { - return lsm_error_failed_io; - } - - char *key_s = malloc(key_len + 1); - key_s[key_len] = '\0'; - - if (key_s == NULL) { - return lsm_error_failed_alloc; - } - - res = fread(key_s, 1, key_len, store->idx_file); - - if (res < key_len) { - return lsm_error_failed_io; - } - - res = fread(db_dim, sizeof(uint64_t), 2, store->idx_file); - - if (res < 2) { - return lsm_error_failed_io; - } - - LSM_RES(lsm_str_init(&key, key_s)); - LSM_RES(lsm_store_insert(&handle, store, key)); - - // Read attributes from database file - if (fseek(store->db_file, db_dim[0], SEEK_SET) != 0) { - return lsm_error_failed_io; - } - - LSM_RES(lsm_entry_read_attrs(handle, store->db_file)); - lsm_entry_close(handle); - - store->idx_file_size += 3 * sizeof(uint64_t) + key_len; - store->db_file_size += db_dim[1]; - } - - return lsm_error_ok; -} From 226873219bd11dbf800f44e6ca9dfa4068290247 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 8 Nov 2023 10:40:12 +0100 Subject: [PATCH 4/4] refactor(lsm): slightly clean up disk write code --- lsm/src/store/lsm_store_disk_write.c | 60 ++++++++++++---------------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/lsm/src/store/lsm_store_disk_write.c b/lsm/src/store/lsm_store_disk_write.c index 8b02319..9813f40 100644 --- a/lsm/src/store/lsm_store_disk_write.c +++ b/lsm/src/store/lsm_store_disk_write.c @@ -22,7 +22,18 @@ static lsm_error lsm_entry_write_str(FILE *f, lsm_str *s) { return lsm_error_ok; } -lsm_error lsm_entry_write_db(uint64_t *size, FILE *db_file, lsm_entry *entry) { +static lsm_error lsm_seek(FILE *f, uint64_t pos) { + if (fseek(f, pos, SEEK_SET) != 0) { + return lsm_error_failed_io; + } + + return lsm_error_ok; +} + +lsm_error lsm_entry_write_db(uint64_t *size, FILE *db_file, lsm_entry *entry, + uint64_t pos) { + LSM_RES(lsm_seek(db_file, pos)); + // First we write how many attributes follow LSM_RES(lsm_entry_write_uint64_t(db_file, entry->attrs.count)); *size = sizeof(uint64_t); @@ -42,7 +53,8 @@ lsm_error lsm_entry_write_db(uint64_t *size, FILE *db_file, lsm_entry *entry) { } lsm_error lsm_entry_write_idx(uint64_t *size, FILE *idx_file, lsm_entry *entry, - uint64_t offset, uint64_t len) { + uint64_t offset, uint64_t len, uint64_t pos) { + LSM_RES(lsm_seek(idx_file, pos)); LSM_RES(lsm_entry_write_uint64_t(idx_file, lsm_str_len(entry->key))); LSM_RES(lsm_entry_write_str(idx_file, entry->key)); LSM_RES(lsm_entry_write_uint64_t(idx_file, offset)); @@ -56,16 +68,9 @@ lsm_error lsm_entry_write_idx(uint64_t *size, FILE *idx_file, lsm_entry *entry, lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { pthread_mutex_lock(&store->db_lock); - // Append entry to end of database file - if (fseek(store->db_file, store->db_file_size, SEEK_SET) != 0) { - pthread_mutex_unlock(&store->db_lock); - - return lsm_error_failed_io; - } - uint64_t entry_size; - lsm_error res = - lsm_entry_write_db(&entry_size, store->db_file, handle->wrapper->entry); + lsm_error res = lsm_entry_write_db( + &entry_size, store->db_file, handle->wrapper->entry, store->db_file_size); fflush(store->db_file); if (res != lsm_error_ok) { @@ -82,40 +87,25 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { // Append entry to index file pthread_mutex_lock(&store->idx_lock); - if (fseek(store->idx_file, store->idx_file_size, SEEK_SET) != 0) { - printf("failed seek, %lu\n", store->idx_file_size); - pthread_mutex_unlock(&store->idx_lock); - - return lsm_error_failed_io; - } - - res = lsm_entry_write_idx(&entry_size, store->idx_file, - handle->wrapper->entry, entry_index, entry_size); + res = + lsm_entry_write_idx(&entry_size, store->idx_file, handle->wrapper->entry, + entry_index, entry_size, store->idx_file_size); if (res == lsm_error_ok) { // Update the counter at the beginning of the file + rewind(store->idx_file); + uint64_t new_block_count = store->idx_file_block_count + 1; - if (fseek(store->idx_file, 0, SEEK_SET) != 0) { - pthread_mutex_unlock(&store->idx_lock); + res = lsm_entry_write_uint64_t(store->idx_file, new_block_count); - return lsm_error_failed_io; + if (res == lsm_error_ok) { + store->idx_file_size += entry_size; + store->idx_file_block_count = new_block_count; } - - size_t r = fwrite(&new_block_count, sizeof(uint64_t), 1, store->idx_file); - - if (r != lsm_error_ok) { - pthread_mutex_unlock(&store->idx_lock); - - return res; - } - - store->idx_file_size += entry_size; - store->idx_file_block_count = new_block_count; } fflush(store->idx_file); - pthread_mutex_unlock(&store->idx_lock); return res;