From 9c249d40c749d99fd2e785d3dc3da0fd0f41b503 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 8 Nov 2023 09:11:07 +0100 Subject: [PATCH] refactor(lsm): better separate store disk functions --- lsm/src/store/lsm_store.c | 76 -------- lsm/src/store/lsm_store_disk_read.c | 183 ++++++++++++++++++ ...sm_store_sync.c => lsm_store_disk_write.c} | 111 ----------- 3 files changed, 183 insertions(+), 187 deletions(-) create mode 100644 lsm/src/store/lsm_store_disk_read.c rename lsm/src/store/{lsm_store_sync.c => lsm_store_disk_write.c} (58%) diff --git a/lsm/src/store/lsm_store.c b/lsm/src/store/lsm_store.c index 43eb475..2345cb8 100644 --- a/lsm/src/store/lsm_store.c +++ b/lsm/src/store/lsm_store.c @@ -30,82 +30,6 @@ lsm_error lsm_store_init(lsm_store **ptr) { return lsm_error_ok; } -lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) { - lsm_store *store; - LSM_RES(lsm_store_init(&store)); - - // Try to open an existing db file or create a new one otherwise - // This shit is why I need to improve the str library - char db_file_path[lsm_str_len(data_path) + strlen(LSM_DB_FILE_NAME) + 2]; - memcpy(db_file_path, lsm_str_ptr(data_path), - lsm_str_len(data_path) * sizeof(char)); - sprintf(&db_file_path[lsm_str_len(data_path)], "/%s", LSM_DB_FILE_NAME); - - FILE *db_file = fopen(db_file_path, "r+b"); - - if (db_file == NULL) { - // Create the file first, then reopen it in extended read - db_file = fopen(db_file_path, "wb"); - - if (db_file == NULL) { - return lsm_error_failed_io; - } - - fclose(db_file); - - FILE *db_file = fopen(db_file_path, "r+b"); - - if (db_file == NULL) { - return lsm_error_failed_io; - } - } - - // Same for idx file - char idx_file_path[lsm_str_len(data_path) + strlen(LSM_IDX_FILE_NAME) + 2]; - memcpy(idx_file_path, lsm_str_ptr(data_path), - lsm_str_len(data_path) * sizeof(char)); - sprintf(&idx_file_path[lsm_str_len(data_path)], "/%s", LSM_IDX_FILE_NAME); - - FILE *idx_file = fopen(idx_file_path, "r+b"); - - if (idx_file == NULL) { - // Create the file first - idx_file = fopen(idx_file_path, "wb"); - - if (idx_file == NULL) { - return lsm_error_failed_io; - } - - // The database code expects the idx file to start with how many blocks it - // contains, so we write that here - uint64_t num = 0; - - if (fwrite(&num, sizeof(uint64_t), 1, idx_file) == 0) { - return lsm_error_failed_io; - } - - fflush(idx_file); - fclose(idx_file); - - // If opening it in extended read mode still fails now, there's a problem - FILE *idx_file = fopen(idx_file_path, "r+b"); - - if (idx_file == NULL) { - return lsm_error_failed_io; - } - } - - store->data_path = data_path; - store->db_file = db_file; - store->idx_file = idx_file; - - LSM_RES(lsm_store_load_db(store)); - - *ptr = store; - - return lsm_error_ok; -} - lsm_error lsm_store_open_read(lsm_entry_handle **out, lsm_store *store, lsm_str *key) { lsm_entry_wrapper *wrapper; diff --git a/lsm/src/store/lsm_store_disk_read.c b/lsm/src/store/lsm_store_disk_read.c new file mode 100644 index 0000000..4c76b76 --- /dev/null +++ b/lsm/src/store/lsm_store_disk_read.c @@ -0,0 +1,183 @@ +#include +#include + +#include "lsm/store_internal.h" + +lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) { + lsm_store *store; + LSM_RES(lsm_store_init(&store)); + + // Try to open an existing db file or create a new one otherwise + // This shit is why I need to improve the str library + char db_file_path[lsm_str_len(data_path) + strlen(LSM_DB_FILE_NAME) + 2]; + memcpy(db_file_path, lsm_str_ptr(data_path), + lsm_str_len(data_path) * sizeof(char)); + sprintf(&db_file_path[lsm_str_len(data_path)], "/%s", LSM_DB_FILE_NAME); + + FILE *db_file = fopen(db_file_path, "r+b"); + + if (db_file == NULL) { + // Create the file first, then reopen it in extended read + db_file = fopen(db_file_path, "wb"); + + if (db_file == NULL) { + return lsm_error_failed_io; + } + + fclose(db_file); + + FILE *db_file = fopen(db_file_path, "r+b"); + + if (db_file == NULL) { + return lsm_error_failed_io; + } + } + + // Same for idx file + char idx_file_path[lsm_str_len(data_path) + strlen(LSM_IDX_FILE_NAME) + 2]; + memcpy(idx_file_path, lsm_str_ptr(data_path), + lsm_str_len(data_path) * sizeof(char)); + sprintf(&idx_file_path[lsm_str_len(data_path)], "/%s", LSM_IDX_FILE_NAME); + + FILE *idx_file = fopen(idx_file_path, "r+b"); + + if (idx_file == NULL) { + // Create the file first + idx_file = fopen(idx_file_path, "wb"); + + if (idx_file == NULL) { + return lsm_error_failed_io; + } + + // The database code expects the idx file to start with how many blocks it + // contains, so we write that here + uint64_t num = 0; + + if (fwrite(&num, sizeof(uint64_t), 1, idx_file) == 0) { + return lsm_error_failed_io; + } + + fflush(idx_file); + fclose(idx_file); + + // If opening it in extended read mode still fails now, there's a problem + FILE *idx_file = fopen(idx_file_path, "r+b"); + + if (idx_file == NULL) { + return lsm_error_failed_io; + } + } + + store->data_path = data_path; + store->db_file = db_file; + store->idx_file = idx_file; + + LSM_RES(lsm_store_load_db(store)); + + *ptr = store; + + return lsm_error_ok; +} + +static lsm_error lsm_entry_read_attrs(lsm_entry_handle *handle, FILE *db_file) { + uint64_t attr_count; + size_t res = fread(&attr_count, sizeof(uint64_t), 1, db_file); + + if (res == 0) { + return lsm_error_failed_io; + } + + // attr_type, val_len + uint64_t nums[2]; + lsm_str *val; + + for (uint64_t i = 0; i < attr_count; i++) { + res = fread(nums, sizeof(uint64_t), 2, db_file); + + if (res < 2) { + return lsm_error_failed_io; + } + + char *val_s = malloc(nums[1] + 1); + val_s[nums[1]] = '\0'; + + if (val_s == NULL) { + return lsm_error_failed_alloc; + } + + uint64_t read = 0; + + while (read < nums[1]) { + read += fread(&val_s[read], 1, nums[1] - read, db_file); + } + + LSM_RES(lsm_str_init(&val, val_s)); + ; + lsm_entry_attr_insert(handle, nums[0], val); + } + + return lsm_error_ok; +} + +lsm_error lsm_store_load_db(lsm_store *store) { + uint64_t key_len; + uint64_t db_dim[2]; + lsm_str *key; + lsm_entry_handle *handle; + + rewind(store->idx_file); + + // idx file starts with block count + size_t res = + fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file); + + if (res == 0) { + return lsm_error_failed_io; + } + + store->idx_file_size += sizeof(uint64_t); + + for (uint64_t i = 0; i < store->idx_file_block_count; i++) { + // Read in idx metadata + res = fread(&key_len, sizeof(uint64_t), 1, store->idx_file); + + if (res == 0) { + return lsm_error_failed_io; + } + + char *key_s = malloc(key_len + 1); + key_s[key_len] = '\0'; + + if (key_s == NULL) { + return lsm_error_failed_alloc; + } + + res = fread(key_s, 1, key_len, store->idx_file); + + if (res < key_len) { + return lsm_error_failed_io; + } + + res = fread(db_dim, sizeof(uint64_t), 2, store->idx_file); + + if (res < 2) { + return lsm_error_failed_io; + } + + LSM_RES(lsm_str_init(&key, key_s)); + LSM_RES(lsm_store_insert(&handle, store, key)); + + // Read attributes from database file + if (fseek(store->db_file, db_dim[0], SEEK_SET) != 0) { + return lsm_error_failed_io; + } + + LSM_RES(lsm_entry_read_attrs(handle, store->db_file)); + lsm_entry_close(handle); + + store->idx_file_size += 3 * sizeof(uint64_t) + key_len; + store->db_file_size += db_dim[1]; + } + + return lsm_error_ok; +} diff --git a/lsm/src/store/lsm_store_sync.c b/lsm/src/store/lsm_store_disk_write.c similarity index 58% rename from lsm/src/store/lsm_store_sync.c rename to lsm/src/store/lsm_store_disk_write.c index 3a668d4..8b02319 100644 --- a/lsm/src/store/lsm_store_sync.c +++ b/lsm/src/store/lsm_store_disk_write.c @@ -1,8 +1,3 @@ -#include -#include -#include - -#include "lsm/store.h" #include "lsm/store_internal.h" static lsm_error lsm_entry_write_uint64_t(FILE *f, uint64_t num) { @@ -110,7 +105,6 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { size_t r = fwrite(&new_block_count, sizeof(uint64_t), 1, store->idx_file); if (r != lsm_error_ok) { - printf("wuck\n"); pthread_mutex_unlock(&store->idx_lock); return res; @@ -118,8 +112,6 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { store->idx_file_size += entry_size; store->idx_file_block_count = new_block_count; - } else { - printf("failed write\n"); } fflush(store->idx_file); @@ -128,106 +120,3 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) { return res; } - -static lsm_error lsm_entry_read_attrs(lsm_entry_handle *handle, FILE *db_file) { - uint64_t attr_count; - size_t res = fread(&attr_count, sizeof(uint64_t), 1, db_file); - - if (res == 0) { - return lsm_error_failed_io; - } - - // attr_type, val_len - uint64_t nums[2]; - lsm_str *val; - - for (uint64_t i = 0; i < attr_count; i++) { - res = fread(nums, sizeof(uint64_t), 2, db_file); - - if (res < 2) { - return lsm_error_failed_io; - } - - char *val_s = malloc(nums[1] + 1); - val_s[nums[1]] = '\0'; - - if (val_s == NULL) { - return lsm_error_failed_alloc; - } - - uint64_t read = 0; - - while (read < nums[1]) { - read += fread(&val_s[read], 1, nums[1] - read, db_file); - } - - LSM_RES(lsm_str_init(&val, val_s)); - ; - lsm_entry_attr_insert(handle, nums[0], val); - } - - return lsm_error_ok; -} - -lsm_error lsm_store_load_db(lsm_store *store) { - uint64_t key_len; - uint64_t db_dim[2]; - lsm_str *key; - lsm_entry_handle *handle; - - rewind(store->idx_file); - - // idx file starts with block count - size_t res = - fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file); - - if (res == 0) { - return lsm_error_failed_io; - } - - store->idx_file_size += sizeof(uint64_t); - - for (uint64_t i = 0; i < store->idx_file_block_count; i++) { - // Read in idx metadata - res = fread(&key_len, sizeof(uint64_t), 1, store->idx_file); - - if (res == 0) { - return lsm_error_failed_io; - } - - char *key_s = malloc(key_len + 1); - key_s[key_len] = '\0'; - - if (key_s == NULL) { - return lsm_error_failed_alloc; - } - - res = fread(key_s, 1, key_len, store->idx_file); - - if (res < key_len) { - return lsm_error_failed_io; - } - - res = fread(db_dim, sizeof(uint64_t), 2, store->idx_file); - - if (res < 2) { - return lsm_error_failed_io; - } - - LSM_RES(lsm_str_init(&key, key_s)); - LSM_RES(lsm_store_insert(&handle, store, key)); - - // Read attributes from database file - if (fseek(store->db_file, db_dim[0], SEEK_SET) != 0) { - return lsm_error_failed_io; - } - - LSM_RES(lsm_entry_read_attrs(handle, store->db_file)); - lsm_entry_close(handle); - - store->idx_file_size += 3 * sizeof(uint64_t) + key_len; - store->db_file_size += db_dim[1]; - } - - return lsm_error_ok; -}