Chewing_Bever
/
lander
1
0
Fork 0
Code Issues 30 Packages Projects 1 Activity

refactor(lsm): better separate store disk functions

lsm
Jef Roosens 2023-11-08 09:11:07 +01:00
parent e10c43dfd6
commit 9c249d40c7
Signed by: Jef Roosens
GPG Key ID: B75D4F293C7052DB
3 changed files with 183 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
lsm/src/store/lsm_store.c
View File

@ -30,82 +30,6 @@ lsm_error lsm_store_init(lsm_store **ptr) {
return lsm_error_ok; return lsm_error_ok;
} }
lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) {
lsm_store *store;
LSM_RES(lsm_store_init(&store));
// Try to open an existing db file or create a new one otherwise
// This shit is why I need to improve the str library
char db_file_path[lsm_str_len(data_path) + strlen(LSM_DB_FILE_NAME) + 2];
memcpy(db_file_path, lsm_str_ptr(data_path),
lsm_str_len(data_path) * sizeof(char));
sprintf(&db_file_path[lsm_str_len(data_path)], "/%s", LSM_DB_FILE_NAME);
FILE *db_file = fopen(db_file_path, "r+b");
if (db_file == NULL) {
// Create the file first, then reopen it in extended read
db_file = fopen(db_file_path, "wb");
if (db_file == NULL) {
return lsm_error_failed_io;
}
fclose(db_file);
FILE *db_file = fopen(db_file_path, "r+b");
if (db_file == NULL) {
return lsm_error_failed_io;
}
}
// Same for idx file
char idx_file_path[lsm_str_len(data_path) + strlen(LSM_IDX_FILE_NAME) + 2];
memcpy(idx_file_path, lsm_str_ptr(data_path),
lsm_str_len(data_path) * sizeof(char));
sprintf(&idx_file_path[lsm_str_len(data_path)], "/%s", LSM_IDX_FILE_NAME);
FILE *idx_file = fopen(idx_file_path, "r+b");
if (idx_file == NULL) {
// Create the file first
idx_file = fopen(idx_file_path, "wb");
if (idx_file == NULL) {
return lsm_error_failed_io;
}
// The database code expects the idx file to start with how many blocks it
// contains, so we write that here
uint64_t num = 0;
if (fwrite(&num, sizeof(uint64_t), 1, idx_file) == 0) {
return lsm_error_failed_io;
}
fflush(idx_file);
fclose(idx_file);
// If opening it in extended read mode still fails now, there's a problem
FILE *idx_file = fopen(idx_file_path, "r+b");
if (idx_file == NULL) {
return lsm_error_failed_io;
}
}
store->data_path = data_path;
store->db_file = db_file;
store->idx_file = idx_file;
LSM_RES(lsm_store_load_db(store));
*ptr = store;
return lsm_error_ok;
}
lsm_error lsm_store_open_read(lsm_entry_handle **out, lsm_store *store, lsm_error lsm_store_open_read(lsm_entry_handle **out, lsm_store *store,
lsm_str *key) { lsm_str *key) {
lsm_entry_wrapper *wrapper; lsm_entry_wrapper *wrapper;

183
lsm/src/store/lsm_store_disk_read.c 100644
View File

@ -0,0 +1,183 @@
#include <stdlib.h>
#include <string.h>
#include "lsm/store_internal.h"
lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) {
lsm_store *store;
LSM_RES(lsm_store_init(&store));
// Try to open an existing db file or create a new one otherwise
// This shit is why I need to improve the str library
char db_file_path[lsm_str_len(data_path) + strlen(LSM_DB_FILE_NAME) + 2];
memcpy(db_file_path, lsm_str_ptr(data_path),
lsm_str_len(data_path) * sizeof(char));
sprintf(&db_file_path[lsm_str_len(data_path)], "/%s", LSM_DB_FILE_NAME);
FILE *db_file = fopen(db_file_path, "r+b");
if (db_file == NULL) {
// Create the file first, then reopen it in extended read
db_file = fopen(db_file_path, "wb");
if (db_file == NULL) {
return lsm_error_failed_io;
}
fclose(db_file);
FILE *db_file = fopen(db_file_path, "r+b");
if (db_file == NULL) {
return lsm_error_failed_io;
}
}
// Same for idx file
char idx_file_path[lsm_str_len(data_path) + strlen(LSM_IDX_FILE_NAME) + 2];
memcpy(idx_file_path, lsm_str_ptr(data_path),
lsm_str_len(data_path) * sizeof(char));
sprintf(&idx_file_path[lsm_str_len(data_path)], "/%s", LSM_IDX_FILE_NAME);
FILE *idx_file = fopen(idx_file_path, "r+b");
if (idx_file == NULL) {
// Create the file first
idx_file = fopen(idx_file_path, "wb");
if (idx_file == NULL) {
return lsm_error_failed_io;
}
// The database code expects the idx file to start with how many blocks it
// contains, so we write that here
uint64_t num = 0;
if (fwrite(&num, sizeof(uint64_t), 1, idx_file) == 0) {
return lsm_error_failed_io;
}
fflush(idx_file);
fclose(idx_file);
// If opening it in extended read mode still fails now, there's a problem
FILE *idx_file = fopen(idx_file_path, "r+b");
if (idx_file == NULL) {
return lsm_error_failed_io;
}
}
store->data_path = data_path;
store->db_file = db_file;
store->idx_file = idx_file;
LSM_RES(lsm_store_load_db(store));
*ptr = store;
return lsm_error_ok;
}
static lsm_error lsm_entry_read_attrs(lsm_entry_handle *handle, FILE *db_file) {
uint64_t attr_count;
size_t res = fread(&attr_count, sizeof(uint64_t), 1, db_file);
if (res == 0) {
return lsm_error_failed_io;
}
// attr_type, val_len
uint64_t nums[2];
lsm_str *val;
for (uint64_t i = 0; i < attr_count; i++) {
res = fread(nums, sizeof(uint64_t), 2, db_file);
if (res < 2) {
return lsm_error_failed_io;
}
char *val_s = malloc(nums[1] + 1);
val_s[nums[1]] = '\0';
if (val_s == NULL) {
return lsm_error_failed_alloc;
}
uint64_t read = 0;
while (read < nums[1]) {
read += fread(&val_s[read], 1, nums[1] - read, db_file);
}
LSM_RES(lsm_str_init(&val, val_s));
;
lsm_entry_attr_insert(handle, nums[0], val);
}
return lsm_error_ok;
}
lsm_error lsm_store_load_db(lsm_store *store) {
uint64_t key_len;
uint64_t db_dim[2];
lsm_str *key;
lsm_entry_handle *handle;
rewind(store->idx_file);
// idx file starts with block count
size_t res =
fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file);
if (res == 0) {
return lsm_error_failed_io;
}
store->idx_file_size += sizeof(uint64_t);
for (uint64_t i = 0; i < store->idx_file_block_count; i++) {
// Read in idx metadata
res = fread(&key_len, sizeof(uint64_t), 1, store->idx_file);
if (res == 0) {
return lsm_error_failed_io;
}
char *key_s = malloc(key_len + 1);
key_s[key_len] = '\0';
if (key_s == NULL) {
return lsm_error_failed_alloc;
}
res = fread(key_s, 1, key_len, store->idx_file);
if (res < key_len) {
return lsm_error_failed_io;
}
res = fread(db_dim, sizeof(uint64_t), 2, store->idx_file);
if (res < 2) {
return lsm_error_failed_io;
}
LSM_RES(lsm_str_init(&key, key_s));
LSM_RES(lsm_store_insert(&handle, store, key));
// Read attributes from database file
if (fseek(store->db_file, db_dim[0], SEEK_SET) != 0) {
return lsm_error_failed_io;
}
LSM_RES(lsm_entry_read_attrs(handle, store->db_file));
lsm_entry_close(handle);
store->idx_file_size += 3 * sizeof(uint64_t) + key_len;
store->db_file_size += db_dim[1];
}
return lsm_error_ok;
}

111
lsm/src/store/lsm_store_sync.c → lsm/src/store/lsm_store_disk_write.c
View File

@ -1,8 +1,3 @@
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "lsm/store.h"
#include "lsm/store_internal.h" #include "lsm/store_internal.h"
static lsm_error lsm_entry_write_uint64_t(FILE *f, uint64_t num) { static lsm_error lsm_entry_write_uint64_t(FILE *f, uint64_t num) {
@ -110,7 +105,6 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) {
size_t r = fwrite(&new_block_count, sizeof(uint64_t), 1, store->idx_file); size_t r = fwrite(&new_block_count, sizeof(uint64_t), 1, store->idx_file);
if (r != lsm_error_ok) { if (r != lsm_error_ok) {
printf("wuck\n");
pthread_mutex_unlock(&store->idx_lock); pthread_mutex_unlock(&store->idx_lock);
return res; return res;
@ -118,8 +112,6 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) {
store->idx_file_size += entry_size; store->idx_file_size += entry_size;
store->idx_file_block_count = new_block_count; store->idx_file_block_count = new_block_count;
} else {
printf("failed write\n");
} }
fflush(store->idx_file); fflush(store->idx_file);
@ -128,106 +120,3 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) {
return res; return res;
} }
static lsm_error lsm_entry_read_attrs(lsm_entry_handle *handle, FILE *db_file) {
uint64_t attr_count;
size_t res = fread(&attr_count, sizeof(uint64_t), 1, db_file);
if (res == 0) {
return lsm_error_failed_io;
}
// attr_type, val_len
uint64_t nums[2];
lsm_str *val;
for (uint64_t i = 0; i < attr_count; i++) {
res = fread(nums, sizeof(uint64_t), 2, db_file);
if (res < 2) {
return lsm_error_failed_io;
}
char *val_s = malloc(nums[1] + 1);
val_s[nums[1]] = '\0';
if (val_s == NULL) {
return lsm_error_failed_alloc;
}
uint64_t read = 0;
while (read < nums[1]) {
read += fread(&val_s[read], 1, nums[1] - read, db_file);
}
LSM_RES(lsm_str_init(&val, val_s));
;
lsm_entry_attr_insert(handle, nums[0], val);
}
return lsm_error_ok;
}
lsm_error lsm_store_load_db(lsm_store *store) {
uint64_t key_len;
uint64_t db_dim[2];
lsm_str *key;
lsm_entry_handle *handle;
rewind(store->idx_file);
// idx file starts with block count
size_t res =
fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file);
if (res == 0) {
return lsm_error_failed_io;
}
store->idx_file_size += sizeof(uint64_t);
for (uint64_t i = 0; i < store->idx_file_block_count; i++) {
// Read in idx metadata
res = fread(&key_len, sizeof(uint64_t), 1, store->idx_file);
if (res == 0) {
return lsm_error_failed_io;
}
char *key_s = malloc(key_len + 1);
key_s[key_len] = '\0';
if (key_s == NULL) {
return lsm_error_failed_alloc;
}
res = fread(key_s, 1, key_len, store->idx_file);
if (res < key_len) {
return lsm_error_failed_io;
}
res = fread(db_dim, sizeof(uint64_t), 2, store->idx_file);
if (res < 2) {
return lsm_error_failed_io;
}
LSM_RES(lsm_str_init(&key, key_s));
LSM_RES(lsm_store_insert(&handle, store, key));
// Read attributes from database file
if (fseek(store->db_file, db_dim[0], SEEK_SET) != 0) {
return lsm_error_failed_io;
}
LSM_RES(lsm_entry_read_attrs(handle, store->db_file));
lsm_entry_close(handle);
store->idx_file_size += 3 * sizeof(uint64_t) + key_len;
store->db_file_size += db_dim[1];
}
return lsm_error_ok;
}