refactor(lsm): better separate store disk functions
parent
e10c43dfd6
commit
9c249d40c7
GPG Key ID:
B75D4F293C7052DB
|
|
@ -30,82 +30,6 @@ lsm_error lsm_store_init(lsm_store **ptr) {
|
|||
return lsm_error_ok;
|
||||
}
|
||||
|
||||
lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) {
|
||||
lsm_store *store;
|
||||
LSM_RES(lsm_store_init(&store));
|
||||
|
||||
// Try to open an existing db file or create a new one otherwise
|
||||
// This shit is why I need to improve the str library
|
||||
char db_file_path[lsm_str_len(data_path) + strlen(LSM_DB_FILE_NAME) + 2];
|
||||
memcpy(db_file_path, lsm_str_ptr(data_path),
|
||||
lsm_str_len(data_path) * sizeof(char));
|
||||
sprintf(&db_file_path[lsm_str_len(data_path)], "/%s", LSM_DB_FILE_NAME);
|
||||
|
||||
FILE *db_file = fopen(db_file_path, "r+b");
|
||||
|
||||
if (db_file == NULL) {
|
||||
// Create the file first, then reopen it in extended read
|
||||
db_file = fopen(db_file_path, "wb");
|
||||
|
||||
if (db_file == NULL) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
fclose(db_file);
|
||||
|
||||
FILE *db_file = fopen(db_file_path, "r+b");
|
||||
|
||||
if (db_file == NULL) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
}
|
||||
|
||||
// Same for idx file
|
||||
char idx_file_path[lsm_str_len(data_path) + strlen(LSM_IDX_FILE_NAME) + 2];
|
||||
memcpy(idx_file_path, lsm_str_ptr(data_path),
|
||||
lsm_str_len(data_path) * sizeof(char));
|
||||
sprintf(&idx_file_path[lsm_str_len(data_path)], "/%s", LSM_IDX_FILE_NAME);
|
||||
|
||||
FILE *idx_file = fopen(idx_file_path, "r+b");
|
||||
|
||||
if (idx_file == NULL) {
|
||||
// Create the file first
|
||||
idx_file = fopen(idx_file_path, "wb");
|
||||
|
||||
if (idx_file == NULL) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
// The database code expects the idx file to start with how many blocks it
|
||||
// contains, so we write that here
|
||||
uint64_t num = 0;
|
||||
|
||||
if (fwrite(&num, sizeof(uint64_t), 1, idx_file) == 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
fflush(idx_file);
|
||||
fclose(idx_file);
|
||||
|
||||
// If opening it in extended read mode still fails now, there's a problem
|
||||
FILE *idx_file = fopen(idx_file_path, "r+b");
|
||||
|
||||
if (idx_file == NULL) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
}
|
||||
|
||||
store->data_path = data_path;
|
||||
store->db_file = db_file;
|
||||
store->idx_file = idx_file;
|
||||
|
||||
LSM_RES(lsm_store_load_db(store));
|
||||
|
||||
*ptr = store;
|
||||
|
||||
return lsm_error_ok;
|
||||
}
|
||||
|
||||
lsm_error lsm_store_open_read(lsm_entry_handle **out, lsm_store *store,
|
||||
lsm_str *key) {
|
||||
lsm_entry_wrapper *wrapper;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,183 @@
|
|||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "lsm/store_internal.h"
|
||||
|
||||
lsm_error lsm_store_load(lsm_store **ptr, lsm_str *data_path) {
|
||||
lsm_store *store;
|
||||
LSM_RES(lsm_store_init(&store));
|
||||
|
||||
// Try to open an existing db file or create a new one otherwise
|
||||
// This shit is why I need to improve the str library
|
||||
char db_file_path[lsm_str_len(data_path) + strlen(LSM_DB_FILE_NAME) + 2];
|
||||
memcpy(db_file_path, lsm_str_ptr(data_path),
|
||||
lsm_str_len(data_path) * sizeof(char));
|
||||
sprintf(&db_file_path[lsm_str_len(data_path)], "/%s", LSM_DB_FILE_NAME);
|
||||
|
||||
FILE *db_file = fopen(db_file_path, "r+b");
|
||||
|
||||
if (db_file == NULL) {
|
||||
// Create the file first, then reopen it in extended read
|
||||
db_file = fopen(db_file_path, "wb");
|
||||
|
||||
if (db_file == NULL) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
fclose(db_file);
|
||||
|
||||
FILE *db_file = fopen(db_file_path, "r+b");
|
||||
|
||||
if (db_file == NULL) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
}
|
||||
|
||||
// Same for idx file
|
||||
char idx_file_path[lsm_str_len(data_path) + strlen(LSM_IDX_FILE_NAME) + 2];
|
||||
memcpy(idx_file_path, lsm_str_ptr(data_path),
|
||||
lsm_str_len(data_path) * sizeof(char));
|
||||
sprintf(&idx_file_path[lsm_str_len(data_path)], "/%s", LSM_IDX_FILE_NAME);
|
||||
|
||||
FILE *idx_file = fopen(idx_file_path, "r+b");
|
||||
|
||||
if (idx_file == NULL) {
|
||||
// Create the file first
|
||||
idx_file = fopen(idx_file_path, "wb");
|
||||
|
||||
if (idx_file == NULL) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
// The database code expects the idx file to start with how many blocks it
|
||||
// contains, so we write that here
|
||||
uint64_t num = 0;
|
||||
|
||||
if (fwrite(&num, sizeof(uint64_t), 1, idx_file) == 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
fflush(idx_file);
|
||||
fclose(idx_file);
|
||||
|
||||
// If opening it in extended read mode still fails now, there's a problem
|
||||
FILE *idx_file = fopen(idx_file_path, "r+b");
|
||||
|
||||
if (idx_file == NULL) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
}
|
||||
|
||||
store->data_path = data_path;
|
||||
store->db_file = db_file;
|
||||
store->idx_file = idx_file;
|
||||
|
||||
LSM_RES(lsm_store_load_db(store));
|
||||
|
||||
*ptr = store;
|
||||
|
||||
return lsm_error_ok;
|
||||
}
|
||||
|
||||
static lsm_error lsm_entry_read_attrs(lsm_entry_handle *handle, FILE *db_file) {
|
||||
uint64_t attr_count;
|
||||
size_t res = fread(&attr_count, sizeof(uint64_t), 1, db_file);
|
||||
|
||||
if (res == 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
// attr_type, val_len
|
||||
uint64_t nums[2];
|
||||
lsm_str *val;
|
||||
|
||||
for (uint64_t i = 0; i < attr_count; i++) {
|
||||
res = fread(nums, sizeof(uint64_t), 2, db_file);
|
||||
|
||||
if (res < 2) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
char *val_s = malloc(nums[1] + 1);
|
||||
val_s[nums[1]] = '\0';
|
||||
|
||||
if (val_s == NULL) {
|
||||
return lsm_error_failed_alloc;
|
||||
}
|
||||
|
||||
uint64_t read = 0;
|
||||
|
||||
while (read < nums[1]) {
|
||||
read += fread(&val_s[read], 1, nums[1] - read, db_file);
|
||||
}
|
||||
|
||||
LSM_RES(lsm_str_init(&val, val_s));
|
||||
;
|
||||
lsm_entry_attr_insert(handle, nums[0], val);
|
||||
}
|
||||
|
||||
return lsm_error_ok;
|
||||
}
|
||||
|
||||
lsm_error lsm_store_load_db(lsm_store *store) {
|
||||
uint64_t key_len;
|
||||
uint64_t db_dim[2];
|
||||
lsm_str *key;
|
||||
lsm_entry_handle *handle;
|
||||
|
||||
rewind(store->idx_file);
|
||||
|
||||
// idx file starts with block count
|
||||
size_t res =
|
||||
fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file);
|
||||
|
||||
if (res == 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
store->idx_file_size += sizeof(uint64_t);
|
||||
|
||||
for (uint64_t i = 0; i < store->idx_file_block_count; i++) {
|
||||
// Read in idx metadata
|
||||
res = fread(&key_len, sizeof(uint64_t), 1, store->idx_file);
|
||||
|
||||
if (res == 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
char *key_s = malloc(key_len + 1);
|
||||
key_s[key_len] = '\0';
|
||||
|
||||
if (key_s == NULL) {
|
||||
return lsm_error_failed_alloc;
|
||||
}
|
||||
|
||||
res = fread(key_s, 1, key_len, store->idx_file);
|
||||
|
||||
if (res < key_len) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
res = fread(db_dim, sizeof(uint64_t), 2, store->idx_file);
|
||||
|
||||
if (res < 2) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
LSM_RES(lsm_str_init(&key, key_s));
|
||||
LSM_RES(lsm_store_insert(&handle, store, key));
|
||||
|
||||
// Read attributes from database file
|
||||
if (fseek(store->db_file, db_dim[0], SEEK_SET) != 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
LSM_RES(lsm_entry_read_attrs(handle, store->db_file));
|
||||
lsm_entry_close(handle);
|
||||
|
||||
store->idx_file_size += 3 * sizeof(uint64_t) + key_len;
|
||||
store->db_file_size += db_dim[1];
|
||||
}
|
||||
|
||||
return lsm_error_ok;
|
||||
}
|
||||
|
|
@ -1,8 +1,3 @@
|
|||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "lsm/store.h"
|
||||
#include "lsm/store_internal.h"
|
||||
|
||||
static lsm_error lsm_entry_write_uint64_t(FILE *f, uint64_t num) {
|
||||
|
|
@ -110,7 +105,6 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) {
|
|||
size_t r = fwrite(&new_block_count, sizeof(uint64_t), 1, store->idx_file);
|
||||
|
||||
if (r != lsm_error_ok) {
|
||||
printf("wuck\n");
|
||||
pthread_mutex_unlock(&store->idx_lock);
|
||||
|
||||
return res;
|
||||
|
|
@ -118,8 +112,6 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) {
|
|||
|
||||
store->idx_file_size += entry_size;
|
||||
store->idx_file_block_count = new_block_count;
|
||||
} else {
|
||||
printf("failed write\n");
|
||||
}
|
||||
|
||||
fflush(store->idx_file);
|
||||
|
|
@ -128,106 +120,3 @@ lsm_error lsm_entry_sync(lsm_store *store, lsm_entry_handle *handle) {
|
|||
|
||||
return res;
|
||||
}
|
||||
|
||||
static lsm_error lsm_entry_read_attrs(lsm_entry_handle *handle, FILE *db_file) {
|
||||
uint64_t attr_count;
|
||||
size_t res = fread(&attr_count, sizeof(uint64_t), 1, db_file);
|
||||
|
||||
if (res == 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
// attr_type, val_len
|
||||
uint64_t nums[2];
|
||||
lsm_str *val;
|
||||
|
||||
for (uint64_t i = 0; i < attr_count; i++) {
|
||||
res = fread(nums, sizeof(uint64_t), 2, db_file);
|
||||
|
||||
if (res < 2) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
char *val_s = malloc(nums[1] + 1);
|
||||
val_s[nums[1]] = '\0';
|
||||
|
||||
if (val_s == NULL) {
|
||||
return lsm_error_failed_alloc;
|
||||
}
|
||||
|
||||
uint64_t read = 0;
|
||||
|
||||
while (read < nums[1]) {
|
||||
read += fread(&val_s[read], 1, nums[1] - read, db_file);
|
||||
}
|
||||
|
||||
LSM_RES(lsm_str_init(&val, val_s));
|
||||
;
|
||||
lsm_entry_attr_insert(handle, nums[0], val);
|
||||
}
|
||||
|
||||
return lsm_error_ok;
|
||||
}
|
||||
|
||||
lsm_error lsm_store_load_db(lsm_store *store) {
|
||||
uint64_t key_len;
|
||||
uint64_t db_dim[2];
|
||||
lsm_str *key;
|
||||
lsm_entry_handle *handle;
|
||||
|
||||
rewind(store->idx_file);
|
||||
|
||||
// idx file starts with block count
|
||||
size_t res =
|
||||
fread(&store->idx_file_block_count, sizeof(uint64_t), 1, store->idx_file);
|
||||
|
||||
if (res == 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
store->idx_file_size += sizeof(uint64_t);
|
||||
|
||||
for (uint64_t i = 0; i < store->idx_file_block_count; i++) {
|
||||
// Read in idx metadata
|
||||
res = fread(&key_len, sizeof(uint64_t), 1, store->idx_file);
|
||||
|
||||
if (res == 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
char *key_s = malloc(key_len + 1);
|
||||
key_s[key_len] = '\0';
|
||||
|
||||
if (key_s == NULL) {
|
||||
return lsm_error_failed_alloc;
|
||||
}
|
||||
|
||||
res = fread(key_s, 1, key_len, store->idx_file);
|
||||
|
||||
if (res < key_len) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
res = fread(db_dim, sizeof(uint64_t), 2, store->idx_file);
|
||||
|
||||
if (res < 2) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
LSM_RES(lsm_str_init(&key, key_s));
|
||||
LSM_RES(lsm_store_insert(&handle, store, key));
|
||||
|
||||
// Read attributes from database file
|
||||
if (fseek(store->db_file, db_dim[0], SEEK_SET) != 0) {
|
||||
return lsm_error_failed_io;
|
||||
}
|
||||
|
||||
LSM_RES(lsm_entry_read_attrs(handle, store->db_file));
|
||||
lsm_entry_close(handle);
|
||||
|
||||
store->idx_file_size += 3 * sizeof(uint64_t) + key_len;
|
||||
store->db_file_size += db_dim[1];
|
||||
}
|
||||
|
||||
return lsm_error_ok;
|
||||
}
|
||||
Loading…
Reference in New Issue