feat(lsm): store data files in nested subdirectories

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/lander/src/branch/dev)
 
+### Added
+
 * HTTP Loop
     * Fully decoupled functionality from Lander-specific code
     * Users can now define custom global & request-local contexts

lsm/include/lsm/store.h

@@ -7,6 +7,8 @@
 #include "lsm.h"
 #include "lsm/str.h"
 
+#define LSM_STORE_DATA_LEVELS 3
+
 /**
  * A handle referencing an entry inside a store. Read/write operations from/to
  * the entry go through this handle.
@@ -155,7 +157,7 @@ lsm_error lsm_store_open_read(lsm_entry_handle **out, lsm_store *store,
  * @param key key to search
  */
 lsm_error lsm_store_open_write(lsm_entry_handle **out, lsm_store *store,
-                              const lsm_str *key);
+                               const lsm_str *key);
 
 /**
  * Close an open entry handle.

lsm/src/_include/lsm/store_internal.h

@@ -133,12 +133,27 @@ lsm_error lsm_entry_disk_update(lsm_entry_handle *handle);
 /**
  * Return the length of the path to this entry's data file
  */
-uint64_t lsm_entry_data_path_len(lsm_entry_handle *handle);
+uint64_t lsm_entry_data_path_len(const lsm_entry_handle *handle);
 
 /**
  * Fill in the entry's data file path in the provided buffer. Use
  * `lsm_entry_data_path_len` to allocate an appropriately-sized buffer
  */
-void lsm_entry_data_path(char *buf, lsm_entry_handle *handle);
+void lsm_entry_data_path(char *buf, const lsm_entry_handle *handle);
+
+/**
+ * Open the entry's data file for reading
+ *
+ * @param handle handle to the entry
+ */
+lsm_error lsm_entry_data_open_read(lsm_entry_handle *handle);
+
+/**
+ * Open the entry's data file for writing. The file and all subdirectories in
+ * the data dir are created as needed.
+ *
+ * @param handle handle to the entry
+ */
+lsm_error lsm_entry_data_open_write(lsm_entry_handle *handle);
 
 #endif

lsm/src/store/lsm_store.c

@@ -48,7 +48,8 @@ lsm_error lsm_store_open_read(lsm_entry_handle **out, lsm_store *store,
   }
 
   lsm_entry_handle *handle;
-  LSM_RES2(lsm_entry_handle_init(&handle), pthread_rwlock_unlock(&wrapper->lock));
+  LSM_RES2(lsm_entry_handle_init(&handle),
+           pthread_rwlock_unlock(&wrapper->lock));
 
   handle->wrapper = wrapper;
   handle->store = store;
@@ -77,7 +78,8 @@ lsm_error lsm_store_open_write(lsm_entry_handle **out, lsm_store *store,
   }
 
   lsm_entry_handle *handle;
-  LSM_RES2(lsm_entry_handle_init(&handle), pthread_rwlock_unlock(&wrapper->lock));
+  LSM_RES2(lsm_entry_handle_init(&handle),
+           pthread_rwlock_unlock(&wrapper->lock));
 
   handle->wrapper = wrapper;
   handle->store = store;
@@ -149,16 +151,9 @@ lsm_error lsm_entry_data_append(lsm_entry_handle *handle, const lsm_str *data) {
 
   // Entries don't open their file unless needed
   if (handle->f == NULL) {
-    char data_path[lsm_entry_data_path_len(handle) + 1];
+    // An entry with no existing data will not have a data file yet, so we set
-    lsm_entry_data_path(data_path, handle);
+    // create to true then
-
+    LSM_RES(lsm_entry_data_open_write(handle));
-    FILE *f = fopen(data_path, "ab");
-
-    if (f == NULL) {
-      return lsm_error_failed_io;
-    }
-
-    handle->f = f;
   }
 
   size_t written = 0;
@@ -187,16 +182,7 @@ lsm_error lsm_entry_data_read(uint64_t *out, char *buf,
 
   // Entries don't open their file unless needed
   if (handle->f == NULL) {
-    char data_path[lsm_entry_data_path_len(handle) + 1];
+    LSM_RES(lsm_entry_data_open_read(handle));
-    lsm_entry_data_path(data_path, handle);
-
-    FILE *f = fopen(data_path, "rb");
-
-    if (f == NULL) {
-      return lsm_error_failed_io;
-    }
-
-    handle->f = f;
   }
 
   uint64_t read;

lsm/src/store/lsm_store_entry.c

@@ -1,9 +1,12 @@
+#include <errno.h>
 #include <fcntl.h>
 #include <pthread.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
 
 #include "lsm.h"
+#include "lsm/store.h"
 #include "lsm/store_internal.h"
 #include "lsm/str.h"
 
@@ -228,24 +231,86 @@ uint64_t lsm_entry_data_len(lsm_entry_handle *handle) {
   return handle->wrapper->entry->data_len;
 }
 
-uint64_t lsm_entry_data_path_len(lsm_entry_handle *handle) {
+uint64_t lsm_entry_data_path_len(const lsm_entry_handle *handle) {
-  // [data path]/[entry key][data file suffix]
+  const lsm_str *data_path = handle->store->data_path;
-  return lsm_str_len(handle->store->data_path) +
+  const lsm_str *key = handle->wrapper->entry->key;
-         lsm_str_len(handle->wrapper->entry->key) +
+
-         strlen(LSM_DATA_FILE_SUFFIX) + 1;
+  uint8_t levels =
+      key->len <= LSM_STORE_DATA_LEVELS ? key->len : LSM_STORE_DATA_LEVELS;
+
+  return data_path->len + 1 + 2 * levels + key->len +
+         strlen(LSM_DATA_FILE_SUFFIX);
 }
-void lsm_entry_data_path(char *buf, lsm_entry_handle *handle) {
-  lsm_str *data_path = handle->store->data_path;
-  lsm_str *key = handle->wrapper->entry->key;
 
-  memcpy(buf, lsm_str_ptr(data_path), lsm_str_len(data_path));
+void lsm_entry_data_path(char *path, const lsm_entry_handle *handle) {
+  const lsm_str *data_path = handle->store->data_path;
+  const lsm_str *key = handle->wrapper->entry->key;
 
-  uint64_t index = lsm_str_len(data_path);
+  uint8_t levels =
-  buf[index] = '/';
+      key->len <= LSM_STORE_DATA_LEVELS ? key->len : LSM_STORE_DATA_LEVELS;
 
-  index += 1;
+  memcpy(path, lsm_str_ptr(data_path), data_path->len);
-  memcpy(&buf[index], lsm_str_ptr(key), lsm_str_len(key));
+  path[data_path->len] = '/';
+
+  uint64_t index = data_path->len + 1;
+
+  // Create each directory in the file hierarchy
+  for (uint8_t i = 0; i < levels; i++) {
+    path[index] = lsm_str_char(key, i);
+    path[index + 1] = '/';
+
+    index += 2;
+  }
+
+  memcpy(&path[index], lsm_str_ptr(key), lsm_str_len(key));
 
   index += lsm_str_len(key);
-  strcpy(&buf[index], LSM_DATA_FILE_SUFFIX);
+  strcpy(&path[index], LSM_DATA_FILE_SUFFIX);
+}
+
+lsm_error lsm_entry_data_open_write(lsm_entry_handle *handle) {
+  char path[lsm_entry_data_path_len(handle) + 1];
+  lsm_entry_data_path(path, handle);
+
+  const lsm_str *data_path = handle->store->data_path;
+  const lsm_str *key = handle->wrapper->entry->key;
+
+  uint8_t levels =
+      key->len <= LSM_STORE_DATA_LEVELS ? key->len : LSM_STORE_DATA_LEVELS;
+
+  // Create all required directories in the path
+  for (uint8_t i = 0; i < levels; i++) {
+    path[data_path->len + 2 * (i + 1)] = '\0';
+
+    if ((mkdir(path, 0755) != 0) && (errno != EEXIST)) {
+      return lsm_error_failed_io;
+    }
+
+    path[data_path->len + 2 * (i + 1)] = '/';
+  }
+
+  FILE *f = fopen(path, "ab");
+
+  if (f == NULL) {
+    return lsm_error_failed_io;
+  }
+
+  handle->f = f;
+
+  return lsm_error_ok;
+}
+
+lsm_error lsm_entry_data_open_read(lsm_entry_handle *handle) {
+  char path[lsm_entry_data_path_len(handle) + 1];
+  lsm_entry_data_path(path, handle);
+
+  FILE *f = fopen(path, "rb");
+
+  if (f == NULL) {
+    return lsm_error_failed_io;
+  }
+
+  handle->f = f;
+
+  return lsm_error_ok;
 }