From 79c31589748f88dc50d01be89c3da905518910c5 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 28 Aug 2024 20:29:47 +0200 Subject: [PATCH 1/3] feat(lsm): implement updating entries --- lsm/src/store/lsm_store_disk_write.c | 34 +++++++++++++++++++++------- lsm/src/store/lsm_store_entry.c | 5 ++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/lsm/src/store/lsm_store_disk_write.c b/lsm/src/store/lsm_store_disk_write.c index 51e9be8..543480c 100644 --- a/lsm/src/store/lsm_store_disk_write.c +++ b/lsm/src/store/lsm_store_disk_write.c @@ -1,3 +1,5 @@ +#include + #include "lsm/store_internal.h" static lsm_error lsm_fwrite(uint64_t *sum, FILE *f, uint64_t size, @@ -125,16 +127,10 @@ lsm_error lsm_entry_disk_insert(lsm_entry_handle *handle) { return res; } -// Marking an entry as removed in the idx file is simply setting the length of -// its entry to zero -lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { - lsm_store *store = handle->store; - const lsm_entry *entry = handle->wrapper->entry; - +static lsm_error lsm_idx_zero_block(lsm_store *store, uint64_t pos) { pthread_mutex_lock(&store->idx.lock); - lsm_error res = - lsm_fseek(store->idx.f, entry->idx_file_offset + sizeof(uint64_t)); + lsm_error res = lsm_fseek(store->idx.f, pos); if (res != lsm_error_ok) { pthread_mutex_unlock(&store->idx.lock); @@ -153,7 +149,29 @@ lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { fflush(store->idx.f); + return lsm_error_ok; +} + +// Marking an entry as removed in the idx file is simply setting the length of +// its entry to zero +lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { + const lsm_entry *entry = handle->wrapper->entry; + + LSM_RES(lsm_idx_zero_block(handle->store, + entry->idx_file_offset * sizeof(uint64_t))); LSM_RES(lsm_entry_data_remove(handle)); return lsm_error_ok; } + +lsm_error lsm_entry_disk_update(lsm_entry_handle *handle) { + // An update is implemented by reinserting the entry at the end of the db file + uint64_t old_idx_index = handle->wrapper->entry->idx_file_offset; + + // TODO is there any way we can make this atomic? If the zero write to the + // index file fails, there are two entries in the db file for the same key. + LSM_RES(lsm_entry_disk_insert(handle)); + LSM_RES(lsm_idx_zero_block(handle->store, old_idx_index * sizeof(uint64_t))); + + return lsm_error_ok; +} diff --git a/lsm/src/store/lsm_store_entry.c b/lsm/src/store/lsm_store_entry.c index 7d33d30..93a570b 100644 --- a/lsm/src/store/lsm_store_entry.c +++ b/lsm/src/store/lsm_store_entry.c @@ -61,6 +61,7 @@ lsm_error lsm_entry_handle_init(lsm_entry_handle **out) { lsm_error lsm_entry_commit(lsm_entry_handle *handle) { uint8_t state_new = handle->states & lsm_entry_handle_state_new; uint8_t state_removed = handle->states & lsm_entry_handle_state_removed; + uint8_t state_updated = handle->states & lsm_entry_handle_state_updated; // Clean new entry if (state_new && !state_removed) { @@ -73,6 +74,8 @@ lsm_error lsm_entry_commit(lsm_entry_handle *handle) { lsm_entry_free(handle->wrapper->entry); handle->wrapper->entry = NULL; + } else if (state_updated && !(state_new || state_removed)) { + LSM_RES(lsm_entry_disk_update(handle)); } // Reset states after committing current changes @@ -99,6 +102,8 @@ void lsm_entry_close(lsm_entry_handle *handle) { handle->wrapper->entry = NULL; } + // TODO rollback uncomitted updates + pthread_rwlock_unlock(&handle->wrapper->lock); free(handle); } From 4d9dfff27e424616964b1e8f5f304739e1420418 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 28 Aug 2024 20:29:47 +0200 Subject: [PATCH 2/3] feat(lsm): implement updating entries --- lsm/src/store/lsm_store_disk_write.c | 34 +++++++++++++++++++++------- lsm/src/store/lsm_store_entry.c | 5 ++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/lsm/src/store/lsm_store_disk_write.c b/lsm/src/store/lsm_store_disk_write.c index 51e9be8..506c793 100644 --- a/lsm/src/store/lsm_store_disk_write.c +++ b/lsm/src/store/lsm_store_disk_write.c @@ -1,3 +1,5 @@ +#include + #include "lsm/store_internal.h" static lsm_error lsm_fwrite(uint64_t *sum, FILE *f, uint64_t size, @@ -125,16 +127,10 @@ lsm_error lsm_entry_disk_insert(lsm_entry_handle *handle) { return res; } -// Marking an entry as removed in the idx file is simply setting the length of -// its entry to zero -lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { - lsm_store *store = handle->store; - const lsm_entry *entry = handle->wrapper->entry; - +static lsm_error lsm_idx_zero_block(lsm_store *store, uint64_t pos) { pthread_mutex_lock(&store->idx.lock); - lsm_error res = - lsm_fseek(store->idx.f, entry->idx_file_offset + sizeof(uint64_t)); + lsm_error res = lsm_fseek(store->idx.f, pos); if (res != lsm_error_ok) { pthread_mutex_unlock(&store->idx.lock); @@ -153,7 +149,29 @@ lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { fflush(store->idx.f); + return lsm_error_ok; +} + +// Marking an entry as removed in the idx file is simply setting the length of +// its entry to zero +lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { + const lsm_entry *entry = handle->wrapper->entry; + + LSM_RES(lsm_idx_zero_block(handle->store, + entry->idx_file_offset + sizeof(uint64_t))); LSM_RES(lsm_entry_data_remove(handle)); return lsm_error_ok; } + +lsm_error lsm_entry_disk_update(lsm_entry_handle *handle) { + // An update is implemented by reinserting the entry at the end of the db file + uint64_t old_idx_index = handle->wrapper->entry->idx_file_offset; + + // TODO is there any way we can make this atomic? If the zero write to the + // index file fails, there are two entries in the db file for the same key. + LSM_RES(lsm_entry_disk_insert(handle)); + LSM_RES(lsm_idx_zero_block(handle->store, old_idx_index + sizeof(uint64_t))); + + return lsm_error_ok; +} diff --git a/lsm/src/store/lsm_store_entry.c b/lsm/src/store/lsm_store_entry.c index 7d33d30..93a570b 100644 --- a/lsm/src/store/lsm_store_entry.c +++ b/lsm/src/store/lsm_store_entry.c @@ -61,6 +61,7 @@ lsm_error lsm_entry_handle_init(lsm_entry_handle **out) { lsm_error lsm_entry_commit(lsm_entry_handle *handle) { uint8_t state_new = handle->states & lsm_entry_handle_state_new; uint8_t state_removed = handle->states & lsm_entry_handle_state_removed; + uint8_t state_updated = handle->states & lsm_entry_handle_state_updated; // Clean new entry if (state_new && !state_removed) { @@ -73,6 +74,8 @@ lsm_error lsm_entry_commit(lsm_entry_handle *handle) { lsm_entry_free(handle->wrapper->entry); handle->wrapper->entry = NULL; + } else if (state_updated && !(state_new || state_removed)) { + LSM_RES(lsm_entry_disk_update(handle)); } // Reset states after committing current changes @@ -99,6 +102,8 @@ void lsm_entry_close(lsm_entry_handle *handle) { handle->wrapper->entry = NULL; } + // TODO rollback uncomitted updates + pthread_rwlock_unlock(&handle->wrapper->lock); free(handle); } From 39524963781d589a2a7da3bed70280b51811f8d8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 28 Aug 2024 22:39:51 +0200 Subject: [PATCH 3/3] feat(lander): implement placeholder keys --- CHANGELOG.md | 5 ++++ include/lander.h | 11 ++++++++ lsm/include/lsm/store.h | 3 ++- lsm/src/store/lsm_store_entry.c | 6 +++-- src/lander/lander_post.c | 31 +++++++++++++++++++++- src/lander/lander_steps.c | 46 +++++++++++++++++++++++++++++++++ src/main.c | 35 ++++++++++++++++++------- 7 files changed, 123 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a31cd42..f179b59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/lander/src/branch/dev) +## Added + +* Ability to generate keys that allow one-time unauthenticated uploads (a.k.a. + generating upload links) + ## Fixed * Failed uploads now no longer leave behind a partial entry file diff --git a/include/lander.h b/include/lander.h index 9ebbdf2..09c53fc 100644 --- a/include/lander.h +++ b/include/lander.h @@ -27,6 +27,7 @@ typedef enum lander_entry_type : uint8_t { lander_entry_type_redirect = 0, lander_entry_type_paste = 1, lander_entry_type_file = 2, + lander_entry_type_placeholder = 3, } lander_entry_type; void *lander_gctx_init(); @@ -61,6 +62,16 @@ lnm_http_step_err lander_post_file(lnm_http_conn *conn); lnm_http_step_err lander_post_file_secure(lnm_http_conn *conn); +lnm_http_step_err lander_post_placeholder(lnm_http_conn *conn); + +lnm_http_step_err lander_post_placeholder_secure(lnm_http_conn *conn); + +/** + * Step that authenticates requests. If the key of the request is for a + * placeholder entry, authentication is granted without an api key. + */ +lnm_http_step_err lander_auth_or_placeholder(lnm_http_conn *conn); + /** * Store the requested header as an attribute, if it's present. */ diff --git a/lsm/include/lsm/store.h b/lsm/include/lsm/store.h index e2d7d86..6018a5e 100644 --- a/lsm/include/lsm/store.h +++ b/lsm/include/lsm/store.h @@ -56,7 +56,8 @@ lsm_error lsm_entry_attr_get_uint8_t(uint8_t *out, lsm_entry_handle *handle, uint8_t type); /** - * Add a new attribute to the entry. + * Add a new attribute to the entry. This overwrites an existing version of this + * attribute. * * @param entry entry to modify * @param type type of attribute to add diff --git a/lsm/src/store/lsm_store_entry.c b/lsm/src/store/lsm_store_entry.c index 93a570b..2d73beb 100644 --- a/lsm/src/store/lsm_store_entry.c +++ b/lsm/src/store/lsm_store_entry.c @@ -211,8 +211,10 @@ lsm_error lsm_entry_attr_remove(lsm_str **out, lsm_entry_handle *handle, lsm_error lsm_entry_attr_insert(lsm_entry_handle *handle, uint8_t type, lsm_str *data) { - if (lsm_entry_attr_present(handle, type)) { - return lsm_error_already_present; + // Remove a previous version of the attribute + lsm_str *out; + if (lsm_entry_attr_remove(&out, handle, type) == lsm_error_ok) { + lsm_str_free(out); } lsm_entry *entry = handle->wrapper->entry; diff --git a/src/lander/lander_post.c b/src/lander/lander_post.c index 7befe03..f074f5a 100644 --- a/src/lander/lander_post.c +++ b/src/lander/lander_post.c @@ -17,7 +17,8 @@ static void randomize_key(char *key, int len) { } /** - * Insert a new entry into the store. + * Insert a new entry into the store. If an entry is already open, this function + * is a no-op. * * @return true on success, false otherwise */ @@ -26,6 +27,12 @@ bool lander_insert_entry(lnm_http_loop_ctx *ctx, bool secure) { lander_gctx *c_gctx = gctx->c; lander_ctx *c_ctx = ctx->c; + // With placeholders, the entry will already be open so an entry should no + // longer be created + if (c_ctx->entry != NULL) { + return true; + } + const char *key_s; size_t key_len = lnm_http_req_route_segment(&key_s, &ctx->req, "key"); @@ -147,3 +154,25 @@ lnm_http_step_err lander_post_file(lnm_http_conn *conn) { lnm_http_step_err lander_post_file_secure(lnm_http_conn *conn) { return __lander_post_file(conn, true); } + +lnm_http_step_err __lander_post_placeholder(lnm_http_conn *conn, bool secure) { + lnm_http_loop_ctx *ctx = conn->ctx; + lander_ctx *c_ctx = ctx->c; + + if (!lander_insert_entry(ctx, secure)) { + return lnm_http_step_err_res; + } + + lsm_entry_attr_insert_uint8_t(c_ctx->entry, lander_attr_type_entry_type, + lander_entry_type_placeholder); + + return lnm_http_step_err_done; +} + +lnm_http_step_err lander_post_placeholder(lnm_http_conn *conn) { + return __lander_post_placeholder(conn, false); +} + +lnm_http_step_err lander_post_placeholder_secure(lnm_http_conn *conn) { + return __lander_post_placeholder(conn, true); +} diff --git a/src/lander/lander_steps.c b/src/lander/lander_steps.c index 8e2d702..502f733 100644 --- a/src/lander/lander_steps.c +++ b/src/lander/lander_steps.c @@ -5,6 +5,7 @@ #include "lnm/loop.h" #include "lander.h" +#include "lsm/store.h" lnm_http_step_err lander_stream_body_to_entry(lnm_http_conn *conn) { lnm_http_loop_ctx *ctx = conn->ctx; @@ -35,3 +36,48 @@ lnm_http_step_err lander_commit_entry(lnm_http_conn *conn) { return lnm_http_step_err_done; } + +lnm_http_step_err lander_auth_or_placeholder(lnm_http_conn *conn) { + lnm_http_loop_ctx *ctx = conn->ctx; + lnm_http_loop_gctx *gctx = ctx->g; + lander_gctx *c_gctx = gctx->c; + lander_ctx *c_ctx = ctx->c; + + const char *key_s; + size_t key_len = lnm_http_req_route_segment(&key_s, &ctx->req, "key"); + + // Only predefined keys can be placeholders + if (key_len == 0) { + return lnm_http_loop_step_auth(conn); + } + + lsm_str *key; + lsm_str_init_copy_n(&key, key_s, key_len); + + lsm_error res = lsm_store_open_write(&c_ctx->entry, c_gctx->store, key); + + lsm_str_free(key); + + switch (res) { + case lsm_error_ok: { + lander_entry_type t; + + // If the entry is a placeholder, the request is always authenticated + if ((lsm_entry_attr_get_uint8_t( + &t, c_ctx->entry, lander_attr_type_entry_type) == lsm_error_ok) && + (t == lander_entry_type_placeholder)) { + return lnm_http_step_err_done; + } else { + return lnm_http_loop_step_auth(conn); + } + } break; + case lsm_error_not_found: + return lnm_http_loop_step_auth(conn); + break; + default: + ctx->res.status = lnm_http_status_internal_server_error; + return lnm_http_step_err_res; + } + + return lnm_http_step_err_done; +} diff --git a/src/main.c b/src/main.c index 12601b3..a6727a6 100644 --- a/src/main.c +++ b/src/main.c @@ -34,62 +34,77 @@ lnm_http_loop *loop_init(lander_gctx *gctx, const char *api_key) { lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/s/"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_auth_or_placeholder, false); lnm_http_route_step_append(route, lander_post_redirect, false); lnm_http_route_step_append(route, lnm_http_loop_step_body_to_buf, false); lnm_http_route_step_append(route, lander_post_redirect_body_to_attr, false); lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/sl/"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_auth_or_placeholder, false); lnm_http_route_step_append(route, lander_post_redirect_secure, false); lnm_http_route_step_append(route, lnm_http_loop_step_body_to_buf, false); lnm_http_route_step_append(route, lander_post_redirect_body_to_attr, false); lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/s/:key"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_auth_or_placeholder, false); lnm_http_route_step_append(route, lander_post_redirect, false); lnm_http_route_step_append(route, lnm_http_loop_step_body_to_buf, false); lnm_http_route_step_append(route, lander_post_redirect_body_to_attr, false); lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/p/"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_auth_or_placeholder, false); lnm_http_route_step_append(route, lander_post_paste, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/pl/"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_auth_or_placeholder, false); lnm_http_route_step_append(route, lander_post_paste_secure, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/p/:key"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_auth_or_placeholder, false); lnm_http_route_step_append(route, lander_post_paste, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/f/"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_auth_or_placeholder, false); lnm_http_route_step_append(route, lander_post_file, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/fl/"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_auth_or_placeholder, false); lnm_http_route_step_append(route, lander_post_file_secure, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/f/:key"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_auth_or_placeholder, false); lnm_http_route_step_append(route, lander_post_file, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); lnm_http_route_step_append(route, lander_commit_entry, true); + lnm_http_router_add(&route, router, lnm_http_method_post, "/h/"); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_post_placeholder, false); + lnm_http_route_step_append(route, lander_commit_entry, true); + + lnm_http_router_add(&route, router, lnm_http_method_post, "/h/:key"); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_post_placeholder, false); + lnm_http_route_step_append(route, lander_commit_entry, true); + + lnm_http_router_add(&route, router, lnm_http_method_post, "/hs/"); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); + lnm_http_route_step_append(route, lander_post_placeholder_secure, false); + lnm_http_route_step_append(route, lander_commit_entry, true); + lnm_http_loop_router_set(hl, router); return hl; @@ -140,5 +155,5 @@ int main() { lnm_linfo("main", "Store loaded containing %lu entries", lsm_store_size(c_gctx->store)); lnm_http_loop *hl = loop_init(c_gctx, api_key); - lnm_http_loop_run(hl, port, 1, 0); + lnm_http_loop_run(hl, port, 1, 1); }