diff --git a/CHANGELOG.md b/CHANGELOG.md index 3795d4a..993ae07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,12 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Rewrite of trie codebase * Introduced a custom data store using an in-memory trie as index and a custom binary on-disk format - * Support for lookup, insert & a basic remove + * Support for lookup & insert * Lander * Replaced old trie implementation with LSM store - * Add support for hosting arbitrary files - * Content type of file is set if provided when uploading file - * Support removing entries ## [0.1.0](https://git.rustybever.be/Chewing_Bever/lander/src/tag/0.1.0) diff --git a/include/http_loop.h b/include/http_loop.h index 131bd6e..e05ee22 100644 --- a/include/http_loop.h +++ b/include/http_loop.h @@ -7,6 +7,7 @@ #include "http/req.h" #include "http/res.h" #include "http/types.h" +#include "trie.h" // Max amount of steps a route can use #define HTTP_LOOP_MAX_STEPS 17 diff --git a/include/lander.h b/include/lander.h index de89437..1bb38d3 100644 --- a/include/lander.h +++ b/include/lander.h @@ -5,15 +5,17 @@ #include "lsm/store.h" extern http_route lander_routes[6]; -extern const char lander_key_charset[]; typedef struct lander_gctx { const char *data_dir; + Trie *trie; lsm_store *store; + } lander_gctx; typedef struct lander_ctx { lsm_entry_handle *entry; + uint64_t remaining_data; } lander_ctx; typedef enum lander_attr_type : uint8_t { @@ -22,14 +24,6 @@ typedef enum lander_attr_type : uint8_t { lander_attr_type_url = 2, } lander_attr_type; -typedef struct { - char *header; - lander_attr_type attr_type; - http_header header_type; -} header_to_attr; - -extern header_to_attr header_to_attrs[]; - typedef enum lander_entry_type : uint8_t { lander_entry_type_redirect = 0, lander_entry_type_paste = 1, @@ -52,19 +46,21 @@ bool lander_post_redirect(event_loop_conn *conn); bool lander_post_paste(event_loop_conn *conn); -bool lander_post_paste(event_loop_conn *conn); +bool lander_post_paste_lsm(event_loop_conn *conn); -bool lander_post_redirect(event_loop_conn *conn); +bool lander_post_redirect_lsm(event_loop_conn *conn); bool lander_stream_body_to_entry(event_loop_conn *conn); bool lander_stream_body_to_client(event_loop_conn *conn); +bool lander_get_entry_lsm(event_loop_conn *conn); + bool lander_post_redirect_body_to_attr(event_loop_conn *conn); bool lander_remove_entry(event_loop_conn *conn); -bool lander_post_file(event_loop_conn *conn); +bool lander_post_file_lsm(event_loop_conn *conn); /** * Parse any custom headers and add them as attributes to the context's LSM @@ -72,6 +68,4 @@ bool lander_post_file(event_loop_conn *conn); */ bool lander_headers_to_attrs(event_loop_conn *conn); -bool lander_attrs_to_headers(event_loop_conn *conn); - #endif diff --git a/landerctl b/landerctl index 74a7ed3..f7bdded 100755 --- a/landerctl +++ b/landerctl @@ -44,13 +44,6 @@ elif [ "$1" = f ]; then -w "${URL}%header{location}" \ -XPOST \ -H "X-Api-Key: $API_KEY" \ - -H "X-Lander-Content-Type: $(file --mime-type --brief $2)" \ --data-binary @"$2" \ "$URL/f/$3" - -elif [ "$1" = d ]; then - curl \ - -XDELETE \ - -H "X-Api-Key: $API_KEY" \ - "$URL/$2" fi diff --git a/src/http_loop/http_loop_steps.c b/src/http_loop/http_loop_steps.c index bcfeae7..dfa8c96 100644 --- a/src/http_loop/http_loop_steps.c +++ b/src/http_loop/http_loop_steps.c @@ -1,5 +1,4 @@ #include -#include #include "http_loop.h" #include "lander.h" diff --git a/src/lander/lander.c b/src/lander/lander.c index 06395cb..79518b0 100644 --- a/src/lander/lander.c +++ b/src/lander/lander.c @@ -1,13 +1,9 @@ #include -#include "http/types.h" #include "http_loop.h" #include "lander.h" #include "lsm/store.h" -const char lander_key_charset[] = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - http_route lander_routes[] = { {.type = http_route_literal, .method = http_get, @@ -19,7 +15,7 @@ http_route lander_routes[] = { .type = http_route_regex, .method = http_get, .path = "^/([^/]+)$", - .steps = {lander_get_entry, lander_attrs_to_headers, NULL}, + .steps = {lander_get_entry_lsm, NULL}, .steps_res = {http_loop_step_write_header, lander_stream_body_to_client, NULL}, }, @@ -35,7 +31,7 @@ http_route lander_routes[] = { .type = http_route_regex, .method = http_post, .path = "^/s(l?)/([^/]*)$", - .steps = {http_loop_step_auth, lander_post_redirect, + .steps = {http_loop_step_auth, lander_post_redirect_lsm, http_loop_step_body_to_buf, lander_post_redirect_body_to_attr, NULL}, .steps_res = {http_loop_step_write_header, http_loop_step_write_body, @@ -45,23 +41,24 @@ http_route lander_routes[] = { .method = http_post, .path = "^/p(l?)/([^/]*)$", .steps = {http_loop_step_auth, http_loop_step_parse_content_length, - lander_post_paste, lander_stream_body_to_entry, NULL}, + lander_post_paste_lsm, lander_stream_body_to_entry, NULL}, .steps_res = {http_loop_step_write_header, http_loop_step_write_body, NULL}}, {.type = http_route_regex, .method = http_post, .path = "^/f(l?)/([^/]*)$", .steps = {http_loop_step_auth, http_loop_step_parse_content_length, - lander_post_file, lander_headers_to_attrs, - lander_stream_body_to_entry, NULL}, + lander_post_file_lsm, lander_headers_to_attrs, lander_stream_body_to_entry, NULL}, .steps_res = {http_loop_step_write_header, http_loop_step_write_body, NULL}}, }; -header_to_attr header_to_attrs[] = { - {"X-Lander-Content-Type", lander_attr_type_content_type, - http_header_content_type}, - {NULL, 0, 0}, +struct { + char *header; + lander_attr_type attr_type; +} header_to_attr_type[] = { + { "X-Lander-Content-Type", lander_attr_type_content_type }, + { NULL, 0 }, }; void *lander_gctx_init() { return calloc(1, sizeof(lander_gctx)); } @@ -74,6 +71,34 @@ void lander_ctx_reset(lander_ctx *ctx) { ctx->entry = NULL; } + + ctx->remaining_data = 0; } void lander_ctx_free(lander_ctx *ctx) { free(ctx); } + +bool lander_headers_to_attrs(event_loop_conn *conn) { + http_loop_ctx *ctx = conn->ctx; + lander_ctx *c_ctx = ctx->c; + + for (size_t i = 0; i < ctx->req.num_headers; i++) { + struct phr_header *header = &ctx->req.headers[i]; + + int j = 0; + + while (header_to_attr_type[j].header != NULL) { + if (strncmp(header->name, header_to_attr_type[j].header, header->name_len) == 0) { + lsm_str *value; + lsm_str_init_copy_n(&value, (char *)header->value, header->value_len); + + lsm_entry_attr_insert(c_ctx->entry, header_to_attr_type[j].attr_type, value); + + break; + } + + j++; + } + } + + return true; +} diff --git a/src/lander/lander_get.c b/src/lander/lander_get.c index bd62a16..03076c8 100644 --- a/src/lander/lander_get.c +++ b/src/lander/lander_get.c @@ -75,7 +75,7 @@ void lander_get_file(event_loop_conn *conn) { ctx->res.body.expected_len = lsm_entry_data_len(c_ctx->entry); } -bool lander_get_entry(event_loop_conn *conn) { +bool lander_get_entry_lsm(event_loop_conn *conn) { http_loop_ctx *ctx = conn->ctx; lander_ctx *c_ctx = ctx->c; http_loop_gctx *gctx = ctx->g; @@ -92,11 +92,9 @@ bool lander_get_entry(event_loop_conn *conn) { break; case lsm_error_not_found: ctx->res.status = http_not_found; - conn->state = event_loop_conn_state_res; return true; default: ctx->res.status = http_internal_server_error; - conn->state = event_loop_conn_state_res; return true; } diff --git a/src/lander/lander_post.c b/src/lander/lander_post.c index cdde6d0..13679cc 100644 --- a/src/lander/lander_post.c +++ b/src/lander/lander_post.c @@ -5,15 +5,71 @@ #include "lsm/store.h" static void randomize_key(char *key, int len) { - size_t charset_len = strlen(lander_key_charset); - for (int i = 0; i < len; i++) { - key[i] = lander_key_charset[rand() % charset_len]; + key[i] = charset[rand() % charset_len]; } key[len] = '\0'; } +// TODO entry leaks if key is already present +static bool add_entry(char **key_ptr, int *key_len_ptr, http_loop_ctx *ctx, + Entry *entry, bool random) { + lander_gctx *c_gctx = ctx->g->c; + + // The first match group matches the "long" path + bool secure = + (ctx->req.regex_groups[1].rm_eo - ctx->req.regex_groups[1].rm_so) == 1; + + char *key; + int key_len = 0; + TrieExitCode res; + + if (random) { + res = trie_add_random(c_gctx->trie, &key, entry, secure); + + if (res == Ok) { + key_len = strlen(key); + } + } else { + key = (char *)&ctx->req.path[ctx->req.regex_groups[2].rm_so]; + key_len = ctx->req.regex_groups[2].rm_eo - ctx->req.regex_groups[2].rm_so; + + res = trie_add_len(c_gctx->trie, key, key_len, entry); + } + + switch (res) { + case Ok: + break; + case AlreadyPresent: + ctx->res.status = http_conflict; + return false; + default: + ctx->res.status = http_internal_server_error; + return false; + } + + // Add a slash to the key and add it as the location header + char *buf = malloc(key_len + 2); + + memcpy(&buf[1], key, key_len); + buf[0] = '/'; + buf[key_len + 1] = '\0'; + + http_res_add_header(&ctx->res, http_header_location, buf, true); + ctx->res.status = http_created; + + if (key_ptr != NULL) { + *key_ptr = key; + } + + if (key_len_ptr != NULL) { + *key_len_ptr = key_len; + } + + return true; +} + /** * Insert a new entry into the store. * @@ -67,7 +123,7 @@ bool lander_insert_entry(http_loop_ctx *ctx) { return true; } -bool lander_post_redirect(event_loop_conn *conn) { +bool lander_post_redirect_lsm(event_loop_conn *conn) { http_loop_ctx *ctx = conn->ctx; lander_ctx *c_ctx = ctx->c; @@ -93,7 +149,7 @@ bool lander_post_redirect_body_to_attr(event_loop_conn *conn) { return true; } -bool lander_post_paste(event_loop_conn *conn) { +bool lander_post_paste_lsm(event_loop_conn *conn) { http_loop_ctx *ctx = conn->ctx; lander_ctx *c_ctx = ctx->c; @@ -108,7 +164,7 @@ bool lander_post_paste(event_loop_conn *conn) { return true; } -bool lander_post_file(event_loop_conn *conn) { +bool lander_post_file_lsm(event_loop_conn *conn) { http_loop_ctx *ctx = conn->ctx; lander_ctx *c_ctx = ctx->c; @@ -122,3 +178,81 @@ bool lander_post_file(event_loop_conn *conn) { return true; } + +bool lander_stream_body_to_entry(event_loop_conn *conn) { + http_loop_ctx *ctx = conn->ctx; + lander_ctx *c_ctx = ctx->c; + + uint64_t to_append = + MIN(conn->rbuf_size - conn->rbuf_read, + ctx->req.body.expected_len - lsm_entry_data_len(c_ctx->entry)); + + lsm_str *data; + lsm_str_init_copy_n(&data, (char *)&conn->rbuf[conn->rbuf_read], to_append); + lsm_entry_data_append(c_ctx->entry, data); + + conn->rbuf_read += to_append; + + lsm_str_free(data); + + return lsm_entry_data_len(c_ctx->entry) == ctx->req.body.expected_len; +} + +bool lander_post_redirect(event_loop_conn *conn) { + http_loop_ctx *ctx = conn->ctx; + bool random = + ctx->req.regex_groups[2].rm_eo == ctx->req.regex_groups[2].rm_so; + + // Allocate a new buffer to pass to the trie + char *url = malloc(ctx->req.body.len + 1); + memcpy(url, ctx->req.body.buf, ctx->req.body.len); + url[ctx->req.body.len] = '\0'; + + Entry *new_entry = entry_new(Redirect, url); + + // The entry duplicates the string + free(url); + + // We don't check the result here, because we would perform the same action + // either way + char *key; + add_entry(&key, NULL, ctx, new_entry, random); + + if (random) { + free(key); + } + + conn->state = event_loop_conn_state_res; + + return true; +} + +bool lander_post_paste(event_loop_conn *conn) { + http_loop_ctx *ctx = conn->ctx; + lander_gctx *c_gctx = ctx->g->c; + + bool random = + ctx->req.regex_groups[2].rm_eo == ctx->req.regex_groups[2].rm_so; + + char *key; + int key_len; + Entry *new_entry = entry_new(Paste, ""); + + if (!add_entry(&key, &key_len, ctx, new_entry, random)) { + conn->state = event_loop_conn_state_res; + + return true; + } + + char *fname = malloc(strlen(c_gctx->data_dir) + 8 + key_len + 1); + sprintf(fname, "%s/pastes/%.*s", c_gctx->data_dir, key_len, key); + + ctx->req.body.fname = fname; + ctx->req.body.fname_owned = true; + + if (random) { + free(key); + } + + return true; +} diff --git a/src/lander/lander_steps.c b/src/lander/lander_steps.c deleted file mode 100644 index c84f9d2..0000000 --- a/src/lander/lander_steps.c +++ /dev/null @@ -1,73 +0,0 @@ -#include - -#include "lander.h" - -bool lander_stream_body_to_entry(event_loop_conn *conn) { - http_loop_ctx *ctx = conn->ctx; - lander_ctx *c_ctx = ctx->c; - - uint64_t to_append = - MIN(conn->rbuf_size - conn->rbuf_read, - ctx->req.body.expected_len - lsm_entry_data_len(c_ctx->entry)); - - lsm_str *data; - lsm_str_init_copy_n(&data, (char *)&conn->rbuf[conn->rbuf_read], to_append); - lsm_entry_data_append(c_ctx->entry, data); - - conn->rbuf_read += to_append; - - lsm_str_free(data); - - return lsm_entry_data_len(c_ctx->entry) == ctx->req.body.expected_len; -} - -bool lander_headers_to_attrs(event_loop_conn *conn) { - http_loop_ctx *ctx = conn->ctx; - lander_ctx *c_ctx = ctx->c; - - for (size_t i = 0; i < ctx->req.num_headers; i++) { - struct phr_header *header = &ctx->req.headers[i]; - - int j = 0; - - while (header_to_attrs[j].header != NULL) { - if (strncmp(header->name, header_to_attrs[j].header, header->name_len) == - 0) { - lsm_str *value; - lsm_str_init_copy_n(&value, (char *)header->value, header->value_len); - - lsm_entry_attr_insert(c_ctx->entry, header_to_attrs[j].attr_type, - value); - - break; - } - - j++; - } - } - - return true; -} - -bool lander_attrs_to_headers(event_loop_conn *conn) { - http_loop_ctx *ctx = conn->ctx; - lander_ctx *c_ctx = ctx->c; - - int j = 0; - lsm_str *value; - - while (header_to_attrs[j].header != NULL) { - if (lsm_entry_attr_get(&value, c_ctx->entry, - header_to_attrs[j].attr_type) == lsm_error_ok) { - char *buf = malloc(lsm_str_len(value) + 1); - memcpy(buf, lsm_str_ptr(value), lsm_str_len(value)); - buf[lsm_str_len(value)] = '\0'; - - http_res_add_header(&ctx->res, header_to_attrs[j].header_type, buf, true); - } - - j++; - } - - return true; -} diff --git a/src/main.c b/src/main.c index fa9c64f..bf0dd9a 100644 --- a/src/main.c +++ b/src/main.c @@ -34,6 +34,20 @@ int main() { critical(1, "Invalid TCP port %s", port_str); } + /* char file_path[strlen(data_dir) + 12 + 1]; */ + /* sprintf(file_path, "%s/lander.data", data_dir); */ + + /* info("Initializing trie from file '%s'", file_path); */ + + /* Trie *trie; */ + /* TrieExitCode res = trie_init(&trie, file_path); */ + + /* if (res != Ok) { */ + /* critical(1, "An error occured while populating the trie."); */ + /* } */ + + /* info("Trie initialized and populated with %i entries", trie_size(trie)); */ + lander_gctx *c_gctx = lander_gctx_init(); c_gctx->data_dir = data_dir_s; @@ -46,7 +60,7 @@ int main() { critical(2, "Failed to load existing store."); } - info("Store loaded containing %lu entries", lsm_store_size(c_gctx->store)); + info("Store loaded containing %lu entries.", lsm_store_size(c_gctx->store)); http_loop *hl = http_loop_init( lander_routes, sizeof(lander_routes) / sizeof(lander_routes[0]), c_gctx, diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..a36505f --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,228 @@ +#include +#include + +#include "crow.h" + +extern "C" { +#include "trie.h" +} + +static const std::string file_path = "lander.data"; +static const std::string index_page = R"( + + + +

r8r.be

+

This is the URL shortener and pastebin accompanying my site, The Rusty Bever.

+ + +)"; + +#define ENV(var, env_var) \ + const char *_##var = getenv(env_var); \ + if (_##var == NULL) { \ + printf("Missing environment variable %s.\n", env_var); \ + return 1; \ + } \ + const std::string var = std::string(_##var); + +#define AUTH() \ + std::string provided_api_key = req.get_header_value("X-Api-Key"); \ + if (api_key.compare(provided_api_key) != 0) { \ + return crow::response(crow::status::UNAUTHORIZED); \ + } + +crow::response add_redirect(std::string base_url, Trie *trie, const char *url, + bool secure) { + Entry *new_entry = entry_new(Redirect, url); + + // The key already gets copied into the trie, so this pointer is safe to use + // ever after unlocking the trie + trie_wlock(trie); + char *key; + TrieExitCode res = trie_add_random(trie, &key, new_entry, secure); + trie_unlock(trie); + + if (res != Ok) { + return crow::response(crow::status::INTERNAL_SERVER_ERROR); + } + + std::string out = base_url + key; + free(key); + + return crow::response(out); +} + +bool store_paste(const char *key, const char *body) { + // Write paste contents to file + std::fstream file; + file.open(std::string("pastes/") + key, std::ios_base::out); + + if (!file.is_open()) { + return false; + } + + file << body; + file.close(); + + return true; +} + +crow::response add_paste(std::string base_url, Trie *trie, const char *body, + bool secure) { + Entry *new_entry = entry_new(Paste, ""); + + trie_wlock(trie); + char *key; + TrieExitCode res = trie_add_random(trie, &key, new_entry, secure); + trie_unlock(trie); + + if (res != Ok) { + return crow::response(crow::status::INTERNAL_SERVER_ERROR); + } + + if (!store_paste(key, body)) { + return crow::response(crow::status::INTERNAL_SERVER_ERROR); + } + + std::string out = base_url + key; + free(key); + + return crow::response(out); +} + +int main() { + // Initialize random seed for generating URLs + srand(time(NULL)); + + ENV(api_key, "LANDER_API_KEY"); + ENV(base_url, "LANDER_BASE_URL"); + + std::cout << "Initializing trie from file '" << file_path << "'..." + << std::endl; + + // Initialize trie and populate from data file + Trie *trie; + int res = trie_init(&trie, file_path.c_str()); + + if (res != 0) { + std::cout << "An error occured while initializing the trie." << std::endl; + + exit(1); + } + + std::cout << "Added " << trie_size(trie) << " entries to trie." << std::endl; + + // Create pastes directory if not present + // TODO don't just ignore errors here + mkdir("pastes", 0700); + + crow::SimpleApp app; + app.loglevel(crow::LogLevel::Info); + + CROW_ROUTE(app, "/").methods(crow::HTTPMethod::Get)( + []() { return crow::response("html", index_page); }); + + // Serve an entry + CROW_ROUTE(app, "/") + .methods(crow::HTTPMethod::Get)( + [trie](crow::response &res, std::string key) { + trie_rlock(trie); + Entry *entry; + TrieExitCode status = trie_search(trie, &entry, key.c_str()); + + if (status == Ok) { + if (entry->type == Redirect) { + res.redirect(entry->string); + } else if (entry->type == Paste) { + res.set_static_file_info("pastes/" + key); + } + } else { + res.code = 404; + } + + res.end(); + trie_unlock(trie); + }); + + // Add a new Redirect with a short randomly generated key + CROW_ROUTE(app, "/s/") + .methods(crow::HTTPMethod::Post)( + [api_key, base_url, trie](const crow::request req) { + AUTH(); + + return add_redirect(base_url, trie, req.body.c_str(), false); + }); + + // Add a new Redirect with a long randomly generated key + CROW_ROUTE(app, "/sl/") + .methods(crow::HTTPMethod::Post)( + [api_key, base_url, trie](const crow::request req) { + AUTH(); + + return add_redirect(base_url, trie, req.body.c_str(), true); + }); + + // Add a new Redirect with a given key + CROW_ROUTE(app, "/s/") + .methods(crow::HTTPMethod::Post)( + [api_key, base_url, trie](const crow::request &req, std::string key) { + AUTH(); + + Entry *new_entry = entry_new(Redirect, req.body.c_str()); + + trie_wlock(trie); + TrieExitCode status = trie_add(trie, key.c_str(), new_entry); + trie_unlock(trie); + + switch (status) { + case Ok: + return crow::response(base_url + key); + case AlreadyPresent: + return crow::response(crow::status::CONFLICT); + default: + return crow::response(crow::status::INTERNAL_SERVER_ERROR); + } + }); + + // Add a new Paste with a short randomly generated key + CROW_ROUTE(app, "/p/") + .methods(crow::HTTPMethod::Post)( + [api_key, base_url, trie](const crow::request &req) { + AUTH(); + + return add_paste(base_url, trie, req.body.c_str(), false); + }); + + // Add a new Paste with a long randomly generated key + CROW_ROUTE(app, "/pl/") + .methods(crow::HTTPMethod::Post)( + [api_key, base_url, trie](const crow::request &req) { + AUTH(); + + return add_paste(base_url, trie, req.body.c_str(), true); + }); + + // Add a paste with a given key + CROW_ROUTE(app, "/p/") + .methods(crow::HTTPMethod::Post)( + [api_key, base_url, trie](const crow::request &req, std::string key) { + AUTH(); + + Entry *new_entry = entry_new(Paste, ""); + trie_wlock(trie); + TrieExitCode status = trie_add(trie, key.c_str(), new_entry); + trie_unlock(trie); + + if (status != Ok) { + return crow::response(crow::status::CONFLICT); + } + + if (!store_paste(key.c_str(), req.body.c_str())) { + return crow::response(crow::status::INTERNAL_SERVER_ERROR); + } + + return crow::response(base_url + key); + }); + app.port(18080).multithreaded().run(); +}