diff --git a/CHANGELOG.md b/CHANGELOG.md index 551e507..aca5284 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,25 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/lander/src/branch/dev) -## Added - -* Ability to generate keys that allow one-time unauthenticated uploads (a.k.a. - generating upload links) - -## Changed - -* LSM - * Switched to a split read/write handle design - * Changes now need to be explicitely committed before being written to - persistent storage - * Changes to entry attributes are now atomically committed - -## Fixed - -* Failed uploads now no longer leave behind a partial entry file -* Size of db file is now correctly calculated when the store contains deleted - entries - ## [0.2.1](https://git.rustybever.be/Chewing_Bever/lander/src/tag/0.2.1) ### Added diff --git a/Makefile b/Makefile index b020898..9cd57fe 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,6 @@ clean: bear: clean bear -- make bear --append -- make build-test - bear --append -- make -C landerctl # Make make aware of the .d files diff --git a/include/lander.h b/include/lander.h index ea679a3..9ebbdf2 100644 --- a/include/lander.h +++ b/include/lander.h @@ -13,11 +13,7 @@ typedef struct lander_gctx { } lander_gctx; typedef struct lander_ctx { - union { - lsm_write_handle *write; - lsm_read_handle *read; - } entry; - bool write; + lsm_entry_handle *entry; } lander_ctx; typedef enum lander_attr_type : uint8_t { @@ -31,7 +27,6 @@ typedef enum lander_entry_type : uint8_t { lander_entry_type_redirect = 0, lander_entry_type_paste = 1, lander_entry_type_file = 2, - lander_entry_type_placeholder = 3, } lander_entry_type; void *lander_gctx_init(); @@ -66,16 +61,6 @@ lnm_http_step_err lander_post_file(lnm_http_conn *conn); lnm_http_step_err lander_post_file_secure(lnm_http_conn *conn); -lnm_http_step_err lander_post_placeholder(lnm_http_conn *conn); - -lnm_http_step_err lander_post_placeholder_secure(lnm_http_conn *conn); - -/** - * Step that authenticates requests. If the key of the request is for a - * placeholder entry, authentication is granted without an api key. - */ -lnm_http_step_err lander_auth_or_placeholder(lnm_http_conn *conn); - /** * Store the requested header as an attribute, if it's present. */ diff --git a/landerctl/.landerrc b/landerctl/.landerrc new file mode 100644 index 0000000..964d3f2 --- /dev/null +++ b/landerctl/.landerrc @@ -0,0 +1,2 @@ +api_key = test +server_url = http://localhost:18080 diff --git a/landerctl/include/landerctl.h b/landerctl/include/landerctl.h index 872a11e..75da26e 100644 --- a/landerctl/include/landerctl.h +++ b/landerctl/include/landerctl.h @@ -5,9 +5,6 @@ #include -/** - * Represents a parsed config file - */ typedef struct landerctl_cfg { const char *api_key; const char *server_url; @@ -34,63 +31,30 @@ typedef enum landerctl_mode { landerctl_mode_short, landerctl_mode_paste, landerctl_mode_file, - landerctl_mode_placeholder, } landerctl_mode; -/** - * Represents parsed CLI arguments - */ -typedef struct landerctl_args { +typedef enum landerctl_err { + landerctl_err_ok = 0, + landerctl_err_not_found +} landerctl_err; + +typedef struct landerctl_ctx { + landerctl_cfg cfg; + landerctl_mode mode; bool secure; bool verbose; - landerctl_cfg cfg; - struct { - char **arr; - int len; - } args; - landerctl_mode mode; -} landerctl_args; - -/** - * Convenience wrapper around a CURL object - */ -typedef struct landerctl_curl { + const char *arg; + const char *key; CURL *curl; struct curl_slist *headers; - char err_msg[CURL_ERROR_SIZE]; -} landerctl_curl; + FILE *data_file; +} landerctl_ctx; -int landerctl_parse_args(landerctl_args *out, int argc, char **argv); +const char *landerctl_err_msg(landerctl_err err); -int landerctl_cmd_short(landerctl_args *args); -int landerctl_cmd_paste(landerctl_args *args); -int landerctl_cmd_file(landerctl_args *args); -int landerctl_cmd_placeholder(landerctl_args *args); - -/** - * Initialize a CURL object - */ -int landerctl_curl_init(landerctl_curl *out); - -/** - * Set common configurations for the CURL shared across all commands - */ -int landerctl_curl_set_common(landerctl_curl *curl, landerctl_args *args, - const char *key); - -/** - * Execute the HTTP request configured in the CURL object - */ -int landerctl_curl_perform(landerctl_curl *curl); - -/** - * Inspect the response code and Location header of a successful HTTP request - */ -int landerctl_curl_inspect(landerctl_curl *curl, landerctl_args *args); - -/** - * Deallocate the CURL object - */ -void landerctl_curl_cleanup(landerctl_curl *curl); +void landerctl_set_common(landerctl_ctx *ctx); +landerctl_err landerctl_post_short(landerctl_ctx *ctx); +landerctl_err landerctl_post_paste(landerctl_ctx *ctx); +landerctl_err landerctl_post_file(landerctl_ctx *ctx); #endif diff --git a/landerctl/src/args.c b/landerctl/src/args.c deleted file mode 100644 index 6043247..0000000 --- a/landerctl/src/args.c +++ /dev/null @@ -1,96 +0,0 @@ -#include -#include -#include - -#include "landerctl.h" - -const char *cfg_file_name = ".landerrc"; -const char *usage = "%s [-SPFsv] [-c CONFIG_FILE] [ARGS]\n"; - -int landerctl_parse_config(landerctl_cfg *cfg, const char *path) { - // Parse default file location instead - if (path == NULL) { - const char *home_dir = getenv("HOME"); - - if (home_dir == NULL) { - path = cfg_file_name; - } else { - // This is a blatant memleak if a custom config file is set, but it really - // doesn't matter for a short-lived CLI tool - char *buf = malloc(strlen(home_dir) + strlen(cfg_file_name) + 2); - sprintf(buf, "%s/%s", home_dir, cfg_file_name); - path = buf; - } - } - - char *err_msg = NULL; - landerctl_cfg_err parse_res = landerctl_cfg_parse(cfg, path); - - switch (parse_res) { - case landerctl_cfg_err_ok: - break; - case landerctl_cfg_err_not_found: - err_msg = "Config file not found"; - break; - case landerctl_cfg_err_invalid: - err_msg = "Invalid config file"; - break; - case landerctl_cfg_err_incomplete: - err_msg = "Incomplete config file"; - break; - } - - if (err_msg != NULL) { - fprintf(stderr, "%s\n", err_msg); - - return 1; - } - - return 0; -} - -int landerctl_parse_args(landerctl_args *args, int argc, char **argv) { - optind = 1; - int c; - - const char *cfg_path = NULL; - - while ((c = getopt(argc, argv, "SPFHsvc:")) != -1) { - switch (c) { - case 'S': - args->mode = landerctl_mode_short; - break; - case 'P': - args->mode = landerctl_mode_paste; - break; - case 'F': - args->mode = landerctl_mode_file; - break; - case 'H': - args->mode = landerctl_mode_placeholder; - break; - case 's': - args->secure = true; - break; - case 'v': - args->verbose = true; - break; - case 'c': - cfg_path = optarg; - break; - case '?': - fprintf(stderr, usage, argv[0]); - return 2; - } - } - - int res; - if ((res = landerctl_parse_config(&args->cfg, cfg_path))) { - return res; - } - - args->args.arr = &argv[optind]; - args->args.len = argc - optind; - - return 0; -} diff --git a/landerctl/src/commands.c b/landerctl/src/commands.c deleted file mode 100644 index 9ec3ebc..0000000 --- a/landerctl/src/commands.c +++ /dev/null @@ -1,174 +0,0 @@ -#include -#include -#include - -#include -#include - -#include "landerctl.h" - -int landerctl_cmd_short(landerctl_args *args) { - // TODO argument count check - int res; - landerctl_curl curl; - - if ((res = landerctl_curl_init(&curl))) { - return res; - } - - const char *key = args->args.len == 2 ? args->args.arr[1] : NULL; - if ((res = landerctl_curl_set_common(&curl, args, key))) { - return res; - } - - const char *url = args->args.arr[0]; - - curl_easy_setopt(curl.curl, CURLOPT_POSTFIELDSIZE, strlen(url)); - curl_easy_setopt(curl.curl, CURLOPT_POSTFIELDS, url); - - res = landerctl_curl_perform(&curl); - - if (res == 0) { - landerctl_curl_inspect(&curl, args); - } - - landerctl_curl_cleanup(&curl); - - return res; -} - -int landerctl_cmd_paste(landerctl_args *args) { - // TODO argument count check - int res; - landerctl_curl curl; - - if ((res = landerctl_curl_init(&curl))) { - return res; - } - - const char *key = args->args.len == 2 ? args->args.arr[1] : NULL; - if ((res = landerctl_curl_set_common(&curl, args, key))) { - return res; - } - - const char *data_path = args->args.arr[0]; - FILE *f = fopen(data_path, "rb"); - - if (f == NULL) { - fprintf(stderr, "Failed to open data file %s\n", args->args.arr[0]); - - return 1; - } - - struct stat sb; - stat(data_path, &sb); - - curl_easy_setopt(curl.curl, CURLOPT_POST, 1L); - curl_easy_setopt(curl.curl, CURLOPT_READDATA, f); - curl_easy_setopt(curl.curl, CURLOPT_POSTFIELDSIZE, sb.st_size); - - res = landerctl_curl_perform(&curl); - - if (res == 0) { - landerctl_curl_inspect(&curl, args); - } - - landerctl_curl_cleanup(&curl); - - return res; -} - -int landerctl_cmd_file(landerctl_args *args) { - // TODO argument count check - int res; - landerctl_curl curl; - - if ((res = landerctl_curl_init(&curl))) { - return res; - } - - const char *key = args->args.len == 2 ? args->args.arr[1] : NULL; - if ((res = landerctl_curl_set_common(&curl, args, key))) { - return res; - } - - const char *data_path = args->args.arr[0]; - FILE *f = fopen(data_path, "rb"); - - if (f == NULL) { - fprintf(stderr, "Failed to open data file %s\n", args->args.arr[0]); - - return 1; - } - - struct stat sb; - stat(data_path, &sb); - - curl_easy_setopt(curl.curl, CURLOPT_POST, 1L); - curl_easy_setopt(curl.curl, CURLOPT_READDATA, f); - curl_easy_setopt(curl.curl, CURLOPT_POSTFIELDSIZE_LARGE, sb.st_size); - curl_easy_setopt(curl.curl, CURLOPT_NOPROGRESS, 0L); - - magic_t cookie = magic_open(MAGIC_MIME_TYPE); - - if (magic_load(cookie, NULL) == 0) { - const char *mime_type = magic_file(cookie, data_path); - - if (mime_type != NULL) { - char content_type_header[strlen(mime_type) + 24]; - sprintf(content_type_header, "X-Lander-Content-Type: %s", mime_type); - - curl.headers = curl_slist_append(curl.headers, content_type_header); - } else { - printf("Couldn't determine mime type; skipping Content-Type header\n"); - } - } else { - printf("Couldn't load magic file; skipping Content-Type header\n"); - } - - char s[strlen(data_path) + 1]; - strcpy(s, data_path); - const char *base_name = basename(s); - - char filename_header[strlen(base_name) + 20]; - sprintf(filename_header, "X-Lander-Filename: %s", base_name); - - curl.headers = curl_slist_append(curl.headers, filename_header); - - res = landerctl_curl_perform(&curl); - - if (res == 0) { - landerctl_curl_inspect(&curl, args); - } - - landerctl_curl_cleanup(&curl); - - return res; -} - -int landerctl_cmd_placeholder(landerctl_args *args) { - // TODO argument count check - int res; - landerctl_curl curl; - - if ((res = landerctl_curl_init(&curl))) { - return res; - } - - const char *key = args->args.arr[0]; - if ((res = landerctl_curl_set_common(&curl, args, key))) { - return res; - } - - curl_easy_setopt(curl.curl, CURLOPT_POST, 1L); - - res = landerctl_curl_perform(&curl); - - if (res == 0) { - landerctl_curl_inspect(&curl, args); - } - - landerctl_curl_cleanup(&curl); - - return res; -} diff --git a/landerctl/src/commands_common.c b/landerctl/src/commands_common.c deleted file mode 100644 index 8cf4e5e..0000000 --- a/landerctl/src/commands_common.c +++ /dev/null @@ -1,125 +0,0 @@ -#include -#include -#include - -#include -#include - -#include "landerctl.h" - -int landerctl_curl_init(landerctl_curl *out) { - curl_global_init(CURL_GLOBAL_ALL); - CURL *curl = curl_easy_init(); - - if (curl == NULL) { - fprintf(stderr, "Failed to initialize cURL client.\n"); - - return 8; - } - - out->curl = curl; - out->headers = NULL; - - return 0; -} - -int landerctl_curl_set_common(landerctl_curl *curl, landerctl_args *args, - const char *key) { - size_t url_len = strlen(args->cfg.server_url) + 4; - - if (key != NULL) { - url_len += strlen(key); - } - - char mode_char; - - switch (args->mode) { - case landerctl_mode_short: - mode_char = 's'; - break; - case landerctl_mode_paste: - mode_char = 'p'; - break; - case landerctl_mode_file: - mode_char = 'f'; - break; - case landerctl_mode_placeholder: - mode_char = 'h'; - break; - // Shouldn't be able to happen - default: - return 10; - } - - char url[url_len + 1]; - - if (key == NULL) { - sprintf(url, "%s/%c%s/", args->cfg.server_url, mode_char, - args->secure ? "l" : ""); - } else { - sprintf(url, "%s/%c%s/%s", args->cfg.server_url, mode_char, - args->secure ? "l" : "", key); - } - - curl_easy_setopt(curl->curl, CURLOPT_URL, url); - - // Add API key header - char api_key_header[strlen(args->cfg.api_key) + 12]; - sprintf(api_key_header, "X-Api-Key: %s", args->cfg.api_key); - - curl->headers = curl_slist_append(curl->headers, api_key_header); - - curl_easy_setopt(curl->curl, CURLOPT_USERAGENT, - "landerctl/" LANDER_VERSION ""); - - if (args->verbose) { - curl_easy_setopt(curl->curl, CURLOPT_VERBOSE, 1L); - } - - curl_easy_setopt(curl->curl, CURLOPT_CAINFO, args->cfg.ca_certs_bundle); - curl_easy_setopt(curl->curl, CURLOPT_ERRORBUFFER, curl->err_msg); - - return 0; -} - -int landerctl_curl_perform(landerctl_curl *curl) { - curl_easy_setopt(curl->curl, CURLOPT_HTTPHEADER, curl->headers); - - int res = curl_easy_perform(curl->curl); - - if (res != CURLE_OK) { - fprintf(stderr, "Libcurl encountered an error (code %i): %s\n", res, - curl->err_msg); - } - - return res; -} - -int landerctl_curl_inspect(landerctl_curl *curl, landerctl_args *args) { - long response_code; - curl_easy_getinfo(curl->curl, CURLINFO_RESPONSE_CODE, &response_code); - - if (response_code < 200 || response_code > 299) { - fprintf(stderr, "HTTP status code %li\n", response_code); - - return 3; - } else { - struct curl_header *location_header; - - if (curl_easy_header(curl->curl, "Location", 0, CURLH_HEADER, -1, - &location_header) == CURLHE_OK) { - printf("%s%s\n", args->cfg.server_url, location_header->value); - } else { - fprintf(stderr, "Server returned a 2xx without a Location header.\n"); - - return 5; - } - } - - return 0; -} - -void landerctl_curl_cleanup(landerctl_curl *curl) { - curl_easy_cleanup(curl->curl); - curl_slist_free_all(curl->headers); -} diff --git a/landerctl/src/main.c b/landerctl/src/main.c index db8155a..fd53944 100644 --- a/landerctl/src/main.c +++ b/landerctl/src/main.c @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -7,31 +9,210 @@ #include "landerctl.h" -int main(int argc, char **argv) { - landerctl_args args = {0}; +const char *cfg_file_name = ".landerrc"; +const char *usage = "%s [-SPFsv] [-c CONFIG_FILE] arg [key]\n"; - int res; - if ((res = landerctl_parse_args(&args, argc, argv))) { - return res; +int main(int argc, char **argv) { + landerctl_ctx ctx = {0}; + + const char *home_dir = getenv("HOME"); + const char *cfg_path; + + if (home_dir == NULL) { + cfg_path = cfg_file_name; + } else { + // This is a blatant memleak if a custom config file is set, but it really + // doesn't matter for a short-lived CLI tool + char *buf = malloc(strlen(home_dir) + strlen(cfg_file_name) + 2); + sprintf(buf, "%s/%s", home_dir, cfg_file_name); + cfg_path = buf; } - switch (args.mode) { + opterr = 0; + int c; + + while ((c = getopt(argc, argv, "SPFsvc:")) != -1) { + switch (c) { + case 'S': + ctx.mode = landerctl_mode_short; + break; + case 'P': + ctx.mode = landerctl_mode_paste; + break; + case 'F': + ctx.mode = landerctl_mode_file; + break; + case 's': + ctx.secure = true; + break; + case 'v': + ctx.verbose = true; + break; + case 'c': + cfg_path = optarg; + break; + case '?': + printf(usage, argv[0]); + exit(2); + } + } + + char *err_msg = NULL; + landerctl_cfg_err parse_res = landerctl_cfg_parse(&ctx.cfg, cfg_path); + + switch (parse_res) { + case landerctl_cfg_err_ok: + break; + case landerctl_cfg_err_not_found: + err_msg = "Config file not found"; + break; + case landerctl_cfg_err_invalid: + err_msg = "Invalid config file"; + break; + case landerctl_cfg_err_incomplete: + err_msg = "Incomplete config file"; + break; + } + + if (err_msg != NULL) { + fprintf(stderr, "%s\n", err_msg); + exit(1); + } + + if (ctx.mode == landerctl_mode_none) { + printf("No mode specified.\n\n"); + printf(usage, argv[0]); + exit(2); + } + + if (optind == argc || (argc - optind > 2)) { + printf(usage, argv[0]); + exit(2); + } + + ctx.arg = argv[optind]; + ctx.key = argc - optind == 2 ? argv[optind + 1] : NULL; + + curl_global_init(CURL_GLOBAL_ALL); + ctx.curl = curl_easy_init(); + + if (ctx.curl == NULL) { + exit(255); + } + + landerctl_set_common(&ctx); + landerctl_err res; + + switch (ctx.mode) { case landerctl_mode_short: - res = landerctl_cmd_short(&args); + res = landerctl_post_short(&ctx); break; case landerctl_mode_paste: - res = landerctl_cmd_paste(&args); + res = landerctl_post_paste(&ctx); break; case landerctl_mode_file: - res = landerctl_cmd_file(&args); - break; - case landerctl_mode_placeholder: - res = landerctl_cmd_placeholder(&args); + res = landerctl_post_file(&ctx); break; default: - res = 7; - break; + return 7; } - return res; + if (res != landerctl_err_ok) { + printf("%s\n", landerctl_err_msg(res)); + exit(6); + } + + if (ctx.verbose) { + curl_easy_setopt(ctx.curl, CURLOPT_VERBOSE, 1L); + } + + curl_easy_setopt(ctx.curl, CURLOPT_HTTPHEADER, ctx.headers); + curl_easy_setopt(ctx.curl, CURLOPT_CAINFO, ctx.cfg.ca_certs_bundle); + + char curl_err_msg[CURL_ERROR_SIZE]; + curl_easy_setopt(ctx.curl, CURLOPT_ERRORBUFFER, curl_err_msg); + + int exit_code = 0; + + if (curl_easy_perform(ctx.curl) == CURLE_OK) { + long response_code; + curl_easy_getinfo(ctx.curl, CURLINFO_RESPONSE_CODE, &response_code); + + if (response_code < 200 || response_code > 299) { + fprintf(stderr, "HTTP status code %li\n", response_code); + exit_code = 3; + } else { + struct curl_header *location_header; + + if (curl_easy_header(ctx.curl, "Location", 0, CURLH_HEADER, -1, + &location_header) == CURLHE_OK) { + printf("%s%s\n", ctx.cfg.server_url, location_header->value); + } else { + fprintf(stderr, "Server returned a 2xx without a Location header.\n"); + exit_code = 5; + } + } + } else { + fprintf(stderr, "Libcurl encountered an error: %s\n", curl_err_msg); + exit_code = 4; + } + + curl_easy_cleanup(ctx.curl); + curl_slist_free_all(ctx.headers); + + if (ctx.data_file != NULL) { + fclose(ctx.data_file); + } + + return exit_code; + + /* struct stat sb; */ + + /* stat(argv[1], &sb); */ + + /* printf("file size: %lu\n", sb.st_size); */ + + /* FILE *f = fopen(argv[1], "rb"); */ + + /* if (f == NULL) { */ + /* printf("Couldn't open file.\n"); */ + /* exit(1); */ + /* } */ + + /* CURL *curl = curl_easy_init(); */ + + /* if (curl == NULL) { */ + /* exit(1); */ + /* } */ + + /* curl_easy_setopt(curl, CURLOPT_URL, "http://localhost:18080/f/"); */ + /* curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); */ + /* curl_easy_setopt(curl, CURLOPT_READDATA, f); */ + + /* curl_off_t file_size = sb.st_size; */ + /* /1* curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, file_size); *1/ */ + /* curl_easy_setopt(curl, CURLOPT_POST, 1L); */ + /* curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, file_size); */ + + /* magic_t cookie = magic_open(MAGIC_MIME_TYPE); */ + /* magic_load(cookie, NULL); */ + /* const char *mime_type = magic_file(cookie, argv[1]); */ + + /* char content_type_header[strlen(mime_type) + 24]; */ + /* sprintf(content_type_header, "X-Lander-Content-Type: %s", mime_type); */ + + /* char content_length_header[32]; */ + /* sprintf(content_length_header, "Content-Length: %lu", sb.st_size); */ + + /* struct curl_slist *list = NULL; */ + /* list = curl_slist_append(list, content_type_header); */ + /* list = curl_slist_append(list, content_length_header); */ + /* list = curl_slist_append(list, "X-Api-Key: test"); */ + + /* curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); */ + + /* curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); */ + /* curl_easy_perform(curl); */ + + /* curl_slist_free_all(list); */ } diff --git a/landerctl/src/post.c b/landerctl/src/post.c new file mode 100644 index 0000000..16ebac4 --- /dev/null +++ b/landerctl/src/post.c @@ -0,0 +1,131 @@ +#include +#include +#include + +#include +#include + +#include "landerctl.h" + +const char *landerctl_err_msg(landerctl_err err) { + switch (err) { + case landerctl_err_not_found: + return "File not found"; + default: + return ""; + } +} + +void landerctl_set_common(landerctl_ctx *ctx) { + size_t url_len = strlen(ctx->cfg.server_url) + 4; + + if (ctx->key != NULL) { + url_len += strlen(ctx->key); + } + + char mode_char; + + switch (ctx->mode) { + case landerctl_mode_short: + mode_char = 's'; + break; + case landerctl_mode_paste: + mode_char = 'p'; + break; + case landerctl_mode_file: + mode_char = 'f'; + break; + // Shouldn't be able to happen + default: + return; + } + + char url[url_len + 1]; + + if (ctx->key == NULL) { + sprintf(url, "%s/%c%s/", ctx->cfg.server_url, mode_char, + ctx->secure ? "l" : ""); + } else { + sprintf(url, "%s/%c%s/%s", ctx->cfg.server_url, mode_char, + ctx->secure ? "l" : "", ctx->key); + } + + curl_easy_setopt(ctx->curl, CURLOPT_URL, url); + + // Add API key header + char api_key_header[strlen(ctx->cfg.api_key) + 12]; + sprintf(api_key_header, "X-Api-Key: %s", ctx->cfg.api_key); + + ctx->headers = curl_slist_append(NULL, api_key_header); + + curl_easy_setopt(ctx->curl, CURLOPT_USERAGENT, + "landerctl/" LANDER_VERSION ""); +} + +landerctl_err landerctl_post_short(landerctl_ctx *ctx) { + curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, strlen(ctx->arg)); + curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->arg); + + return landerctl_err_ok; +} + +landerctl_err landerctl_post_paste(landerctl_ctx *ctx) { + ctx->data_file = fopen(ctx->arg, "rb"); + + if (ctx->data_file == NULL) { + return landerctl_err_not_found; + } + + struct stat sb; + stat(ctx->arg, &sb); + + curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); + curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx->data_file); + curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, sb.st_size); + + return landerctl_err_ok; +} + +landerctl_err landerctl_post_file(landerctl_ctx *ctx) { + ctx->data_file = fopen(ctx->arg, "rb"); + + if (ctx->data_file == NULL) { + return landerctl_err_not_found; + } + + struct stat sb; + stat(ctx->arg, &sb); + + curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); + curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx->data_file); + curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE_LARGE, sb.st_size); + curl_easy_setopt(ctx->curl, CURLOPT_NOPROGRESS, 0L); + + magic_t cookie = magic_open(MAGIC_MIME_TYPE); + + if (magic_load(cookie, NULL) == 0) { + const char *mime_type = magic_file(cookie, ctx->arg); + + if (mime_type != NULL) { + char content_type_header[strlen(mime_type) + 24]; + sprintf(content_type_header, "X-Lander-Content-Type: %s", mime_type); + + ctx->headers = curl_slist_append(ctx->headers, content_type_header); + } else { + printf("Couldn't determine mime type; skipping Content-Type header\n"); + } + } else { + printf("Couldn't load magic file; skipping Content-Type header\n"); + } + + char s[strlen(ctx->arg) + 1]; + strcpy(s, ctx->arg); + const char *base_name = basename(s); + + char filename_header[strlen(base_name) + 20]; + sprintf(filename_header, "X-Lander-Filename: %s", base_name); + + ctx->headers = curl_slist_append(ctx->headers, filename_header); + + return landerctl_err_ok; +} diff --git a/lsm/include/lsm.h b/lsm/include/lsm.h index 078a57f..0a1639f 100644 --- a/lsm/include/lsm.h +++ b/lsm/include/lsm.h @@ -19,8 +19,6 @@ } \ } -#define LSM_MAX(x, y) ((x) > (y) ? (x) : (y)) - typedef enum lsm_error { lsm_error_ok = 0, lsm_error_failed_alloc = 1, diff --git a/lsm/include/lsm/store.h b/lsm/include/lsm/store.h index 6aaabc7..e2d7d86 100644 --- a/lsm/include/lsm/store.h +++ b/lsm/include/lsm/store.h @@ -10,14 +10,93 @@ #define LSM_STORE_DATA_LEVELS 3 /** - * Read-only handle to an entry in the store + * A handle referencing an entry inside a store. Read/write operations from/to + * the entry go through this handle. */ -typedef struct lsm_read_handle lsm_read_handle; +typedef struct lsm_entry_handle lsm_entry_handle; /** - * Writeable handle to an entry in the store + * Checks whether the entry has an attribute with the specified type. + * + * @param entry entry to check + * @param type type of attribute to check for */ -typedef struct lsm_write_handle lsm_write_handle; +bool lsm_entry_attr_present(lsm_entry_handle *handle, uint8_t type); + +/** + * Retrieve the contents of an attribute from an entry, if present + * + * @param out where to store pointer to attribute data + * @param entry entry to search for + * @param type type of attribute to return + */ +lsm_error lsm_entry_attr_get(lsm_str **out, lsm_entry_handle *handle, + uint8_t type); + +/** + * Convenience wrapper around `lsm_entry_attr_get` that can be used if we know + * beforehand the attribute value is a 64-bit number. + * + * @param out where to store attribute data + * @param entry entry to search for + * @param type type of attribute to return + */ +lsm_error lsm_entry_attr_get_uint64_t(uint64_t *out, lsm_entry_handle *handle, + uint8_t type); + +/** + * Convenience wrapper around `lsm_entry_attr_get` that can be used if we know + * beforehand the attribute value is an 8-bit number. + * + * @param out where to store attribute data + * @param entry entry to search for + * @param type type of attribute to return + */ +lsm_error lsm_entry_attr_get_uint8_t(uint8_t *out, lsm_entry_handle *handle, + uint8_t type); + +/** + * Add a new attribute to the entry. + * + * @param entry entry to modify + * @param type type of attribute to add + * @param data data of attribute; ownership of pointer is taken over + */ +lsm_error lsm_entry_attr_insert(lsm_entry_handle *handle, uint8_t type, + lsm_str *data); + +/** + * Convenience wrapper around `lsm_entry_attr_insert` that can be used if the + * data to be stored is a 64-bit number. + * + * @param entry entry to modify + * @param type type of attribute to add + * @param data data of attribute + */ +lsm_error lsm_entry_attr_insert_uint64_t(lsm_entry_handle *handle, uint8_t type, + uint64_t data); + +/** + * Convenience wrapper around `lsm_entry_attr_insert` that can be used if the + * data to be stored is an 8-bit number. + * + * @param entry entry to modify + * @param type type of attribute to add + * @param data data of attribute + */ +lsm_error lsm_entry_attr_insert_uint8_t(lsm_entry_handle *handle, uint8_t type, + uint8_t data); + +/** + * Remove an atribute from the given entry, if present. + * + * @param out pointer to store removed data pointer in. If NULL, data pointer + * will be leaked. + * @param entry entry to remove attribute from + * @param type type of attribute to remove + */ +lsm_error lsm_entry_attr_remove(lsm_str **out, lsm_entry_handle *handle, + uint8_t type); /** * A store consisting of LSM entries. @@ -65,7 +144,7 @@ void lsm_store_free(lsm_store *store); * @param store store to retrieve entry from * @param key key to search */ -lsm_error lsm_store_open_read(lsm_read_handle **out, lsm_store *store, +lsm_error lsm_store_open_read(lsm_entry_handle **out, lsm_store *store, const lsm_str *key); /** @@ -77,9 +156,26 @@ lsm_error lsm_store_open_read(lsm_read_handle **out, lsm_store *store, * @param store store to retrieve entry from * @param key key to search */ -lsm_error lsm_store_open_write(lsm_write_handle **out, lsm_store *store, +lsm_error lsm_store_open_write(lsm_entry_handle **out, lsm_store *store, const lsm_str *key); +/** + * Commit any changes to the persistent storage. Any changes, insertions or + * deletions that occured without a commit are reverted when the handle is + * closed. + * + * @param handle handle to the entry + */ +lsm_error lsm_entry_commit(lsm_entry_handle *handle); + +/** + * Close an open entry handle. + * + * @param store store the handle's entry is stored in + * @param handle handle to close + */ +void lsm_entry_close(lsm_entry_handle *handle); + /** * Insert a new entry into the store, returning a write handle to the newly * created entry. @@ -88,43 +184,57 @@ lsm_error lsm_store_open_write(lsm_write_handle **out, lsm_store *store, * @param store store to modify * @param key key to add; ownership of key pointer is taken over */ -lsm_error lsm_store_open_new(lsm_write_handle **out, lsm_store *store, - lsm_str *key); +lsm_error lsm_store_insert(lsm_entry_handle **out, lsm_store *store, + lsm_str *key); -bool lsm_read_attr_present(lsm_read_handle *handle, uint8_t type); -lsm_error lsm_read_attr_get(const lsm_str **out, const lsm_read_handle *handle, - uint8_t type); -lsm_error lsm_read_attr_get_uint64_t(uint64_t *out, - const lsm_read_handle *handle, - uint8_t type); -lsm_error lsm_read_attr_get_uint8_t(uint8_t *out, const lsm_read_handle *handle, - uint8_t type); -uint64_t lsm_read_data_len(const lsm_read_handle *handle); -lsm_error lsm_read_data_read(uint64_t *out, char *buf, lsm_read_handle *handle, - uint64_t len); -void lsm_read_close(lsm_read_handle *handle); +/** + * Mark the entry as removed. + * + * @param handle handle to entry to remove + */ +void lsm_entry_remove(lsm_entry_handle *handle); -bool lsm_write_attr_present(const lsm_write_handle *handle, uint8_t type); -lsm_error lsm_write_attr_get(const lsm_str **out, - const lsm_write_handle *handle, uint8_t type); -lsm_error lsm_write_attr_get_uint64_t(uint64_t *out, - const lsm_write_handle *handle, - uint8_t type); -lsm_error lsm_write_attr_get_uint8_t(uint8_t *out, - const lsm_write_handle *handle, - uint8_t type); -lsm_error lsm_write_attr_remove(lsm_str **out, lsm_write_handle *handle, - uint8_t type); -lsm_error lsm_write_attr_insert(lsm_write_handle *handle, uint8_t type, - lsm_str *data); -lsm_error lsm_write_attr_insert_uint64_t(lsm_write_handle *handle, uint8_t type, - uint64_t data); -lsm_error lsm_write_attr_insert_uint8_t(lsm_write_handle *handle, uint8_t type, - uint8_t data); -uint64_t lsm_write_data_len(const lsm_write_handle *handle); -lsm_error lsm_write_data_append(lsm_write_handle *handle, const lsm_str *data); -void lsm_write_remove(lsm_write_handle *handle); -void lsm_write_close(lsm_write_handle *handle); -lsm_error lsm_write_commit(lsm_write_handle *handle); +/** + * Append new data to the given entry, which is expected to be in the store. + * + * This function will append either to disk or to memory, depending on the + * length of the entry's data. + * + * @param store store the entry is stored in + * @param entry entry to append data to + * @param data data to append + */ +lsm_error lsm_entry_data_append(lsm_entry_handle *handle, const lsm_str *data); + +/** + * Same as `lsm_entry_data_append`, except that it takes a direct char array. + * + * @param store store the entry is stored in + * @param entry entry to append data to + * @param data data to append + * @param len length of data array + */ +lsm_error lsm_entry_data_append_raw(lsm_entry_handle *handle, char *data, + uint64_t len); + +/** + * Read a number of bytes from the entry's data field. The position from which + * data is read is dependent on previous read calls. + * + * @param out where to write how many bytes were read + * @param buf buffer to store read data in + * @param handle entry handle to read from + * @param len how many bytes to read at most + */ +lsm_error lsm_entry_data_read(uint64_t *out, char *buf, + lsm_entry_handle *handle, uint64_t len); + +/** + * Return the length of the entry's data. + * + * @param handle entry handle to return length for + * @return length of the data + */ +uint64_t lsm_entry_data_len(lsm_entry_handle *handle); #endif diff --git a/lsm/src/_include/lsm/store_internal.h b/lsm/src/_include/lsm/store_internal.h index 33015a0..1b84b60 100644 --- a/lsm/src/_include/lsm/store_internal.h +++ b/lsm/src/_include/lsm/store_internal.h @@ -48,11 +48,6 @@ lsm_error lsm_entry_init(lsm_entry **ptr); */ void lsm_entry_free(lsm_entry *entry); -/** - * Allocates a new entry that's identical to the provided one. - */ -lsm_error lsm_entry_clone(lsm_entry **out, const lsm_entry *entry); - /** * Deallocate an existing lsm_entry object. * @@ -74,6 +69,19 @@ typedef enum lsm_entry_handle_state : uint8_t { lsm_entry_handle_state_removed = 1 << 2, } lsm_entry_handle_state; +struct lsm_entry_handle { + lsm_entry_wrapper *wrapper; + lsm_store *store; + // Either read or append, depending on how it was opened + FILE *f; + // Current position in the file pointer + uint64_t pos; + // Required to determine in what way the database files need to be synced + uint64_t states; +}; + +lsm_error lsm_entry_handle_init(lsm_entry_handle **out); + struct lsm_store { lsm_trie *trie; lsm_str *data_path; @@ -106,45 +114,47 @@ lsm_error lsm_store_load_db(lsm_store *store); * * @param handle handle to added entry */ -lsm_error lsm_entry_disk_insert(lsm_store *store, lsm_entry *entry); +lsm_error lsm_entry_disk_insert(lsm_entry_handle *handle); /** * Remove an entry from the database. * * @param handle handle to the removed entry */ -lsm_error lsm_entry_disk_remove(lsm_store *store, lsm_entry *entry); +lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle); /** * Update an existing entry already in the store. * * @param handle to updated entry */ -lsm_error lsm_entry_disk_update(lsm_store *store, lsm_entry *entry); +lsm_error lsm_entry_disk_update(lsm_entry_handle *handle); /** * Return the length of the path to this entry's data file */ -uint64_t lsm_entry_data_path_len(const lsm_store *store, - const lsm_entry *entry); +uint64_t lsm_entry_data_path_len(const lsm_entry_handle *handle); /** * Fill in the entry's data file path in the provided buffer. Use * `lsm_entry_data_path_len` to allocate an appropriately-sized buffer */ -void lsm_entry_data_path(char *path, const lsm_store *store, - const lsm_entry *entry); +void lsm_entry_data_path(char *buf, const lsm_entry_handle *handle); /** * Open the entry's data file for reading * * @param handle handle to the entry */ -lsm_error lsm_entry_data_open_read(lsm_read_handle *handle); +lsm_error lsm_entry_data_open_read(lsm_entry_handle *handle); -lsm_error lsm_entry_data_open(FILE **out, const lsm_store *store, - const lsm_entry *entry, const char *mode); -lsm_error lsm_entry_data_mkdirs(const lsm_store *store, const lsm_entry *entry); +/** + * Open the entry's data file for writing. The file and all subdirectories in + * the data dir are created as needed. + * + * @param handle handle to the entry + */ +lsm_error lsm_entry_data_open_write(lsm_entry_handle *handle); /** * Remove the entry's data file if present and close its handle. Any uncommitted @@ -152,130 +162,6 @@ lsm_error lsm_entry_data_mkdirs(const lsm_store *store, const lsm_entry *entry); * * @param handle handle to the entry */ -lsm_error lsm_entry_data_remove(const lsm_store *store, const lsm_entry *entry); - -/** - * Checks whether the entry has an attribute with the specified type. - * - * @param entry entry to check - * @param type type of attribute to check for - */ -bool lsm_entry_attr_present(const lsm_entry *entry, uint8_t type); - -/** - * Retrieve the contents of an attribute from an entry, if present - * - * @param out where to store pointer to attribute data - * @param entry entry to search for - * @param type type of attribute to return - */ -lsm_error lsm_entry_attr_get(const lsm_str **out, const lsm_entry *entry, - uint8_t type); - -/** - * Convenience wrapper around `lsm_entry_attr_get` that can be used if we know - * beforehand the attribute value is a 64-bit number. - * - * @param out where to store attribute data - * @param entry entry to search for - * @param type type of attribute to return - */ -lsm_error lsm_entry_attr_get_uint64_t(uint64_t *out, const lsm_entry *entry, - uint8_t type); - -/** - * Convenience wrapper around `lsm_entry_attr_get` that can be used if we know - * beforehand the attribute value is an 8-bit number. - * - * @param out where to store attribute data - * @param entry entry to search for - * @param type type of attribute to return - */ -lsm_error lsm_entry_attr_get_uint8_t(uint8_t *out, const lsm_entry *entry, - uint8_t type); - -/** - * Add a new attribute to the entry. This overwrites an existing version of this - * attribute. - * - * @param entry entry to modify - * @param type type of attribute to add - * @param data data of attribute; ownership of pointer is taken over - */ -lsm_error lsm_entry_attr_insert(lsm_entry *entry, uint8_t type, lsm_str *data); - -/** - * Convenience wrapper around `lsm_entry_attr_insert` that can be used if the - * data to be stored is a 64-bit number. - * - * @param entry entry to modify - * @param type type of attribute to add - * @param data data of attribute - */ -lsm_error lsm_entry_attr_insert_uint64_t(lsm_entry *entry, uint8_t type, - uint64_t data); - -/** - * Convenience wrapper around `lsm_entry_attr_insert` that can be used if the - * data to be stored is an 8-bit number. - * - * @param entry entry to modify - * @param type type of attribute to add - * @param data data of attribute - */ -lsm_error lsm_entry_attr_insert_uint8_t(lsm_entry *entry, uint8_t type, - uint8_t data); - -/** - * Remove an atribute from the given entry, if present. - * - * @param out pointer to store removed data pointer in. If NULL, data pointer - * will be leaked. - * @param entry entry to remove attribute from - * @param type type of attribute to remove - */ -lsm_error lsm_entry_attr_remove(lsm_str **out, lsm_entry *entry, uint8_t type); - -/**************************************** - *************** Handles *************** - ***************************************/ - -struct lsm_read_handle { - lsm_entry_wrapper *wrapper; - lsm_store *store; - - struct { - FILE *f; - uint64_t pos; - } data; -}; - -struct lsm_write_handle { - lsm_entry_wrapper *wrapper; - lsm_store *store; - - lsm_entry *dirty; - bool removed; - - struct { - FILE *f; - uint64_t pos; - } data; -}; - -/** - * Allocate a new `lsm_read_handle` object - */ -lsm_error lsm_read_handle_init(lsm_read_handle **out); - -/** - * Allocate a new `lsm_write_handle` object - */ -lsm_error lsm_write_handle_init(lsm_write_handle **out); - -/** - * Commit changes solely to the memory part of the store. - */ -void lsm_write_commit_mem(lsm_write_handle *handle); +lsm_error lsm_entry_data_remove(lsm_entry_handle *handle); #endif diff --git a/lsm/src/store/lsm_handle_read.c b/lsm/src/store/lsm_handle_read.c deleted file mode 100644 index 576b68a..0000000 --- a/lsm/src/store/lsm_handle_read.c +++ /dev/null @@ -1,79 +0,0 @@ -#include - -#include "lsm/store_internal.h" - -lsm_error lsm_read_handle_init(lsm_read_handle **out) { - lsm_read_handle *handle = calloc(1, sizeof(lsm_read_handle)); - - if (handle == NULL) { - return lsm_error_failed_alloc; - } - - *out = handle; - - return lsm_error_ok; -} - -bool lsm_read_attr_present(lsm_read_handle *handle, uint8_t type) { - return lsm_entry_attr_present(handle->wrapper->entry, type); -} - -lsm_error lsm_read_attr_get(const lsm_str **out, const lsm_read_handle *handle, - uint8_t type) { - return lsm_entry_attr_get(out, handle->wrapper->entry, type); -} - -lsm_error lsm_read_attr_get_uint64_t(uint64_t *out, - const lsm_read_handle *handle, - uint8_t type) { - return lsm_entry_attr_get_uint64_t(out, handle->wrapper->entry, type); -} - -lsm_error lsm_read_attr_get_uint8_t(uint8_t *out, const lsm_read_handle *handle, - uint8_t type) { - return lsm_entry_attr_get_uint8_t(out, handle->wrapper->entry, type); -} - -uint64_t lsm_read_data_len(const lsm_read_handle *handle) { - return handle->wrapper->entry->data_len; -} - -lsm_error lsm_read_data_read(uint64_t *out, char *buf, lsm_read_handle *handle, - uint64_t len) { - const lsm_entry *entry = handle->wrapper->entry; - - if (entry->data_len == 0) { - *out = 0; - - return lsm_error_ok; - } - - // Entries don't open their file unless needed - if (handle->data.f == NULL) { - LSM_RES(lsm_entry_data_open(&handle->data.f, handle->store, - handle->wrapper->entry, "rb")); - } - - uint64_t read; - - read = fread(buf, sizeof(char), len, handle->data.f); - - if ((read == 0) && (ferror(handle->data.f) != 0)) { - return lsm_error_failed_io; - } - - handle->data.pos += read; - *out = read; - - return lsm_error_ok; -} - -void lsm_read_close(lsm_read_handle *handle) { - if (handle->data.f != NULL) { - fclose(handle->data.f); - handle->data.f = NULL; - } - - pthread_rwlock_unlock(&handle->wrapper->lock); - free(handle); -} diff --git a/lsm/src/store/lsm_handle_write.c b/lsm/src/store/lsm_handle_write.c deleted file mode 100644 index 27993b9..0000000 --- a/lsm/src/store/lsm_handle_write.c +++ /dev/null @@ -1,183 +0,0 @@ -#include - -#include "lsm/store_internal.h" - -lsm_error lsm_write_handle_init(lsm_write_handle **out) { - lsm_write_handle *handle = calloc(1, sizeof(lsm_write_handle)); - - if (handle == NULL) { - return lsm_error_failed_alloc; - } - - *out = handle; - - return lsm_error_ok; -} - -bool lsm_write_attr_present(const lsm_write_handle *handle, uint8_t type) { - return lsm_entry_attr_present(handle->wrapper->entry, type); -} - -lsm_error lsm_write_attr_get(const lsm_str **out, - const lsm_write_handle *handle, uint8_t type) { - lsm_entry *entry = - handle->dirty == NULL ? handle->wrapper->entry : handle->dirty; - return lsm_entry_attr_get(out, entry, type); -} - -lsm_error lsm_write_attr_get_uint64_t(uint64_t *out, - const lsm_write_handle *handle, - uint8_t type) { - lsm_entry *entry = - handle->dirty == NULL ? handle->wrapper->entry : handle->dirty; - return lsm_entry_attr_get_uint64_t(out, entry, type); -} - -lsm_error lsm_write_attr_get_uint8_t(uint8_t *out, - const lsm_write_handle *handle, - uint8_t type) { - lsm_entry *entry = - handle->dirty == NULL ? handle->wrapper->entry : handle->dirty; - return lsm_entry_attr_get_uint8_t(out, entry, type); -} - -lsm_error lsm_write_attr_remove(lsm_str **out, lsm_write_handle *handle, - uint8_t type) { - if (handle->dirty == NULL) { - LSM_RES(lsm_entry_clone(&handle->dirty, handle->wrapper->entry)); - } - - return lsm_entry_attr_remove(out, handle->dirty, type); -} - -lsm_error lsm_write_attr_insert(lsm_write_handle *handle, uint8_t type, - lsm_str *data) { - if (handle->dirty == NULL) { - LSM_RES(lsm_entry_clone(&handle->dirty, handle->wrapper->entry)); - } - - return lsm_entry_attr_insert(handle->dirty, type, data); -} - -lsm_error lsm_write_attr_insert_uint64_t(lsm_write_handle *handle, uint8_t type, - uint64_t data) { - if (handle->dirty == NULL) { - LSM_RES(lsm_entry_clone(&handle->dirty, handle->wrapper->entry)); - } - - return lsm_entry_attr_insert_uint64_t(handle->dirty, type, data); -} - -lsm_error lsm_write_attr_insert_uint8_t(lsm_write_handle *handle, uint8_t type, - uint8_t data) { - if (handle->dirty == NULL) { - LSM_RES(lsm_entry_clone(&handle->dirty, handle->wrapper->entry)); - } - - return lsm_entry_attr_insert_uint8_t(handle->dirty, type, data); -} - -uint64_t lsm_write_data_len(const lsm_write_handle *handle) { - lsm_entry *entry = - handle->dirty == NULL ? handle->wrapper->entry : handle->dirty; - - return entry->data_len; -} - -lsm_error lsm_write_data_append(lsm_write_handle *handle, const lsm_str *data) { - if (lsm_str_len(data) == 0) { - return lsm_error_ok; - } - - if (handle->dirty == NULL) { - LSM_RES(lsm_entry_clone(&handle->dirty, handle->wrapper->entry)); - } - - lsm_entry *entry = handle->dirty; - - uint64_t new_len = entry->data_len + lsm_str_len(data); - const char *data_s = lsm_str_ptr(data); - - // Entries don't open their file unless needed - if (handle->data.f == NULL) { - // An entry with no existing data will not have a data file yet, so we set - // create to true then - LSM_RES(lsm_entry_data_mkdirs(handle->store, entry)); - LSM_RES(lsm_entry_data_open(&handle->data.f, handle->store, entry, "ab")); - } - - size_t written = 0; - - // TODO what happens when I/O fails? - while (written < data->len) { - written += fwrite(&data_s[written], sizeof(char), data->len - written, - handle->data.f); - } - - entry->data_len = new_len; - - return lsm_error_ok; -} - -void lsm_write_remove(lsm_write_handle *handle) { handle->removed = true; } - -void lsm_write_close(lsm_write_handle *handle) { - if (handle->data.f != NULL) { - fclose(handle->data.f); - handle->data.f = NULL; - } - - if (handle->dirty != NULL) { - // Entry was never committed to store, so any created data file should be - // removed - if (handle->wrapper->entry == NULL) { - lsm_entry_data_remove(handle->store, handle->dirty); - } - - lsm_entry_free(handle->dirty); - } - - pthread_rwlock_unlock(&handle->wrapper->lock); - free(handle); -} - -lsm_error lsm_write_commit(lsm_write_handle *handle) { - if (handle->removed && (handle->wrapper->entry != NULL)) { - LSM_RES(lsm_entry_disk_remove(handle->store, handle->wrapper->entry)); - - lsm_entry_free(handle->wrapper->entry); - handle->wrapper->entry = NULL; - handle->removed = false; - - return lsm_error_ok; - } - - if (handle->dirty == NULL) { - return lsm_error_ok; - } - - if (handle->wrapper->entry == NULL) { - LSM_RES(lsm_entry_disk_insert(handle->store, handle->dirty)); - } else { - LSM_RES(lsm_entry_disk_update(handle->store, handle->dirty)); - lsm_entry_free(handle->wrapper->entry); - } - - handle->wrapper->entry = handle->dirty; - handle->dirty = NULL; - - return lsm_error_ok; -} - -void lsm_write_commit_mem(lsm_write_handle *handle) { - if (handle->dirty == NULL) { - return; - } - - if (handle->wrapper->entry != NULL) { - lsm_entry_free(handle->wrapper->entry); - } - - handle->wrapper->entry = handle->dirty; - handle->dirty = NULL; -} diff --git a/lsm/src/store/lsm_store.c b/lsm/src/store/lsm_store.c index 3e9aeee..7b3ed36 100644 --- a/lsm/src/store/lsm_store.c +++ b/lsm/src/store/lsm_store.c @@ -28,7 +28,7 @@ uint64_t lsm_store_size(const lsm_store *store) { return lsm_trie_size(store->trie); } -lsm_error lsm_store_open_read(lsm_read_handle **out, lsm_store *store, +lsm_error lsm_store_open_read(lsm_entry_handle **out, lsm_store *store, const lsm_str *key) { lsm_entry_wrapper *wrapper; @@ -47,8 +47,8 @@ lsm_error lsm_store_open_read(lsm_read_handle **out, lsm_store *store, return lsm_error_not_found; } - lsm_read_handle *handle; - LSM_RES2(lsm_read_handle_init(&handle), + lsm_entry_handle *handle; + LSM_RES2(lsm_entry_handle_init(&handle), pthread_rwlock_unlock(&wrapper->lock)); handle->wrapper = wrapper; @@ -58,7 +58,7 @@ lsm_error lsm_store_open_read(lsm_read_handle **out, lsm_store *store, return lsm_error_ok; } -lsm_error lsm_store_open_write(lsm_write_handle **out, lsm_store *store, +lsm_error lsm_store_open_write(lsm_entry_handle **out, lsm_store *store, const lsm_str *key) { lsm_entry_wrapper *wrapper; LSM_RES(lsm_trie_search((void **)&wrapper, store->trie, key)); @@ -77,8 +77,8 @@ lsm_error lsm_store_open_write(lsm_write_handle **out, lsm_store *store, return lsm_error_not_found; } - lsm_write_handle *handle; - LSM_RES2(lsm_write_handle_init(&handle), + lsm_entry_handle *handle; + LSM_RES2(lsm_entry_handle_init(&handle), pthread_rwlock_unlock(&wrapper->lock)); handle->wrapper = wrapper; @@ -88,8 +88,8 @@ lsm_error lsm_store_open_write(lsm_write_handle **out, lsm_store *store, return lsm_error_ok; } -lsm_error lsm_store_open_new(lsm_write_handle **out, lsm_store *store, - lsm_str *key) { +lsm_error lsm_store_insert(lsm_entry_handle **out, lsm_store *store, + lsm_str *key) { // TODO what happens when two inserts to the same key happen at the same time? lsm_entry_wrapper *wrapper; @@ -117,16 +117,84 @@ lsm_error lsm_store_open_new(lsm_write_handle **out, lsm_store *store, LSM_RES2(lsm_entry_init(&entry), pthread_rwlock_unlock(&wrapper->lock)); entry->key = key; + wrapper->entry = entry; - lsm_write_handle *handle; - LSM_RES2(lsm_write_handle_init(&handle), + lsm_entry_handle *handle; + LSM_RES2(lsm_entry_handle_init(&handle), pthread_rwlock_unlock(&wrapper->lock)); + // No need to set the handle's file, as the entry doesn't have any data yet handle->wrapper = wrapper; handle->store = store; - handle->dirty = entry; + + // Newly inserted entries are always dirty + handle->states |= lsm_entry_handle_state_new; *out = handle; return lsm_error_ok; } + +void lsm_entry_remove(lsm_entry_handle *handle) { + handle->states |= lsm_entry_handle_state_removed; +} + +lsm_error lsm_entry_data_append(lsm_entry_handle *handle, const lsm_str *data) { + if (lsm_str_len(data) == 0) { + return lsm_error_ok; + } + + lsm_entry *entry = handle->wrapper->entry; + + uint64_t new_len = entry->data_len + lsm_str_len(data); + const char *data_s = lsm_str_ptr(data); + + // Entries don't open their file unless needed + if (handle->f == NULL) { + // An entry with no existing data will not have a data file yet, so we set + // create to true then + LSM_RES(lsm_entry_data_open_write(handle)); + } + + size_t written = 0; + + // TODO what happens when I/O fails? + while (written < data->len) { + written += + fwrite(&data_s[written], sizeof(char), data->len - written, handle->f); + } + + entry->data_len = new_len; + handle->states |= lsm_entry_handle_state_updated; + + return lsm_error_ok; +} + +lsm_error lsm_entry_data_read(uint64_t *out, char *buf, + lsm_entry_handle *handle, uint64_t len) { + const lsm_entry *entry = handle->wrapper->entry; + + if (entry->data_len == 0) { + *out = 0; + + return lsm_error_ok; + } + + // Entries don't open their file unless needed + if (handle->f == NULL) { + LSM_RES(lsm_entry_data_open_read(handle)); + } + + uint64_t read; + + read = fread(buf, sizeof(char), len, handle->f); + + if ((read == 0) && (ferror(handle->f) != 0)) { + return lsm_error_failed_io; + } + + handle->pos += read; + *out = read; + + return lsm_error_ok; +} diff --git a/lsm/src/store/lsm_store_disk_read.c b/lsm/src/store/lsm_store_disk_read.c index 9b092c5..721b4f3 100644 --- a/lsm/src/store/lsm_store_disk_read.c +++ b/lsm/src/store/lsm_store_disk_read.c @@ -130,7 +130,7 @@ static lsm_error lsm_entry_read_str(lsm_str **out, uint64_t *sum, FILE *f) { return lsm_str_init(out, buf); } -static lsm_error lsm_entry_read_attrs(uint64_t *sum, lsm_entry *entry, +static lsm_error lsm_entry_read_attrs(uint64_t *sum, lsm_entry_handle *handle, FILE *db_file) { uint8_t attr_count; LSM_RES(lsm_fread(&attr_count, sum, db_file, sizeof(uint8_t), 1)); @@ -142,12 +142,11 @@ static lsm_error lsm_entry_read_attrs(uint64_t *sum, lsm_entry *entry, for (uint64_t i = 0; i < attr_count; i++) { LSM_RES(lsm_fread(&attr_type, sum, db_file, sizeof(uint8_t), 1)); LSM_RES(lsm_entry_read_str(&val, sum, db_file)); - LSM_RES(lsm_entry_attr_insert(entry, attr_type, val)); + LSM_RES(lsm_entry_attr_insert(handle, attr_type, val)); } return lsm_error_ok; } - static lsm_error lsm_fseek(FILE *f, uint64_t pos) { if (fseek(f, pos, SEEK_SET) != 0) { return lsm_error_failed_io; @@ -164,19 +163,19 @@ lsm_error lsm_store_insert_from_db(lsm_store *store, uint64_t pos, LSM_RES(lsm_fseek(store->db.f, pos)); lsm_str *key; - LSM_RES(lsm_entry_read_str(&key, NULL, store->db.f)); + LSM_RES(lsm_entry_read_str(&key, &store->db.size, store->db.f)); - lsm_write_handle *handle; - LSM_RES(lsm_store_open_new(&handle, store, key)); + lsm_entry_handle *handle; + LSM_RES(lsm_store_insert(&handle, store, key)); - LSM_RES(lsm_fread(&handle->dirty->data_len, NULL, store->db.f, - sizeof(uint64_t), 1)); - LSM_RES(lsm_entry_read_attrs(NULL, handle->dirty, store->db.f)); + LSM_RES(lsm_fread(&handle->wrapper->entry->data_len, &store->db.size, + store->db.f, sizeof(uint64_t), 1)); + LSM_RES(lsm_entry_read_attrs(&store->db.size, handle, store->db.f)); - handle->dirty->idx_file_offset = idx_file_offset; + handle->wrapper->entry->idx_file_offset = idx_file_offset; - lsm_write_commit_mem(handle); - lsm_write_close(handle); + handle->states = 0; + lsm_entry_close(handle); return lsm_error_ok; } @@ -190,8 +189,6 @@ lsm_error lsm_store_load_db(lsm_store *store) { LSM_RES(lsm_fread(&store->idx.block_count, &store->idx.size, store->idx.f, sizeof(uint64_t), 1)); - uint64_t db_file_size = 0; - for (uint64_t i = 0; i < store->idx.block_count; i++) { uint64_t idx_file_offset = store->idx.size; @@ -204,14 +201,7 @@ lsm_error lsm_store_load_db(lsm_store *store) { } LSM_RES(lsm_store_insert_from_db(store, db_dim[0], idx_file_offset)); - - // The non-zeroed entry with the largest index determines the actual size of - // the file. This way, any zeroed entries at the end of the file can be - // overwritten. - db_file_size = LSM_MAX(db_file_size, db_dim[0] + db_dim[1]); } - store->db.size = db_file_size; - return lsm_error_ok; } diff --git a/lsm/src/store/lsm_store_disk_write.c b/lsm/src/store/lsm_store_disk_write.c index 5b67e06..51e9be8 100644 --- a/lsm/src/store/lsm_store_disk_write.c +++ b/lsm/src/store/lsm_store_disk_write.c @@ -1,5 +1,3 @@ -#include - #include "lsm/store_internal.h" static lsm_error lsm_fwrite(uint64_t *sum, FILE *f, uint64_t size, @@ -74,14 +72,16 @@ lsm_error lsm_write_idx_entry(uint64_t *size, FILE *idx_file, uint64_t offset, return lsm_error_ok; } -lsm_error lsm_entry_disk_insert(lsm_store *store, lsm_entry *entry) { +lsm_error lsm_entry_disk_insert(lsm_entry_handle *handle) { + lsm_store *store = handle->store; + pthread_mutex_lock(&store->db.lock); uint64_t db_entry_index = store->db.size; uint64_t db_entry_size; - lsm_error res = - lsm_write_db_entry(&db_entry_size, store->db.f, entry, store->db.size); + lsm_error res = lsm_write_db_entry(&db_entry_size, store->db.f, + handle->wrapper->entry, store->db.size); fflush(store->db.f); pthread_mutex_unlock(&store->db.lock); @@ -115,7 +115,7 @@ lsm_error lsm_entry_disk_insert(lsm_store *store, lsm_entry *entry) { store->idx.block_count = new_block_count; store->db.size += db_entry_size; - entry->idx_file_offset = idx_entry_index; + handle->wrapper->entry->idx_file_offset = idx_entry_index; } } @@ -125,10 +125,16 @@ lsm_error lsm_entry_disk_insert(lsm_store *store, lsm_entry *entry) { return res; } -static lsm_error lsm_idx_zero_block(lsm_store *store, uint64_t pos) { +// Marking an entry as removed in the idx file is simply setting the length of +// its entry to zero +lsm_error lsm_entry_disk_remove(lsm_entry_handle *handle) { + lsm_store *store = handle->store; + const lsm_entry *entry = handle->wrapper->entry; + pthread_mutex_lock(&store->idx.lock); - lsm_error res = lsm_fseek(store->idx.f, pos); + lsm_error res = + lsm_fseek(store->idx.f, entry->idx_file_offset + sizeof(uint64_t)); if (res != lsm_error_ok) { pthread_mutex_unlock(&store->idx.lock); @@ -147,26 +153,7 @@ static lsm_error lsm_idx_zero_block(lsm_store *store, uint64_t pos) { fflush(store->idx.f); - return lsm_error_ok; -} - -// Marking an entry as removed in the idx file is simply setting the length of -// its entry to zero -lsm_error lsm_entry_disk_remove(lsm_store *store, lsm_entry *entry) { - LSM_RES(lsm_idx_zero_block(store, entry->idx_file_offset + sizeof(uint64_t))); - LSM_RES(lsm_entry_data_remove(store, entry)); - - return lsm_error_ok; -} - -lsm_error lsm_entry_disk_update(lsm_store *store, lsm_entry *entry) { - // An update is implemented by reinserting the entry at the end of the db file - uint64_t old_idx_index = entry->idx_file_offset; - - // TODO is there any way we can make this atomic? If the zero write to the - // index file fails, there are two entries in the db file for the same key. - LSM_RES(lsm_entry_disk_insert(store, entry)); - LSM_RES(lsm_idx_zero_block(store, old_idx_index + sizeof(uint64_t))); + LSM_RES(lsm_entry_data_remove(handle)); return lsm_error_ok; } diff --git a/lsm/src/store/lsm_store_entry.c b/lsm/src/store/lsm_store_entry.c index dbc124c..7d33d30 100644 --- a/lsm/src/store/lsm_store_entry.c +++ b/lsm/src/store/lsm_store_entry.c @@ -24,45 +24,12 @@ lsm_error lsm_entry_init(lsm_entry **ptr) { void lsm_entry_free(lsm_entry *entry) { if (entry->attrs.count > 0) { - for (size_t i = 0; i < entry->attrs.count; i++) { - lsm_str_free(entry->attrs.items[i].str); - } - free(entry->attrs.items); } free(entry); } -lsm_error lsm_entry_clone(lsm_entry **out, const lsm_entry *entry) { - lsm_entry *new; - LSM_RES(lsm_entry_init(&new)); - - lsm_str_init_copy_n(&new->key, lsm_str_ptr(entry->key), - lsm_str_len(entry->key)); - - for (int i = 0; i < 4; i++) { - new->attrs.bitmap[i] = entry->attrs.bitmap[i]; - } - - new->attrs.count = entry->attrs.count; - new->attrs.items = malloc(sizeof(lsm_attr) * entry->attrs.count); - - for (size_t i = 0; i < entry->attrs.count; i++) { - new->attrs.items[i].type = entry->attrs.items[i].type; - lsm_str_init_copy_n(&new->attrs.items[i].str, - lsm_str_ptr(entry->attrs.items[i].str), - lsm_str_len(entry->attrs.items[i].str)); - } - - new->data_len = entry->data_len; - new->idx_file_offset = entry->idx_file_offset; - - *out = new; - - return lsm_error_ok; -} - lsm_error lsm_entry_wrapper_init(lsm_entry_wrapper **ptr) { lsm_entry_wrapper *wrap = calloc(1, sizeof(lsm_entry_wrapper)); @@ -79,16 +46,75 @@ lsm_error lsm_entry_wrapper_init(lsm_entry_wrapper **ptr) { void lsm_entry_wrapper_free(lsm_entry_wrapper *wrapper) { free(wrapper); } -bool lsm_entry_attr_present(const lsm_entry *entry, uint8_t type) { - return (entry->attrs.bitmap[type / 64] & (((uint64_t)1) << (type % 64))) != 0; +lsm_error lsm_entry_handle_init(lsm_entry_handle **out) { + lsm_entry_handle *handle = calloc(1, sizeof(lsm_entry_handle)); + + if (handle == NULL) { + return lsm_error_failed_alloc; + } + + *out = handle; + + return lsm_error_ok; } -lsm_error lsm_entry_attr_get(const lsm_str **out, const lsm_entry *entry, +lsm_error lsm_entry_commit(lsm_entry_handle *handle) { + uint8_t state_new = handle->states & lsm_entry_handle_state_new; + uint8_t state_removed = handle->states & lsm_entry_handle_state_removed; + + // Clean new entry + if (state_new && !state_removed) { + LSM_RES(lsm_entry_disk_insert(handle)); + } + // Previously stored entry that needs to be removed; should be removed from db + // file as well + else if (state_removed && !state_new) { + LSM_RES(lsm_entry_disk_remove(handle)); + + lsm_entry_free(handle->wrapper->entry); + handle->wrapper->entry = NULL; + } + + // Reset states after committing current changes + handle->states = 0; + + return lsm_error_ok; +} + +void lsm_entry_close(lsm_entry_handle *handle) { + if (handle->f != NULL) { + fclose(handle->f); + handle->f = NULL; + } + + uint8_t state_new = handle->states & lsm_entry_handle_state_new; + /* bool state_updated = handle->states & lsm_entry_handle_state_updated; */ + + // New entries create a wrapper in the trie that should be removed if not + // committed + if (state_new) { + lsm_entry_data_remove(handle); + + lsm_entry_free(handle->wrapper->entry); + handle->wrapper->entry = NULL; + } + + pthread_rwlock_unlock(&handle->wrapper->lock); + free(handle); +} + +bool lsm_entry_attr_present(lsm_entry_handle *handle, uint8_t type) { + return (handle->wrapper->entry->attrs.bitmap[type / 64] & + (((uint64_t)1) << (type % 64))) != 0; +} + +lsm_error lsm_entry_attr_get(lsm_str **out, lsm_entry_handle *handle, uint8_t type) { - if (!lsm_entry_attr_present(entry, type)) { + if (!lsm_entry_attr_present(handle, type)) { return lsm_error_not_found; } + lsm_entry *entry = handle->wrapper->entry; uint64_t i = 0; while (entry->attrs.items[i].type != type) { @@ -100,11 +126,11 @@ lsm_error lsm_entry_attr_get(const lsm_str **out, const lsm_entry *entry, return lsm_error_ok; } -lsm_error lsm_entry_attr_get_uint64_t(uint64_t *out, const lsm_entry *entry, +lsm_error lsm_entry_attr_get_uint64_t(uint64_t *out, lsm_entry_handle *handle, uint8_t type) { - const lsm_str *s; + lsm_str *s; - LSM_RES(lsm_entry_attr_get(&s, entry, type)); + LSM_RES(lsm_entry_attr_get(&s, handle, type)); uint64_t num = 0; @@ -117,22 +143,25 @@ lsm_error lsm_entry_attr_get_uint64_t(uint64_t *out, const lsm_entry *entry, return lsm_error_ok; } -lsm_error lsm_entry_attr_get_uint8_t(uint8_t *out, const lsm_entry *entry, +lsm_error lsm_entry_attr_get_uint8_t(uint8_t *out, lsm_entry_handle *handle, uint8_t type) { - const lsm_str *s; + lsm_str *s; - LSM_RES(lsm_entry_attr_get(&s, entry, type)); + LSM_RES(lsm_entry_attr_get(&s, handle, type)); *out = lsm_str_char(s, 0); return lsm_error_ok; } -lsm_error lsm_entry_attr_remove(lsm_str **out, lsm_entry *entry, uint8_t type) { - if (!lsm_entry_attr_present(entry, type)) { +lsm_error lsm_entry_attr_remove(lsm_str **out, lsm_entry_handle *handle, + uint8_t type) { + if (!lsm_entry_attr_present(handle, type)) { return lsm_error_not_found; } + lsm_entry *entry = handle->wrapper->entry; + if (entry->attrs.count == 1) { *out = entry->attrs.items[0].str; @@ -170,16 +199,19 @@ lsm_error lsm_entry_attr_remove(lsm_str **out, lsm_entry *entry, uint8_t type) { entry->attrs.count--; entry->attrs.bitmap[type / 64] &= ~(((uint64_t)1) << (type % 64)); + handle->states |= lsm_entry_handle_state_updated; + return lsm_error_ok; } -lsm_error lsm_entry_attr_insert(lsm_entry *entry, uint8_t type, lsm_str *data) { - // Remove a previous version of the attribute - lsm_str *out; - if (lsm_entry_attr_remove(&out, entry, type) == lsm_error_ok) { - lsm_str_free(out); +lsm_error lsm_entry_attr_insert(lsm_entry_handle *handle, uint8_t type, + lsm_str *data) { + if (lsm_entry_attr_present(handle, type)) { + return lsm_error_already_present; } + lsm_entry *entry = handle->wrapper->entry; + lsm_attr *new_attrs = realloc(entry->attrs.items, (entry->attrs.count + 1) * sizeof(lsm_attr)); @@ -194,31 +226,36 @@ lsm_error lsm_entry_attr_insert(lsm_entry *entry, uint8_t type, lsm_str *data) { entry->attrs.count++; entry->attrs.bitmap[type / 64] |= ((uint64_t)1) << (type % 64); + handle->states |= lsm_entry_handle_state_updated; + return lsm_error_ok; } -lsm_error lsm_entry_attr_insert_uint64_t(lsm_entry *entry, uint8_t type, +lsm_error lsm_entry_attr_insert_uint64_t(lsm_entry_handle *handle, uint8_t type, uint64_t data) { lsm_str *s; LSM_RES( lsm_str_init_copy_n(&s, (char *)&data, sizeof(uint64_t) / sizeof(char))); - return lsm_entry_attr_insert(entry, type, s); + return lsm_entry_attr_insert(handle, type, s); } -lsm_error lsm_entry_attr_insert_uint8_t(lsm_entry *entry, uint8_t type, +lsm_error lsm_entry_attr_insert_uint8_t(lsm_entry_handle *handle, uint8_t type, uint8_t data) { lsm_str *s; LSM_RES( lsm_str_init_copy_n(&s, (char *)&data, sizeof(uint8_t) / sizeof(char))); - return lsm_entry_attr_insert(entry, type, s); + return lsm_entry_attr_insert(handle, type, s); } -uint64_t lsm_entry_data_path_len(const lsm_store *store, - const lsm_entry *entry) { - const lsm_str *data_path = store->data_path; - const lsm_str *key = entry->key; +uint64_t lsm_entry_data_len(lsm_entry_handle *handle) { + return handle->wrapper->entry->data_len; +} + +uint64_t lsm_entry_data_path_len(const lsm_entry_handle *handle) { + const lsm_str *data_path = handle->store->data_path; + const lsm_str *key = handle->wrapper->entry->key; uint8_t levels = key->len <= LSM_STORE_DATA_LEVELS ? key->len : LSM_STORE_DATA_LEVELS; @@ -227,10 +264,9 @@ uint64_t lsm_entry_data_path_len(const lsm_store *store, strlen(LSM_DATA_FILE_SUFFIX); } -void lsm_entry_data_path(char *path, const lsm_store *store, - const lsm_entry *entry) { - const lsm_str *data_path = store->data_path; - const lsm_str *key = entry->key; +void lsm_entry_data_path(char *path, const lsm_entry_handle *handle) { + const lsm_str *data_path = handle->store->data_path; + const lsm_str *key = handle->wrapper->entry->key; uint8_t levels = key->len > LSM_STORE_DATA_LEVELS ? LSM_STORE_DATA_LEVELS : key->len; @@ -255,13 +291,12 @@ void lsm_entry_data_path(char *path, const lsm_store *store, strcpy(&path[index], LSM_DATA_FILE_SUFFIX); } -lsm_error lsm_entry_data_mkdirs(const lsm_store *store, - const lsm_entry *entry) { - char path[lsm_entry_data_path_len(store, entry) + 1]; - lsm_entry_data_path(path, store, entry); +lsm_error lsm_entry_data_open_write(lsm_entry_handle *handle) { + char path[lsm_entry_data_path_len(handle) + 1]; + lsm_entry_data_path(path, handle); - const lsm_str *data_path = store->data_path; - const lsm_str *key = entry->key; + const lsm_str *data_path = handle->store->data_path; + const lsm_str *key = handle->wrapper->entry->key; uint8_t levels = key->len <= LSM_STORE_DATA_LEVELS ? key->len : LSM_STORE_DATA_LEVELS; @@ -278,32 +313,43 @@ lsm_error lsm_entry_data_mkdirs(const lsm_store *store, path[data_path->len + 2 * (i + 1)] = '/'; } - return lsm_error_ok; -} - -lsm_error lsm_entry_data_open(FILE **out, const lsm_store *store, - const lsm_entry *entry, const char *mode) { - char path[lsm_entry_data_path_len(store, entry) + 1]; - lsm_entry_data_path(path, store, entry); - - FILE *f = fopen(path, mode); + FILE *f = fopen(path, "ab"); if (f == NULL) { return lsm_error_failed_io; } - if (out != NULL) { - *out = f; - } + handle->f = f; return lsm_error_ok; } -lsm_error lsm_entry_data_remove(const lsm_store *store, - const lsm_entry *entry) { +lsm_error lsm_entry_data_open_read(lsm_entry_handle *handle) { + char path[lsm_entry_data_path_len(handle) + 1]; + lsm_entry_data_path(path, handle); + + FILE *f = fopen(path, "rb"); + + if (f == NULL) { + return lsm_error_failed_io; + } + + handle->f = f; + + return lsm_error_ok; +} + +lsm_error lsm_entry_data_remove(lsm_entry_handle *handle) { + const lsm_entry *entry = handle->wrapper->entry; + if (entry->data_len > 0) { - char data_path[lsm_entry_data_path_len(store, entry) + 1]; - lsm_entry_data_path(data_path, store, entry); + if (handle->f != NULL) { + fclose(handle->f); + handle->f = NULL; + } + + char data_path[lsm_entry_data_path_len(handle) + 1]; + lsm_entry_data_path(data_path, handle); if (remove(data_path) != 0) { return lsm_error_failed_io; diff --git a/src/lander/lander.c b/src/lander/lander.c index 2ebe98e..ff44b5c 100644 --- a/src/lander/lander.c +++ b/src/lander/lander.c @@ -24,15 +24,10 @@ lnm_err lander_ctx_init(void **c_ctx, void *gctx) { } void lander_ctx_reset(lander_ctx *ctx) { - if (ctx->entry.read != NULL) { - if (ctx->write) { - lsm_write_close(ctx->entry.write); - } else { - lsm_read_close(ctx->entry.read); - } + if (ctx->entry != NULL) { + lsm_entry_close(ctx->entry); - ctx->entry.read = NULL; - ctx->write = false; + ctx->entry = NULL; } } @@ -50,23 +45,16 @@ void lander_header_to_attr(lnm_http_loop_ctx *ctx, const char *header_name, lsm_str *value; lsm_str_init_copy_n(&value, (char *)header_value, header_value_len); - lsm_write_attr_insert(c_ctx->entry.write, attr_type, value); + lsm_entry_attr_insert(c_ctx->entry, attr_type, value); } } void lander_attr_to_header(lnm_http_loop_ctx *ctx, lander_attr_type attr_type, lnm_http_header header_type) { lander_ctx *c_ctx = ctx->c; - const lsm_str *value; - lsm_error res; + lsm_str *value; - if (c_ctx->write) { - res = lsm_write_attr_get(&value, c_ctx->entry.write, attr_type); - } else { - res = lsm_read_attr_get(&value, c_ctx->entry.read, attr_type); - } - - if (res == lsm_error_ok) { + if (lsm_entry_attr_get(&value, c_ctx->entry, attr_type) == lsm_error_ok) { lnm_http_res_add_header_len(&ctx->res, header_type, (char *)lsm_str_ptr(value), lsm_str_len(value), false); diff --git a/src/lander/lander_delete.c b/src/lander/lander_delete.c index f634f13..f13326b 100644 --- a/src/lander/lander_delete.c +++ b/src/lander/lander_delete.c @@ -15,9 +15,9 @@ lnm_http_step_err lander_remove_entry(lnm_http_conn *conn) { lsm_str *key; lsm_str_init_copy_n(&key, (char *)key_s, key_len); - switch (lsm_store_open_write(&c_ctx->entry.write, c_gctx->store, key)) { + switch (lsm_store_open_write(&c_ctx->entry, c_gctx->store, key)) { case lsm_error_ok: - lsm_write_remove(c_ctx->entry.write); + lsm_entry_remove(c_ctx->entry); break; case lsm_error_not_found: ctx->res.status = lnm_http_status_not_found; diff --git a/src/lander/lander_get.c b/src/lander/lander_get.c index efaf494..7d4a186 100644 --- a/src/lander/lander_get.c +++ b/src/lander/lander_get.c @@ -36,17 +36,17 @@ lnm_http_step_err lander_get_redirect(lnm_http_conn *conn) { lander_ctx *c_ctx = ctx->c; // For redirects, the URL is stored as an in-memory attribute - const lsm_str *url_attr_val; + lsm_str *url_attr_val; // This shouldn't be able to happen - if (lsm_read_attr_get(&url_attr_val, c_ctx->entry.read, - lander_attr_type_url) != lsm_error_ok) { + if (lsm_entry_attr_get(&url_attr_val, c_ctx->entry, lander_attr_type_url) != + lsm_error_ok) { lnm_lerror("lander", "%s", "Entry of type redirect detected without URL attribute"); ctx->res.status = lnm_http_status_internal_server_error; - lsm_read_close(c_ctx->entry.read); - c_ctx->entry.read = NULL; + lsm_entry_close(c_ctx->entry); + c_ctx->entry = NULL; return lnm_http_step_err_res; } @@ -68,7 +68,7 @@ lnm_err lander_entry_data_streamer(uint64_t *written, char *buf, lnm_http_loop_ctx *ctx = conn->ctx; lander_ctx *c_ctx = ctx->c; - lsm_read_data_read(written, buf, c_ctx->entry.read, len); + lsm_entry_data_read(written, buf, c_ctx->entry, len); return lnm_err_ok; } @@ -78,7 +78,7 @@ lnm_http_step_err lander_get_paste(lnm_http_conn *conn) { lander_ctx *c_ctx = ctx->c; lnm_http_res_body_set_fn(&ctx->res, lander_entry_data_streamer, - lsm_read_data_len(c_ctx->entry.read)); + lsm_entry_data_len(c_ctx->entry)); lnm_http_res_add_header(&ctx->res, lnm_http_header_content_type, "text/plain", false); @@ -90,7 +90,7 @@ lnm_http_step_err lander_get_file(lnm_http_conn *conn) { lander_ctx *c_ctx = ctx->c; lnm_http_res_body_set_fn(&ctx->res, lander_entry_data_streamer, - lsm_read_data_len(c_ctx->entry.read)); + lsm_entry_data_len(c_ctx->entry)); lander_attr_to_header(ctx, lander_attr_type_content_type, lnm_http_header_content_type); @@ -108,8 +108,7 @@ lnm_http_step_err lander_get_entry(lnm_http_conn *conn) { lsm_str *key; lsm_str_init_copy_n(&key, (char *)key_s, key_len); - lsm_error lsm_res = - lsm_store_open_read(&c_ctx->entry.read, c_gctx->store, key); + lsm_error lsm_res = lsm_store_open_read(&c_ctx->entry, c_gctx->store, key); lsm_str_free(key); switch (lsm_res) { @@ -124,8 +123,8 @@ lnm_http_step_err lander_get_entry(lnm_http_conn *conn) { } lander_entry_type t; - lsm_read_attr_get_uint8_t((uint8_t *)&t, c_ctx->entry.read, - lander_attr_type_entry_type); + lsm_entry_attr_get_uint8_t((uint8_t *)&t, c_ctx->entry, + lander_attr_type_entry_type); lnm_http_step_err res; diff --git a/src/lander/lander_post.c b/src/lander/lander_post.c index 1a8c961..7befe03 100644 --- a/src/lander/lander_post.c +++ b/src/lander/lander_post.c @@ -17,8 +17,7 @@ static void randomize_key(char *key, int len) { } /** - * Insert a new entry into the store. If an entry is already open, this function - * is a no-op. + * Insert a new entry into the store. * * @return true on success, false otherwise */ @@ -27,12 +26,6 @@ bool lander_insert_entry(lnm_http_loop_ctx *ctx, bool secure) { lander_gctx *c_gctx = gctx->c; lander_ctx *c_ctx = ctx->c; - // With placeholders, the entry will already be open so an entry should no - // longer be created - if (c_ctx->entry.write != NULL) { - return true; - } - const char *key_s; size_t key_len = lnm_http_req_route_segment(&key_s, &ctx->req, "key"); @@ -50,12 +43,11 @@ bool lander_insert_entry(lnm_http_loop_ctx *ctx, bool secure) { } // TODO free key on error - switch (lsm_store_open_new(&c_ctx->entry.write, c_gctx->store, key)) { + switch (lsm_store_insert(&c_ctx->entry, c_gctx->store, key)) { case lsm_error_already_present: ctx->res.status = lnm_http_status_conflict; return false; case lsm_error_ok: - c_ctx->write = true; break; default: ctx->res.status = lnm_http_status_internal_server_error; @@ -83,7 +75,7 @@ static lnm_http_step_err __lander_post_redirect(lnm_http_conn *conn, return lnm_http_step_err_res; } - lsm_write_attr_insert_uint8_t(c_ctx->entry.write, lander_attr_type_entry_type, + lsm_entry_attr_insert_uint8_t(c_ctx->entry, lander_attr_type_entry_type, lander_entry_type_redirect); return lnm_http_step_err_done; @@ -103,7 +95,7 @@ lnm_http_step_err lander_post_redirect_body_to_attr(lnm_http_conn *conn) { lsm_str *attr_value; lsm_str_init_copy_n(&attr_value, ctx->req.body.buf, ctx->req.body.len); - lsm_write_attr_insert(c_ctx->entry.write, lander_attr_type_url, attr_value); + lsm_entry_attr_insert(c_ctx->entry, lander_attr_type_url, attr_value); return lnm_http_step_err_done; } @@ -116,7 +108,7 @@ static lnm_http_step_err __lander_post_paste(lnm_http_conn *conn, bool secure) { return lnm_http_step_err_res; } - lsm_write_attr_insert_uint8_t(c_ctx->entry.write, lander_attr_type_entry_type, + lsm_entry_attr_insert_uint8_t(c_ctx->entry, lander_attr_type_entry_type, lander_entry_type_paste); lander_header_to_attr(ctx, "X-Lander-Filename", lander_attr_type_file_name); @@ -139,7 +131,7 @@ static lnm_http_step_err __lander_post_file(lnm_http_conn *conn, bool secure) { return lnm_http_step_err_res; } - lsm_write_attr_insert_uint8_t(c_ctx->entry.write, lander_attr_type_entry_type, + lsm_entry_attr_insert_uint8_t(c_ctx->entry, lander_attr_type_entry_type, lander_entry_type_file); lander_header_to_attr(ctx, "X-Lander-Content-Type", lander_attr_type_content_type); @@ -155,25 +147,3 @@ lnm_http_step_err lander_post_file(lnm_http_conn *conn) { lnm_http_step_err lander_post_file_secure(lnm_http_conn *conn) { return __lander_post_file(conn, true); } - -lnm_http_step_err __lander_post_placeholder(lnm_http_conn *conn, bool secure) { - lnm_http_loop_ctx *ctx = conn->ctx; - lander_ctx *c_ctx = ctx->c; - - if (!lander_insert_entry(ctx, secure)) { - return lnm_http_step_err_res; - } - - lsm_write_attr_insert_uint8_t(c_ctx->entry.write, lander_attr_type_entry_type, - lander_entry_type_placeholder); - - return lnm_http_step_err_done; -} - -lnm_http_step_err lander_post_placeholder(lnm_http_conn *conn) { - return __lander_post_placeholder(conn, false); -} - -lnm_http_step_err lander_post_placeholder_secure(lnm_http_conn *conn) { - return __lander_post_placeholder(conn, true); -} diff --git a/src/lander/lander_steps.c b/src/lander/lander_steps.c index 68bed99..8e2d702 100644 --- a/src/lander/lander_steps.c +++ b/src/lander/lander_steps.c @@ -5,25 +5,24 @@ #include "lnm/loop.h" #include "lander.h" -#include "lsm/store.h" lnm_http_step_err lander_stream_body_to_entry(lnm_http_conn *conn) { lnm_http_loop_ctx *ctx = conn->ctx; lander_ctx *c_ctx = ctx->c; - uint64_t to_append = LNM_MIN(conn->r.size - conn->r.read, - ctx->req.body.expected_len - - lsm_write_data_len(c_ctx->entry.write)); + uint64_t to_append = + LNM_MIN(conn->r.size - conn->r.read, + ctx->req.body.expected_len - lsm_entry_data_len(c_ctx->entry)); lsm_str *data; lsm_str_init_copy_n(&data, (char *)&conn->r.buf[conn->r.read], to_append); - lsm_write_data_append(c_ctx->entry.write, data); + lsm_entry_data_append(c_ctx->entry, data); conn->r.read += to_append; lsm_str_free(data); - return lsm_write_data_len(c_ctx->entry.write) == ctx->req.body.expected_len + return lsm_entry_data_len(c_ctx->entry) == ctx->req.body.expected_len ? lnm_http_step_err_done : lnm_http_step_err_io_needed; } @@ -32,55 +31,7 @@ lnm_http_step_err lander_commit_entry(lnm_http_conn *conn) { lnm_http_loop_ctx *ctx = conn->ctx; lander_ctx *c_ctx = ctx->c; - lsm_write_commit(c_ctx->entry.write); - - return lnm_http_step_err_done; -} - -lnm_http_step_err lander_auth_or_placeholder(lnm_http_conn *conn) { - lnm_http_loop_ctx *ctx = conn->ctx; - lnm_http_loop_gctx *gctx = ctx->g; - lander_gctx *c_gctx = gctx->c; - lander_ctx *c_ctx = ctx->c; - - const char *key_s; - size_t key_len = lnm_http_req_route_segment(&key_s, &ctx->req, "key"); - - // Only predefined keys can be placeholders - if (key_len == 0) { - return lnm_http_loop_step_auth(conn); - } - - lsm_str *key; - lsm_str_init_copy_n(&key, key_s, key_len); - - lsm_error res = lsm_store_open_write(&c_ctx->entry.write, c_gctx->store, key); - - lsm_str_free(key); - - switch (res) { - case lsm_error_ok: { - c_ctx->write = true; - - lander_entry_type t; - - // If the entry is a placeholder, the request is always authenticated - if ((lsm_write_attr_get_uint8_t(&t, c_ctx->entry.write, - lander_attr_type_entry_type) == - lsm_error_ok) && - (t == lander_entry_type_placeholder)) { - return lnm_http_step_err_done; - } else { - return lnm_http_loop_step_auth(conn); - } - } break; - case lsm_error_not_found: - return lnm_http_loop_step_auth(conn); - break; - default: - ctx->res.status = lnm_http_status_internal_server_error; - return lnm_http_step_err_res; - } + lsm_entry_commit(c_ctx->entry); return lnm_http_step_err_done; } diff --git a/src/main.c b/src/main.c index 980e9f7..f0f8e65 100644 --- a/src/main.c +++ b/src/main.c @@ -3,7 +3,6 @@ #include #include "lnm/http/loop.h" -#include "lnm/http/route.h" #include "lnm/log.h" #include "lander.h" @@ -31,79 +30,54 @@ lnm_http_loop *loop_init(lander_gctx *gctx, const char *api_key) { lnm_http_router_add(&route, router, lnm_http_method_delete, "/:key"); lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_remove_entry, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/s/"); - lnm_http_route_step_append(route, lander_auth_or_placeholder, false); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_post_redirect, false); lnm_http_route_step_append(route, lnm_http_loop_step_body_to_buf, false); lnm_http_route_step_append(route, lander_post_redirect_body_to_attr, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/sl/"); - lnm_http_route_step_append(route, lander_auth_or_placeholder, false); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_post_redirect_secure, false); lnm_http_route_step_append(route, lnm_http_loop_step_body_to_buf, false); lnm_http_route_step_append(route, lander_post_redirect_body_to_attr, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/s/:key"); - lnm_http_route_step_append(route, lander_auth_or_placeholder, false); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_post_redirect, false); lnm_http_route_step_append(route, lnm_http_loop_step_body_to_buf, false); lnm_http_route_step_append(route, lander_post_redirect_body_to_attr, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/p/"); - lnm_http_route_step_append(route, lander_auth_or_placeholder, false); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_post_paste, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/pl/"); - lnm_http_route_step_append(route, lander_auth_or_placeholder, false); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_post_paste_secure, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/p/:key"); - lnm_http_route_step_append(route, lander_auth_or_placeholder, false); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_post_paste, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/f/"); - lnm_http_route_step_append(route, lander_auth_or_placeholder, false); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_post_file, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/fl/"); - lnm_http_route_step_append(route, lander_auth_or_placeholder, false); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_post_file_secure, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_router_add(&route, router, lnm_http_method_post, "/f/:key"); - lnm_http_route_step_append(route, lander_auth_or_placeholder, false); + lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); lnm_http_route_step_append(route, lander_post_file, false); lnm_http_route_step_append(route, lander_stream_body_to_entry, false); - lnm_http_route_step_append(route, lander_commit_entry, true); - - lnm_http_router_add(&route, router, lnm_http_method_post, "/h/"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); - lnm_http_route_step_append(route, lander_post_placeholder, false); - lnm_http_route_step_append(route, lander_commit_entry, true); - - lnm_http_router_add(&route, router, lnm_http_method_post, "/h/:key"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); - lnm_http_route_step_append(route, lander_post_placeholder, false); - lnm_http_route_step_append(route, lander_commit_entry, true); - - lnm_http_router_add(&route, router, lnm_http_method_post, "/hl/"); - lnm_http_route_step_append(route, lnm_http_loop_step_auth, false); - lnm_http_route_step_append(route, lander_post_placeholder_secure, false); - lnm_http_route_step_append(route, lander_commit_entry, true); lnm_http_loop_router_set(hl, router); @@ -155,5 +129,5 @@ int main() { lnm_linfo("main", "Store loaded containing %lu entries", lsm_store_size(c_gctx->store)); lnm_http_loop *hl = loop_init(c_gctx, api_key); - lnm_http_loop_run(hl, port, 1, 1); + lnm_http_loop_run(hl, port, 1, 0); }