From b4c8216ebcb127a88d712c3e990a7088148f99b0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 5 Apr 2022 09:58:02 +0200 Subject: [PATCH] Added support for optionally deploying to subdir (for docs later) --- .gitignore | 1 + Cargo.lock | 15 +++++ Cargo.toml | 1 + curl | 8 +++ src/api/deploy.rs | 143 ++++++++++++++++++++++++---------------------- src/main.rs | 4 +- 6 files changed, 103 insertions(+), 69 deletions(-) create mode 100755 curl diff --git a/.gitignore b/.gitignore index ac67213..7d13d31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target /data/ +*.tar.gz diff --git a/Cargo.lock b/Cargo.lock index 4e88874..b6af5d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -791,6 +791,20 @@ name = "serde" version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "serde_json" @@ -843,6 +857,7 @@ dependencies = [ "hyper", "metrics", "metrics-exporter-prometheus", + "serde", "serde_json", "tar", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 4ee7af8..241fd20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,4 @@ uuid = { version = "1.0.0-alpha.1", features = ["v4"] } serde_json = "1.0.79" metrics = "0.18.1" metrics-exporter-prometheus = "0.9.0" +serde = { version = "1.0", features = ["derive"] } diff --git a/curl b/curl new file mode 100755 index 0000000..f6ec314 --- /dev/null +++ b/curl @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +curl \ + -XPOST \ + -T test.tar.gz \ + -H 'Authorization: Bearer test' \ + -v \ + http://localhost:3000/api/deploy?dir=docs diff --git a/src/api/deploy.rs b/src/api/deploy.rs index 2fe8198..47be32d 100644 --- a/src/api/deploy.rs +++ b/src/api/deploy.rs @@ -1,19 +1,27 @@ use std::{collections::HashSet, io::ErrorKind, path::Path}; use axum::{ - extract::{BodyStream, Extension}, + extract::{BodyStream, Extension, Query}, http::StatusCode, response::IntoResponse, }; use flate2::read::GzDecoder; use futures_util::TryStreamExt; +use serde::Deserialize; use tar::Archive; use tokio_util::io::StreamReader; -use crate::STATIC_DIR_NAME; +use crate::DEFAULT_STATIC_DIR_NAME; + +#[derive(Deserialize)] +pub struct StaticDirParams +{ + dir: Option, +} pub async fn post_deploy( Extension(data_dir): Extension, + Query(params): Query, res: BodyStream, ) -> impl IntoResponse { @@ -26,71 +34,72 @@ pub async fn post_deploy( let mut file = tokio::fs::File::create(&file_path).await.unwrap(); tokio::io::copy(&mut read, &mut file).await; - // Extract the contents of the tarball synchronously - match tokio::task::spawn_blocking(move || { - let file = match std::fs::File::open(file_path) { - Ok(v) => v, - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR, - }; - let tar = GzDecoder::new(file); - let mut archive = Archive::new(tar); + let mut static_path = Path::new(&data_dir).join(DEFAULT_STATIC_DIR_NAME); - let mut paths = HashSet::new(); - - let entries = match archive.entries() { - Ok(e) => e, - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR, - }; - - // Extract each entry into the output directory - let static_dir = Path::new(&data_dir).join(STATIC_DIR_NAME); - for entry_res in entries { - if let Ok(mut entry) = entry_res { - if let Err(_) = entry.unpack_in(&static_dir) { - return StatusCode::INTERNAL_SERVER_ERROR; - } - - if let Ok(path) = entry.path() { - paths.insert(path.into_owned()); - } - } else { - return StatusCode::INTERNAL_SERVER_ERROR; - } - } - - // Remove any old files that weren't present in new archive - let mut items = vec![]; - - // Start by populating the vec with the initial files - let iter = match static_dir.read_dir() { - Ok(v) => v, - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR, - }; - iter.filter_map(|r| r.ok()) - .for_each(|e| items.push(e.path())); - - // As long as there are still items in the vec, we keep going - while items.len() > 0 { - let item = items.pop().unwrap(); - tracing::debug!("{:?}", item); - - if !paths.contains(item.strip_prefix(&static_dir).unwrap()) { - if item.is_dir() { - std::fs::remove_dir_all(item); - } else { - std::fs::remove_file(item); - } - } else if let Ok(iter) = item.read_dir() { - iter.filter_map(|r| r.ok()) - .for_each(|e| items.push(e.path())); - } - } - - StatusCode::OK - }) - .await - { - Ok(s) => s, - Err(_) => StatusCode::INTERNAL_SERVER_ERROR, + if params.dir.is_some() { + static_path = static_path.join(params.dir.unwrap()); } + + // Make sure the static directory exists + tokio::fs::create_dir_all(&static_path).await; + + let fp_clone = file_path.clone(); + // Extract the contents of the tarball synchronously + let res = + match tokio::task::spawn_blocking(move || process_archive(&fp_clone, &static_path)).await { + Ok(_) => StatusCode::OK, + Err(_) => StatusCode::INTERNAL_SERVER_ERROR, + }; + + // Remove archive file after use + tokio::fs::remove_file(&file_path).await; + + res +} + +fn process_archive(archive_path: &Path, static_dir: &Path) -> Result<(), ()> +{ + let file = std::fs::File::open(archive_path).map_err(|_| ())?; + let tar = GzDecoder::new(file); + let mut archive = Archive::new(tar); + + let mut paths = HashSet::new(); + + let entries = archive.entries().map_err(|_| ())?; + // Extract each entry into the output directory + for entry_res in entries { + let mut entry = entry_res.map_err(|_| ())?; + entry.unpack_in(static_dir).map_err(|_| ())?; + + if let Ok(path) = entry.path() { + paths.insert(path.into_owned()); + } + } + + // Remove any old files that weren't present in new archive + let mut items = vec![]; + + // Start by populating the vec with the initial files + let iter = static_dir.read_dir().map_err(|_| ())?; + iter.filter_map(|r| r.ok()) + .for_each(|e| items.push(e.path())); + + // As long as there are still items in the vec, we keep going + while items.len() > 0 { + let item = items.pop().unwrap(); + tracing::debug!("{:?}", item); + + if !paths.contains(item.strip_prefix(&static_dir).unwrap()) { + if item.is_dir() { + std::fs::remove_dir_all(item); + } else { + std::fs::remove_file(item); + } + } else if let Ok(iter) = item.read_dir() { + iter.filter_map(|r| r.ok()) + .for_each(|e| items.push(e.path())); + } + } + + Ok(()) } diff --git a/src/main.rs b/src/main.rs index 3e74b13..cbc95b9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,7 +14,7 @@ mod api; mod matrix; mod metrics; -const STATIC_DIR_NAME: &str = "static"; +const DEFAULT_STATIC_DIR_NAME: &str = "static"; #[tokio::main] async fn main() @@ -30,7 +30,7 @@ async fn main() // Get required variables from env vars let api_key = std::env::var("API_KEY").expect("No API_KEY was provided."); let data_dir = std::env::var("DATA_DIR").expect("No DATA_DIR was provided."); - let static_dir = format!("{}/{}", data_dir, STATIC_DIR_NAME); + let static_dir = format!("{}/{}", data_dir, DEFAULT_STATIC_DIR_NAME); std::fs::create_dir_all(&static_dir);