From 19d255b98c76c0bfed2cc347302b3c3d865ee45e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 7 Jun 2023 21:15:15 +0200 Subject: [PATCH 01/20] feat: show backup time in message --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/server/process.rs | 15 +++++++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5124e00..0991d89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,7 +10,7 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "alex" -version = "0.1.0" +version = "0.2.0" dependencies = [ "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index 4c0ba69..1de2d06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "alex" -version = "0.1.0" +version = "0.2.0" description = "Wrapper around Minecraft server processes, designed to complement Docker image installations." authors = ["Jef Roosens"] edition = "2021" diff --git a/src/server/process.rs b/src/server/process.rs index a9a9f45..233a678 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -84,6 +84,7 @@ impl ServerProcess { // We wait some time to (hopefully) ensure the save-all call has completed std::thread::sleep(std::time::Duration::from_secs(10)); + let start_time = chrono::offset::Local::now(); let res = self.create_backup_archive(); if res.is_ok() { @@ -93,10 +94,20 @@ impl ServerProcess { // The server's save feature needs to be enabled again even if the archive failed to create self.custom("save-on")?; + let duration = chrono::offset::Local::now() - start_time; + let duration_str = format!( + "{}m{}s", + duration.num_seconds() / 60, + duration.num_seconds() % 60 + ); + if res.is_ok() { - self.custom("say server backed up successfully")?; + self.custom(&format!("say server backed up in {}", duration_str))?; } else { - self.custom("an error occured while backing up the server")?; + self.custom(&format!( + "an error occured after {} while backing up the server", + duration_str + ))?; } res From 90033aa91ecd39476601a9cfcf046fd08e664804 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 9 Jun 2023 09:25:51 +0200 Subject: [PATCH 02/20] refactor: move backup logic to separate module --- .gitignore | 2 +- src/server/backups.rs | 108 ++++++++++++++++++++++++++++++++++++++++++ src/server/mod.rs | 2 + src/server/process.rs | 93 +++++------------------------------- 4 files changed, 123 insertions(+), 82 deletions(-) create mode 100644 src/server/backups.rs diff --git a/.gitignore b/.gitignore index 4259b1b..3695da7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ target/ # testing files *.jar -data/ +data*/ diff --git a/src/server/backups.rs b/src/server/backups.rs new file mode 100644 index 0000000..37e6021 --- /dev/null +++ b/src/server/backups.rs @@ -0,0 +1,108 @@ +use flate2::write::GzEncoder; +use flate2::Compression; +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; + +#[link(name = "c")] +extern "C" { + fn geteuid() -> u32; + fn getegid() -> u32; +} + +static FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; + +pub struct BackupManager { + backup_dir: PathBuf, + config_dir: PathBuf, + world_dir: PathBuf, + max_backups: u64, + start_time: Option>, +} + +impl BackupManager { + pub fn open( + backup_dir: PathBuf, + config_dir: PathBuf, + world_dir: PathBuf, + max_backups: u64, + ) -> Self { + BackupManager { + backup_dir, + config_dir, + world_dir, + max_backups, + start_time: None, + } + } + + pub fn create_archive(&mut self) -> io::Result<()> { + let start_time = chrono::offset::Local::now(); + self.start_time = Some(start_time); + + let filename = format!("{}", start_time.format(FILENAME_FORMAT)); + let path = self.backup_dir.join(filename); + let tar_gz = File::create(path)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut tar = tar::Builder::new(enc); + + tar.append_dir_all("worlds", &self.world_dir)?; + + // Add all files from the config directory that aren't the cache + for entry in self + .config_dir + .read_dir()? + .filter_map(|e| e.ok()) + .filter(|e| e.file_name() != "cache") + { + let tar_path = Path::new("config").join(entry.file_name()); + + if entry.file_type()?.is_dir() { + tar.append_dir_all(tar_path, entry.path())?; + } else { + tar.append_path_with_name(entry.path(), tar_path)?; + } + } + + // TODO re-add this info file in some way + // We add a file to the backup describing for what version it was made + // let info = format!("{} {}", self.type_, self.version); + // let info_bytes = info.as_bytes(); + + // let mut header = tar::Header::new_gnu(); + // header.set_size(info_bytes.len().try_into().unwrap()); + // header.set_mode(0o100644); + // unsafe { + // header.set_gid(getegid().into()); + // header.set_uid(geteuid().into()); + // } + + // tar.append_data(&mut header, "info.txt", info_bytes)?; + + Ok(()) + } + + /// Remove the oldest backups + pub fn remove_old_backups(&mut self) -> std::io::Result<()> { + // The naming format used allows us to sort the backups by name and still get a sorting by + // creation time + let mut backups = self + .backup_dir + .read_dir()? + .filter_map(|res| res.map(|e| e.path()).ok()) + .collect::>(); + backups.sort(); + + let max_backups: usize = self.max_backups.try_into().unwrap(); + + if backups.len() > max_backups { + let excess_backups = backups.len() - max_backups; + + for backup in &backups[0..excess_backups] { + std::fs::remove_file(backup)?; + } + } + + Ok(()) + } +} diff --git a/src/server/mod.rs b/src/server/mod.rs index e3e3131..ed5cb21 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,5 +1,7 @@ +mod backups; mod command; mod process; +pub use backups::BackupManager; pub use command::{ServerCommand, ServerType}; pub use process::ServerProcess; diff --git a/src/server/process.rs b/src/server/process.rs index 233a678..f503c84 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -1,3 +1,4 @@ +use crate::server::BackupManager; use crate::server::ServerType; use flate2::write::GzEncoder; use flate2::Compression; @@ -5,12 +6,6 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Child; -#[link(name = "c")] -extern "C" { - fn geteuid() -> u32; - fn getegid() -> u32; -} - pub struct ServerProcess { type_: ServerType, version: String, @@ -19,6 +14,7 @@ pub struct ServerProcess { backup_dir: PathBuf, max_backups: u64, child: Child, + backups: BackupManager, } impl ServerProcess { @@ -31,6 +27,13 @@ impl ServerProcess { max_backups: u64, child: Child, ) -> ServerProcess { + let backup_manager = BackupManager::open( + backup_dir.clone(), + config_dir.clone(), + world_dir.clone(), + max_backups, + ); + ServerProcess { type_, version, @@ -39,6 +42,7 @@ impl ServerProcess { backup_dir, max_backups, child, + backups: backup_manager, } } @@ -85,10 +89,10 @@ impl ServerProcess { std::thread::sleep(std::time::Duration::from_secs(10)); let start_time = chrono::offset::Local::now(); - let res = self.create_backup_archive(); + let res = self.backups.create_archive(); if res.is_ok() { - self.remove_old_backups()?; + self.backups.remove_old_backups()?; } // The server's save feature needs to be enabled again even if the archive failed to create @@ -112,77 +116,4 @@ impl ServerProcess { res } - - /// Create a new compressed backup archive of the server's data. - fn create_backup_archive(&mut self) -> std::io::Result<()> { - // Create a gzip-compressed tarball of the worlds folder - let filename = format!( - "{}", - chrono::offset::Local::now().format("%Y-%m-%d_%H-%M-%S.tar.gz") - ); - let path = self.backup_dir.join(filename); - let tar_gz = std::fs::File::create(path)?; - let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut tar = tar::Builder::new(enc); - - tar.append_dir_all("worlds", &self.world_dir)?; - - // Add all files from the config directory that aren't the cache - for entry in self - .config_dir - .read_dir()? - .filter_map(|e| e.ok()) - .filter(|e| e.file_name() != "cache") - { - let tar_path = Path::new("config").join(entry.file_name()); - - if entry.file_type()?.is_dir() { - tar.append_dir_all(tar_path, entry.path())?; - } else { - tar.append_path_with_name(entry.path(), tar_path)?; - } - } - - // We add a file to the backup describing for what version it was made - let info = format!("{} {}", self.type_, self.version); - let info_bytes = info.as_bytes(); - - let mut header = tar::Header::new_gnu(); - header.set_size(info_bytes.len().try_into().unwrap()); - header.set_mode(0o100644); - unsafe { - header.set_gid(getegid().into()); - header.set_uid(geteuid().into()); - } - - tar.append_data(&mut header, "info.txt", info_bytes)?; - - // Backup file gets finalized in the drop - - Ok(()) - } - - /// Remove the oldest backups - fn remove_old_backups(&mut self) -> std::io::Result<()> { - // The naming format used allows us to sort the backups by name and still get a sorting by - // creation time - let mut backups = self - .backup_dir - .read_dir()? - .filter_map(|res| res.map(|e| e.path()).ok()) - .collect::>(); - backups.sort(); - - let max_backups: usize = self.max_backups.try_into().unwrap(); - - if backups.len() > max_backups { - let excess_backups = backups.len() - max_backups; - - for backup in &backups[0..excess_backups] { - std::fs::remove_file(backup)?; - } - } - - Ok(()) - } } From 3e0324703d5b4d666ff1ca2a3b03d250191b6de0 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 9 Jun 2023 10:11:02 +0200 Subject: [PATCH 03/20] feat: implement own listing of files --- src/server/backups.rs | 50 ++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index 37e6021..e5fd4de 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -18,6 +18,7 @@ pub struct BackupManager { world_dir: PathBuf, max_backups: u64, start_time: Option>, + files: Vec<(PathBuf, PathBuf)> } impl BackupManager { @@ -33,9 +34,40 @@ impl BackupManager { world_dir, max_backups, start_time: None, + files: Vec::new() } } + fn set_files_to_backup(&mut self) -> io::Result<()> { + let mut dirs = vec![ + (PathBuf::from("worlds"), self.world_dir.clone()), + (PathBuf::from("config"), self.config_dir.clone()), + ]; + self.files.clear(); + + while let Some((path_in_tar, path)) = dirs.pop() { + for res in path.read_dir()? { + let entry = res?; + + if entry.file_name() == "cache" { + continue; + } + + let new_path_in_tar = path_in_tar.join(entry.file_name()); + + // All dirs get expanded recursively, while all files get returned as output + // NOTE: does this remove empty directories from backups? Is this a problem? + if entry.file_type()?.is_dir() { + dirs.push((new_path_in_tar, entry.path())); + } else { + self.files.push((new_path_in_tar, entry.path())); + } + } + } + + Ok(()) + } + pub fn create_archive(&mut self) -> io::Result<()> { let start_time = chrono::offset::Local::now(); self.start_time = Some(start_time); @@ -46,22 +78,10 @@ impl BackupManager { let enc = GzEncoder::new(tar_gz, Compression::default()); let mut tar = tar::Builder::new(enc); - tar.append_dir_all("worlds", &self.world_dir)?; + self.set_files_to_backup()?; - // Add all files from the config directory that aren't the cache - for entry in self - .config_dir - .read_dir()? - .filter_map(|e| e.ok()) - .filter(|e| e.file_name() != "cache") - { - let tar_path = Path::new("config").join(entry.file_name()); - - if entry.file_type()?.is_dir() { - tar.append_dir_all(tar_path, entry.path())?; - } else { - tar.append_path_with_name(entry.path(), tar_path)?; - } + for (path_in_tar, path) in &self.files { + tar.append_path_with_name(path, path_in_tar)?; } // TODO re-add this info file in some way From b1c0bbb3af574d6a2468be8f28883856d818cc85 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 9 Jun 2023 10:42:17 +0200 Subject: [PATCH 04/20] refactor: use utc time --- src/server/backups.rs | 4 ++-- src/server/process.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index e5fd4de..d567f42 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -17,7 +17,7 @@ pub struct BackupManager { config_dir: PathBuf, world_dir: PathBuf, max_backups: u64, - start_time: Option>, + start_time: Option>, files: Vec<(PathBuf, PathBuf)> } @@ -69,7 +69,7 @@ impl BackupManager { } pub fn create_archive(&mut self) -> io::Result<()> { - let start_time = chrono::offset::Local::now(); + let start_time = chrono::offset::Utc::now(); self.start_time = Some(start_time); let filename = format!("{}", start_time.format(FILENAME_FORMAT)); diff --git a/src/server/process.rs b/src/server/process.rs index f503c84..7555aa3 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -88,7 +88,7 @@ impl ServerProcess { // We wait some time to (hopefully) ensure the save-all call has completed std::thread::sleep(std::time::Duration::from_secs(10)); - let start_time = chrono::offset::Local::now(); + let start_time = chrono::offset::Utc::now(); let res = self.backups.create_archive(); if res.is_ok() { @@ -98,7 +98,7 @@ impl ServerProcess { // The server's save feature needs to be enabled again even if the archive failed to create self.custom("save-on")?; - let duration = chrono::offset::Local::now() - start_time; + let duration = chrono::offset::Utc::now() - start_time; let duration_str = format!( "{}m{}s", duration.num_seconds() / 60, From 45d736d1bb515879ef7049d4bc51541775300d14 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Jun 2023 11:40:18 +0200 Subject: [PATCH 05/20] chore: bump version --- CHANGELOG.md | 2 ++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4339669..3160ee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/alex/src/branch/dev) +## [0.2.0](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.0) + ### Added * Rudimentary signal handling for gently stopping server diff --git a/Cargo.lock b/Cargo.lock index 5124e00..0991d89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,7 +10,7 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "alex" -version = "0.1.0" +version = "0.2.0" dependencies = [ "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index 4c0ba69..1de2d06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "alex" -version = "0.1.0" +version = "0.2.0" description = "Wrapper around Minecraft server processes, designed to complement Docker image installations." authors = ["Jef Roosens"] edition = "2021" From acb3cfd8e6c172a2f4201d051bd222e99e4c8130 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Jun 2023 11:51:18 +0200 Subject: [PATCH 06/20] chore: update readme --- README.md | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f6215c3..4e00b78 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,26 @@ -# mc-wrapper +# Alex -A wrapper around a standard Minecraft server, written in Rust. \ No newline at end of file +Alex is a wrapper around a typical Minecraft server process. It acts as the +parent process, and sits in between the user's input and the server's stdin. +This allows Alex to support additional commands that execute Rust code. + +## Why + +The primary usecase for this is backups. A common problem I've had with +Minecraft backups is that they fail, because the server is writing to one of +the region files as the backup is being created. Alex solves this be sending +`save-off` and `save-all` to the server, before creating the tarball. +Afterwards, saving is enabled again with `save-on`. + +## Features + +* Create safe backups as gzip-compressed tarballs using the `backup` command +* Automatically create backups periodically +* Properly configures the process (working directory, optimisation flags) +* Configure everything as CLI arguments or environment variables + +## Installation + +Alex is distributed as statically compiled binaries for Linux amd64 and arm64. +These can be found +[here](https://git.rustybever.be/Chewing_Bever/alex/packages). From b08ba3853fa3ad1bc98e197e269599e8f2861e76 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Jun 2023 12:49:31 +0200 Subject: [PATCH 07/20] feat: add --dry flag --- CHANGELOG.md | 4 +++ Dockerfile | 2 +- src/main.rs | 15 ++++++++++- src/server/command.rs | 63 ++++++++++++++++++++++++++++++++++--------- 4 files changed, 69 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3160ee1..9124e7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/alex/src/branch/dev) +### Added + +* `--dry` flag to inspect command that will be run + ## [0.2.0](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.0) ### Added diff --git a/Dockerfile b/Dockerfile index cb3b9e0..d754728 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN cargo build && \ # We use ${:-} instead of a default value because the argument is always passed # to the build, it'll just be blank most likely -FROM eclipse-temurin:17-jre-alpine +FROM eclipse-temurin:18-jre-alpine # Build arguments ARG MC_VERSION=1.19.4 diff --git a/src/main.rs b/src/main.rs index 9f993a9..10500b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -66,6 +66,11 @@ struct Cli { /// How frequently to perform a backup, in minutes; 0 to disable. #[arg(short = 't', long, default_value_t = 0, env = "ALEX_FREQUENCY")] frequency: u64, + + /// Don't actually run the server, but simply output the server configuration that would have + /// been ran + #[arg(short, long, default_value_t = false)] + dry: bool, } fn backups_thread(counter: Arc>, frequency: u64) { @@ -85,7 +90,7 @@ fn main() -> io::Result<()> { let (_, mut signals) = signals::install_signal_handlers()?; let cli = Cli::parse(); - let cmd = server::ServerCommand::new(cli.type_, &cli.server_version) + let mut cmd = server::ServerCommand::new(cli.type_, &cli.server_version) .java(&cli.java) .jar(cli.jar) .config(cli.config) @@ -94,6 +99,14 @@ fn main() -> io::Result<()> { .xms(cli.xms) .xmx(cli.xmx) .max_backups(cli.max_backups); + cmd.canonicalize()?; + + if cli.dry { + print!("{}", cmd); + + return Ok(()); + } + let counter = Arc::new(Mutex::new(cmd.spawn()?)); if cli.frequency > 0 { diff --git a/src/server/command.rs b/src/server/command.rs index 7b6d948..f410c88 100644 --- a/src/server/command.rs +++ b/src/server/command.rs @@ -105,14 +105,19 @@ impl ServerCommand { Ok(()) } - pub fn spawn(self) -> std::io::Result { + /// Canonicalize all paths to absolute paths. Without this command, all paths will be + /// interpreted relatively from the config directory. + pub fn canonicalize(&mut self) -> std::io::Result<()> { // To avoid any issues, we use absolute paths for everything when spawning the process - let jar = self.jar.canonicalize()?; - let config_dir = self.config_dir.canonicalize()?; - let world_dir = self.world_dir.canonicalize()?; - let backup_dir = self.backup_dir.canonicalize()?; + self.jar = self.jar.canonicalize()?; + self.config_dir = self.config_dir.canonicalize()?; + self.world_dir = self.world_dir.canonicalize()?; + self.backup_dir = self.backup_dir.canonicalize()?; - self.accept_eula()?; + Ok(()) + } + + fn create_cmd(&self) -> std::process::Command { let mut cmd = Command::new(&self.java); // Apply JVM optimisation flags @@ -155,24 +160,56 @@ impl ServerCommand { ]); } - cmd.current_dir(&config_dir) + cmd.current_dir(&self.config_dir) .arg("-jar") - .arg(&jar) + .arg(&self.jar) .arg("--universe") - .arg(&world_dir) + .arg(&self.world_dir) .arg("--nogui") .stdin(Stdio::piped()); + cmd + } + + pub fn spawn(&mut self) -> std::io::Result { + let mut cmd = self.create_cmd(); + self.accept_eula()?; let child = cmd.spawn()?; Ok(ServerProcess::new( self.type_, - self.version, - config_dir, - world_dir, - backup_dir, + self.version.clone(), + self.config_dir.clone(), + self.world_dir.clone(), + self.backup_dir.clone(), self.max_backups, child, )) } } + +impl fmt::Display for ServerCommand { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let cmd = self.create_cmd(); + + write!(f, "Command: {}\n", self.java)?; + write!(f, "Working dir: {}\n", self.config_dir.as_path().display())?; + + // Print command env vars + write!(f, "Environment:\n")?; + + for (key, val) in cmd.get_envs().filter(|(_, v)| v.is_some()) { + let val = val.unwrap(); + write!(f, " {}={}\n", key.to_string_lossy(), val.to_string_lossy())?; + } + + // Print command arguments + write!(f, "Arguments:\n")?; + + for arg in cmd.get_args() { + write!(f, " {}\n", arg.to_string_lossy())?; + } + + Ok(()) + } +} From 5ae23c931a607e4f48f9eb1230656e6ad9b98a85 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Jun 2023 13:00:42 +0200 Subject: [PATCH 08/20] feat: change jvm flags order --- CHANGELOG.md | 4 ++++ src/server/command.rs | 31 +++++++++++++++++++------------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9124e7c..2c02234 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `--dry` flag to inspect command that will be run +### Changed + +* JVM flags now narrowely follow Aikar's specifications + ## [0.2.0](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.0) ### Added diff --git a/src/server/command.rs b/src/server/command.rs index f410c88..e725a38 100644 --- a/src/server/command.rs +++ b/src/server/command.rs @@ -131,15 +131,6 @@ impl ServerCommand { "-XX:+UnlockExperimentalVMOptions", "-XX:+DisableExplicitGC", "-XX:+AlwaysPreTouch", - "-XX:G1HeapWastePercent=5", - "-XX:G1MixedGCCountTarget=4", - "-XX:G1MixedGCLiveThresholdPercent=90", - "-XX:G1RSetUpdatingPauseTimePercent=5", - "-XX:SurvivorRatio=32", - "-XX:+PerfDisableSharedMem", - "-XX:MaxTenuringThreshold=1", - "-Dusing.aikars.flags=https://mcflags.emc.gs", - "-Daikars.new.flags=true", ]); if self.xms > 12 * 1024 { @@ -148,18 +139,34 @@ impl ServerCommand { "-XX:G1MaxNewSizePercent=50", "-XX:G1HeapRegionSize=16M", "-XX:G1ReservePercent=15", - "-XX:InitiatingHeapOccupancyPercent=20", ]); } else { cmd.args([ "-XX:G1NewSizePercent=30", "-XX:G1MaxNewSizePercent=40", "-XX:G1HeapRegionSize=8M", - "-XX:G1ReservePercent=15", - "-XX:InitiatingHeapOccupancyPercent=15", + "-XX:G1ReservePercent=20", ]); } + cmd.args(["-XX:G1HeapWastePercent=5", "-XX:G1MixedGCCountTarget=4"]); + + if self.xms > 12 * 1024 { + cmd.args(["-XX:InitiatingHeapOccupancyPercent=20"]); + } else { + cmd.args(["-XX:InitiatingHeapOccupancyPercent=15"]); + } + + cmd.args([ + "-XX:G1MixedGCLiveThresholdPercent=90", + "-XX:G1RSetUpdatingPauseTimePercent=5", + "-XX:SurvivorRatio=32", + "-XX:+PerfDisableSharedMem", + "-XX:MaxTenuringThreshold=1", + "-Dusing.aikars.flags=https://mcflags.emc.gs", + "-Daikars.new.flags=true", + ]); + cmd.current_dir(&self.config_dir) .arg("-jar") .arg(&self.jar) From ce3dcdd4b1e49d64938e9154457cc8a8f96845e7 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Jun 2023 13:01:47 +0200 Subject: [PATCH 09/20] chore: please clippy --- src/server/command.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/server/command.rs b/src/server/command.rs index e725a38..641c6b8 100644 --- a/src/server/command.rs +++ b/src/server/command.rs @@ -199,22 +199,22 @@ impl fmt::Display for ServerCommand { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let cmd = self.create_cmd(); - write!(f, "Command: {}\n", self.java)?; - write!(f, "Working dir: {}\n", self.config_dir.as_path().display())?; + writeln!(f, "Command: {}", self.java)?; + writeln!(f, "Working dir: {}", self.config_dir.as_path().display())?; // Print command env vars - write!(f, "Environment:\n")?; + writeln!(f, "Environment:")?; for (key, val) in cmd.get_envs().filter(|(_, v)| v.is_some()) { let val = val.unwrap(); - write!(f, " {}={}\n", key.to_string_lossy(), val.to_string_lossy())?; + writeln!(f, " {}={}", key.to_string_lossy(), val.to_string_lossy())?; } // Print command arguments - write!(f, "Arguments:\n")?; + writeln!(f, "Arguments:")?; for arg in cmd.get_args() { - write!(f, " {}\n", arg.to_string_lossy())?; + writeln!(f, " {}", arg.to_string_lossy())?; } Ok(()) From 375a68fbd6890cce15e94965944f655fd1bb878b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Jun 2023 13:02:27 +0200 Subject: [PATCH 10/20] chore: bump versions --- CHANGELOG.md | 2 ++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c02234..75eed18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/alex/src/branch/dev) +## [0.2.1](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.1) + ### Added * `--dry` flag to inspect command that will be run diff --git a/Cargo.lock b/Cargo.lock index 0991d89..e802625 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,7 +10,7 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "alex" -version = "0.2.0" +version = "0.2.1" dependencies = [ "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index 1de2d06..43e1246 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "alex" -version = "0.2.0" +version = "0.2.1" description = "Wrapper around Minecraft server processes, designed to complement Docker image installations." authors = ["Jef Roosens"] edition = "2021" From 9ce8199d5fabffc4b652b4f62ee8d753738631d9 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 13 Jun 2023 13:44:08 +0200 Subject: [PATCH 11/20] fix: use correct env var for backup dir --- CHANGELOG.md | 6 ++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- Dockerfile | 5 +++-- src/main.rs | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75eed18..7b9570a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/alex/src/branch/dev) +## [0.2.2](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.2) + +### Fixed + +* Use correct env var for backup directory + ## [0.2.1](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.1) ### Added diff --git a/Cargo.lock b/Cargo.lock index e802625..3e798dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,7 +10,7 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "alex" -version = "0.2.1" +version = "0.2.2" dependencies = [ "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index 43e1246..2c4045b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "alex" -version = "0.2.1" +version = "0.2.2" description = "Wrapper around Minecraft server processes, designed to complement Docker image installations." authors = ["Jef Roosens"] edition = "2021" diff --git a/Dockerfile b/Dockerfile index d754728..dba2674 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ FROM eclipse-temurin:18-jre-alpine # Build arguments ARG MC_VERSION=1.19.4 -ARG PAPERMC_VERSION=545 +ARG PAPERMC_VERSION=525 RUN addgroup -Sg 1000 paper && \ adduser -SHG paper -u 1000 paper @@ -61,4 +61,5 @@ EXPOSE 25565 # Switch to non-root user USER paper:paper -ENTRYPOINT ["/bin/alex", "paper"] +ENTRYPOINT ["/bin/dumb-init", "--"] +CMD ["/bin/alex", "paper"] diff --git a/src/main.rs b/src/main.rs index 10500b3..2cd84ff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,7 +46,7 @@ struct Cli { long, value_name = "BACKUP_DIR", default_value = "../backups", - env = "ALEX_WORLD_DIR" + env = "ALEX_BACKUP_DIR" )] backup: PathBuf, /// Java command to run the server jar with From 42c7a7cc5b30a81a61b2202916f5152ba5b8d15b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Jun 2023 15:09:07 +0200 Subject: [PATCH 12/20] wip --- .cargo/config.toml | 4 ++-- src/server/backups.rs | 55 +++++++++++++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index d1675c8..37bb90a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,3 @@ [alias] -runs = "run -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper.jar" -runrs = "run --release -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper.jar" +runs = "run -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" +runrs = "run --release -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" diff --git a/src/server/backups.rs b/src/server/backups.rs index d567f42..b117e22 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -3,6 +3,8 @@ use flate2::Compression; use std::fs::File; use std::io; use std::path::{Path, PathBuf}; +use chrono::{Utc, Local}; +use std::collections::HashSet; #[link(name = "c")] extern "C" { @@ -17,8 +19,10 @@ pub struct BackupManager { config_dir: PathBuf, world_dir: PathBuf, max_backups: u64, - start_time: Option>, - files: Vec<(PathBuf, PathBuf)> + /// Start time of the last successful backup + last_start_time: Option>, + /// Files contained in the last successful backup + last_files: HashSet<(PathBuf, PathBuf)> } impl BackupManager { @@ -33,17 +37,17 @@ impl BackupManager { config_dir, world_dir, max_backups, - start_time: None, - files: Vec::new() + last_start_time: None, + last_files: HashSet::new() } } - fn set_files_to_backup(&mut self) -> io::Result<()> { + fn files_to_backup(&mut self) -> io::Result> { let mut dirs = vec![ (PathBuf::from("worlds"), self.world_dir.clone()), (PathBuf::from("config"), self.config_dir.clone()), ]; - self.files.clear(); + let mut files: HashSet<(PathBuf, PathBuf)> = HashSet::new(); while let Some((path_in_tar, path)) = dirs.pop() { for res in path.read_dir()? { @@ -60,28 +64,49 @@ impl BackupManager { if entry.file_type()?.is_dir() { dirs.push((new_path_in_tar, entry.path())); } else { - self.files.push((new_path_in_tar, entry.path())); + // Only add files that have been updated since the last backup (incremental backup) + if let Some(last_start_time) = self.last_start_time { + let last_modified = entry.path().metadata()?.modified(); + + if let Ok(last_modified) = last_modified { + let t: chrono::DateTime = last_modified.into(); + let t = t.with_timezone(&Local); + + if t < last_start_time { + continue + } + } + } + + files.insert((new_path_in_tar, entry.path())); } } } - Ok(()) + Ok(files) } pub fn create_archive(&mut self) -> io::Result<()> { let start_time = chrono::offset::Utc::now(); - self.start_time = Some(start_time); let filename = format!("{}", start_time.format(FILENAME_FORMAT)); let path = self.backup_dir.join(filename); let tar_gz = File::create(path)?; let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut tar = tar::Builder::new(enc); + let mut ar = tar::Builder::new(enc); - self.set_files_to_backup()?; + let files = self.files_to_backup()?; - for (path_in_tar, path) in &self.files { - tar.append_path_with_name(path, path_in_tar)?; + for (path_in_tar, path) in &files { + ar.append_path_with_name(path, path_in_tar)?; + } + + let deleted_files = self.last_files.difference(&files); + + println!("{} {}", files.len(), self.last_files.len()); + + for (path_in_tar, path) in deleted_files { + println!("{path_in_tar:?}: {path:?}"); } // TODO re-add this info file in some way @@ -98,6 +123,10 @@ impl BackupManager { // } // tar.append_data(&mut header, "info.txt", info_bytes)?; + + // After a successful backup, we store the original metadata + self.last_start_time = Some(start_time); + self.last_files = files; Ok(()) } From 90aa929b738b462aa2ebbb6b47da02830b7d58f6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 7 Jun 2023 21:15:15 +0200 Subject: [PATCH 13/20] feat: show backup time in message --- src/server/process.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/server/process.rs b/src/server/process.rs index a9a9f45..233a678 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -84,6 +84,7 @@ impl ServerProcess { // We wait some time to (hopefully) ensure the save-all call has completed std::thread::sleep(std::time::Duration::from_secs(10)); + let start_time = chrono::offset::Local::now(); let res = self.create_backup_archive(); if res.is_ok() { @@ -93,10 +94,20 @@ impl ServerProcess { // The server's save feature needs to be enabled again even if the archive failed to create self.custom("save-on")?; + let duration = chrono::offset::Local::now() - start_time; + let duration_str = format!( + "{}m{}s", + duration.num_seconds() / 60, + duration.num_seconds() % 60 + ); + if res.is_ok() { - self.custom("say server backed up successfully")?; + self.custom(&format!("say server backed up in {}", duration_str))?; } else { - self.custom("an error occured while backing up the server")?; + self.custom(&format!( + "an error occured after {} while backing up the server", + duration_str + ))?; } res From 4958257f6e28c179d4ac81e1c23d037093438fda Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 9 Jun 2023 09:25:51 +0200 Subject: [PATCH 14/20] refactor: move backup logic to separate module --- .gitignore | 2 +- src/server/backups.rs | 108 ++++++++++++++++++++++++++++++++++++++++++ src/server/mod.rs | 2 + src/server/process.rs | 93 +++++------------------------------- 4 files changed, 123 insertions(+), 82 deletions(-) create mode 100644 src/server/backups.rs diff --git a/.gitignore b/.gitignore index 4259b1b..3695da7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ target/ # testing files *.jar -data/ +data*/ diff --git a/src/server/backups.rs b/src/server/backups.rs new file mode 100644 index 0000000..37e6021 --- /dev/null +++ b/src/server/backups.rs @@ -0,0 +1,108 @@ +use flate2::write::GzEncoder; +use flate2::Compression; +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; + +#[link(name = "c")] +extern "C" { + fn geteuid() -> u32; + fn getegid() -> u32; +} + +static FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; + +pub struct BackupManager { + backup_dir: PathBuf, + config_dir: PathBuf, + world_dir: PathBuf, + max_backups: u64, + start_time: Option>, +} + +impl BackupManager { + pub fn open( + backup_dir: PathBuf, + config_dir: PathBuf, + world_dir: PathBuf, + max_backups: u64, + ) -> Self { + BackupManager { + backup_dir, + config_dir, + world_dir, + max_backups, + start_time: None, + } + } + + pub fn create_archive(&mut self) -> io::Result<()> { + let start_time = chrono::offset::Local::now(); + self.start_time = Some(start_time); + + let filename = format!("{}", start_time.format(FILENAME_FORMAT)); + let path = self.backup_dir.join(filename); + let tar_gz = File::create(path)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut tar = tar::Builder::new(enc); + + tar.append_dir_all("worlds", &self.world_dir)?; + + // Add all files from the config directory that aren't the cache + for entry in self + .config_dir + .read_dir()? + .filter_map(|e| e.ok()) + .filter(|e| e.file_name() != "cache") + { + let tar_path = Path::new("config").join(entry.file_name()); + + if entry.file_type()?.is_dir() { + tar.append_dir_all(tar_path, entry.path())?; + } else { + tar.append_path_with_name(entry.path(), tar_path)?; + } + } + + // TODO re-add this info file in some way + // We add a file to the backup describing for what version it was made + // let info = format!("{} {}", self.type_, self.version); + // let info_bytes = info.as_bytes(); + + // let mut header = tar::Header::new_gnu(); + // header.set_size(info_bytes.len().try_into().unwrap()); + // header.set_mode(0o100644); + // unsafe { + // header.set_gid(getegid().into()); + // header.set_uid(geteuid().into()); + // } + + // tar.append_data(&mut header, "info.txt", info_bytes)?; + + Ok(()) + } + + /// Remove the oldest backups + pub fn remove_old_backups(&mut self) -> std::io::Result<()> { + // The naming format used allows us to sort the backups by name and still get a sorting by + // creation time + let mut backups = self + .backup_dir + .read_dir()? + .filter_map(|res| res.map(|e| e.path()).ok()) + .collect::>(); + backups.sort(); + + let max_backups: usize = self.max_backups.try_into().unwrap(); + + if backups.len() > max_backups { + let excess_backups = backups.len() - max_backups; + + for backup in &backups[0..excess_backups] { + std::fs::remove_file(backup)?; + } + } + + Ok(()) + } +} diff --git a/src/server/mod.rs b/src/server/mod.rs index e3e3131..ed5cb21 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,5 +1,7 @@ +mod backups; mod command; mod process; +pub use backups::BackupManager; pub use command::{ServerCommand, ServerType}; pub use process::ServerProcess; diff --git a/src/server/process.rs b/src/server/process.rs index 233a678..f503c84 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -1,3 +1,4 @@ +use crate::server::BackupManager; use crate::server::ServerType; use flate2::write::GzEncoder; use flate2::Compression; @@ -5,12 +6,6 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Child; -#[link(name = "c")] -extern "C" { - fn geteuid() -> u32; - fn getegid() -> u32; -} - pub struct ServerProcess { type_: ServerType, version: String, @@ -19,6 +14,7 @@ pub struct ServerProcess { backup_dir: PathBuf, max_backups: u64, child: Child, + backups: BackupManager, } impl ServerProcess { @@ -31,6 +27,13 @@ impl ServerProcess { max_backups: u64, child: Child, ) -> ServerProcess { + let backup_manager = BackupManager::open( + backup_dir.clone(), + config_dir.clone(), + world_dir.clone(), + max_backups, + ); + ServerProcess { type_, version, @@ -39,6 +42,7 @@ impl ServerProcess { backup_dir, max_backups, child, + backups: backup_manager, } } @@ -85,10 +89,10 @@ impl ServerProcess { std::thread::sleep(std::time::Duration::from_secs(10)); let start_time = chrono::offset::Local::now(); - let res = self.create_backup_archive(); + let res = self.backups.create_archive(); if res.is_ok() { - self.remove_old_backups()?; + self.backups.remove_old_backups()?; } // The server's save feature needs to be enabled again even if the archive failed to create @@ -112,77 +116,4 @@ impl ServerProcess { res } - - /// Create a new compressed backup archive of the server's data. - fn create_backup_archive(&mut self) -> std::io::Result<()> { - // Create a gzip-compressed tarball of the worlds folder - let filename = format!( - "{}", - chrono::offset::Local::now().format("%Y-%m-%d_%H-%M-%S.tar.gz") - ); - let path = self.backup_dir.join(filename); - let tar_gz = std::fs::File::create(path)?; - let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut tar = tar::Builder::new(enc); - - tar.append_dir_all("worlds", &self.world_dir)?; - - // Add all files from the config directory that aren't the cache - for entry in self - .config_dir - .read_dir()? - .filter_map(|e| e.ok()) - .filter(|e| e.file_name() != "cache") - { - let tar_path = Path::new("config").join(entry.file_name()); - - if entry.file_type()?.is_dir() { - tar.append_dir_all(tar_path, entry.path())?; - } else { - tar.append_path_with_name(entry.path(), tar_path)?; - } - } - - // We add a file to the backup describing for what version it was made - let info = format!("{} {}", self.type_, self.version); - let info_bytes = info.as_bytes(); - - let mut header = tar::Header::new_gnu(); - header.set_size(info_bytes.len().try_into().unwrap()); - header.set_mode(0o100644); - unsafe { - header.set_gid(getegid().into()); - header.set_uid(geteuid().into()); - } - - tar.append_data(&mut header, "info.txt", info_bytes)?; - - // Backup file gets finalized in the drop - - Ok(()) - } - - /// Remove the oldest backups - fn remove_old_backups(&mut self) -> std::io::Result<()> { - // The naming format used allows us to sort the backups by name and still get a sorting by - // creation time - let mut backups = self - .backup_dir - .read_dir()? - .filter_map(|res| res.map(|e| e.path()).ok()) - .collect::>(); - backups.sort(); - - let max_backups: usize = self.max_backups.try_into().unwrap(); - - if backups.len() > max_backups { - let excess_backups = backups.len() - max_backups; - - for backup in &backups[0..excess_backups] { - std::fs::remove_file(backup)?; - } - } - - Ok(()) - } } From 29d6713486bae0bd38d8bbebe533c02fa93f8fbd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 9 Jun 2023 10:11:02 +0200 Subject: [PATCH 15/20] feat: implement own listing of files --- src/server/backups.rs | 50 ++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index 37e6021..e5fd4de 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -18,6 +18,7 @@ pub struct BackupManager { world_dir: PathBuf, max_backups: u64, start_time: Option>, + files: Vec<(PathBuf, PathBuf)> } impl BackupManager { @@ -33,9 +34,40 @@ impl BackupManager { world_dir, max_backups, start_time: None, + files: Vec::new() } } + fn set_files_to_backup(&mut self) -> io::Result<()> { + let mut dirs = vec![ + (PathBuf::from("worlds"), self.world_dir.clone()), + (PathBuf::from("config"), self.config_dir.clone()), + ]; + self.files.clear(); + + while let Some((path_in_tar, path)) = dirs.pop() { + for res in path.read_dir()? { + let entry = res?; + + if entry.file_name() == "cache" { + continue; + } + + let new_path_in_tar = path_in_tar.join(entry.file_name()); + + // All dirs get expanded recursively, while all files get returned as output + // NOTE: does this remove empty directories from backups? Is this a problem? + if entry.file_type()?.is_dir() { + dirs.push((new_path_in_tar, entry.path())); + } else { + self.files.push((new_path_in_tar, entry.path())); + } + } + } + + Ok(()) + } + pub fn create_archive(&mut self) -> io::Result<()> { let start_time = chrono::offset::Local::now(); self.start_time = Some(start_time); @@ -46,22 +78,10 @@ impl BackupManager { let enc = GzEncoder::new(tar_gz, Compression::default()); let mut tar = tar::Builder::new(enc); - tar.append_dir_all("worlds", &self.world_dir)?; + self.set_files_to_backup()?; - // Add all files from the config directory that aren't the cache - for entry in self - .config_dir - .read_dir()? - .filter_map(|e| e.ok()) - .filter(|e| e.file_name() != "cache") - { - let tar_path = Path::new("config").join(entry.file_name()); - - if entry.file_type()?.is_dir() { - tar.append_dir_all(tar_path, entry.path())?; - } else { - tar.append_path_with_name(entry.path(), tar_path)?; - } + for (path_in_tar, path) in &self.files { + tar.append_path_with_name(path, path_in_tar)?; } // TODO re-add this info file in some way From 703a25e8beb36e4546d588808a3f11d59f7c80bd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 9 Jun 2023 10:42:17 +0200 Subject: [PATCH 16/20] refactor: use utc time --- src/server/backups.rs | 4 ++-- src/server/process.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index e5fd4de..d567f42 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -17,7 +17,7 @@ pub struct BackupManager { config_dir: PathBuf, world_dir: PathBuf, max_backups: u64, - start_time: Option>, + start_time: Option>, files: Vec<(PathBuf, PathBuf)> } @@ -69,7 +69,7 @@ impl BackupManager { } pub fn create_archive(&mut self) -> io::Result<()> { - let start_time = chrono::offset::Local::now(); + let start_time = chrono::offset::Utc::now(); self.start_time = Some(start_time); let filename = format!("{}", start_time.format(FILENAME_FORMAT)); diff --git a/src/server/process.rs b/src/server/process.rs index f503c84..7555aa3 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -88,7 +88,7 @@ impl ServerProcess { // We wait some time to (hopefully) ensure the save-all call has completed std::thread::sleep(std::time::Duration::from_secs(10)); - let start_time = chrono::offset::Local::now(); + let start_time = chrono::offset::Utc::now(); let res = self.backups.create_archive(); if res.is_ok() { @@ -98,7 +98,7 @@ impl ServerProcess { // The server's save feature needs to be enabled again even if the archive failed to create self.custom("save-on")?; - let duration = chrono::offset::Local::now() - start_time; + let duration = chrono::offset::Utc::now() - start_time; let duration_str = format!( "{}m{}s", duration.num_seconds() / 60, From b7a678e32f5af5bdcb10f430771767db22a51431 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Jun 2023 15:09:07 +0200 Subject: [PATCH 17/20] feat: lots of backup stuff --- .cargo/config.toml | 4 +- src/server/backups.rs | 155 +++++++++++++++++++++++++----------------- src/server/mod.rs | 1 + src/server/path.rs | 19 ++++++ src/server/process.rs | 2 +- 5 files changed, 116 insertions(+), 65 deletions(-) create mode 100644 src/server/path.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index d1675c8..37bb90a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,3 @@ [alias] -runs = "run -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper.jar" -runrs = "run --release -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper.jar" +runs = "run -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" +runrs = "run --release -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" diff --git a/src/server/backups.rs b/src/server/backups.rs index d567f42..e0937f1 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -1,5 +1,7 @@ +use chrono::{Local, Utc}; use flate2::write::GzEncoder; use flate2::Compression; +use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io; use std::path::{Path, PathBuf}; @@ -10,15 +12,95 @@ extern "C" { fn getegid() -> u32; } -static FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; +const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; + +pub enum BackupType { + Full, + Incremental, +} + +/// Represents a successful backup +pub struct Backup { + previous: Option>, + /// When the backup was started (also corresponds to the name) + start_time: chrono::DateTime, + /// Type of the backup + type_: BackupType, + /// What files were added/modified in each part of the tarball. + pub added: HashMap>, + /// What files were removed in this backup, in comparison to the previous backup. For full + /// backups, this will always be empty, as they do not consider previous backups. + /// The map stores a separate list for each top-level directory, as the contents of these + /// directories can come for different source directories. + pub removed: HashMap>, +} + +fn files(src_dir: PathBuf) -> io::Result> { + let mut dirs = vec![src_dir.clone()]; + let mut files: HashSet = HashSet::new(); + + while let Some(dir) = dirs.pop() { + for res in dir.read_dir()? { + let entry = res?; + + if entry.file_name() == "cache" { + continue; + } + + if entry.file_type()?.is_dir() { + dirs.push(entry.path()); + } else { + files.insert(entry.path().strip_prefix(&src_dir).unwrap().to_path_buf()); + } + } + } + + Ok(files) +} + +impl Backup { + /// Create a new full backup + pub fn create>( + backup_dir: P, + dirs: Vec<(PathBuf, PathBuf)>, + ) -> io::Result { + let backup_dir = backup_dir.as_ref(); + let start_time = chrono::offset::Utc::now(); + + let filename = format!("{}", start_time.format(FILENAME_FORMAT)); + let path = backup_dir.join(filename); + let tar_gz = File::create(path)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut ar = tar::Builder::new(enc); + + let mut added: HashMap> = HashMap::new(); + + for (dir_in_tar, src_dir) in dirs { + let files = files(src_dir.clone())?; + + for path in &files { + ar.append_path_with_name(dir_in_tar.join(&path), src_dir.join(&path))?; + } + + added.insert(dir_in_tar, files); + } + + Ok(Backup { + previous: None, + type_: BackupType::Full, + start_time, + added, + removed: HashMap::new(), + }) + } +} pub struct BackupManager { backup_dir: PathBuf, config_dir: PathBuf, world_dir: PathBuf, max_backups: u64, - start_time: Option>, - files: Vec<(PathBuf, PathBuf)> + last_backup: Option, } impl BackupManager { @@ -33,75 +115,24 @@ impl BackupManager { config_dir, world_dir, max_backups, - start_time: None, - files: Vec::new() + last_backup: None, } } - fn set_files_to_backup(&mut self) -> io::Result<()> { - let mut dirs = vec![ - (PathBuf::from("worlds"), self.world_dir.clone()), + pub fn create_backup(&mut self) -> io::Result<()> { + let dirs = vec![ (PathBuf::from("config"), self.config_dir.clone()), + (PathBuf::from("worlds"), self.world_dir.clone()), ]; - self.files.clear(); - - while let Some((path_in_tar, path)) = dirs.pop() { - for res in path.read_dir()? { - let entry = res?; - - if entry.file_name() == "cache" { - continue; - } - - let new_path_in_tar = path_in_tar.join(entry.file_name()); - - // All dirs get expanded recursively, while all files get returned as output - // NOTE: does this remove empty directories from backups? Is this a problem? - if entry.file_type()?.is_dir() { - dirs.push((new_path_in_tar, entry.path())); - } else { - self.files.push((new_path_in_tar, entry.path())); - } - } + if let Some(last_backup) = &self.last_backup { + todo!(); + } else { + self.last_backup = Some(Backup::create(&self.backup_dir, dirs)?); } Ok(()) } - pub fn create_archive(&mut self) -> io::Result<()> { - let start_time = chrono::offset::Utc::now(); - self.start_time = Some(start_time); - - let filename = format!("{}", start_time.format(FILENAME_FORMAT)); - let path = self.backup_dir.join(filename); - let tar_gz = File::create(path)?; - let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut tar = tar::Builder::new(enc); - - self.set_files_to_backup()?; - - for (path_in_tar, path) in &self.files { - tar.append_path_with_name(path, path_in_tar)?; - } - - // TODO re-add this info file in some way - // We add a file to the backup describing for what version it was made - // let info = format!("{} {}", self.type_, self.version); - // let info_bytes = info.as_bytes(); - - // let mut header = tar::Header::new_gnu(); - // header.set_size(info_bytes.len().try_into().unwrap()); - // header.set_mode(0o100644); - // unsafe { - // header.set_gid(getegid().into()); - // header.set_uid(geteuid().into()); - // } - - // tar.append_data(&mut header, "info.txt", info_bytes)?; - - Ok(()) - } - /// Remove the oldest backups pub fn remove_old_backups(&mut self) -> std::io::Result<()> { // The naming format used allows us to sort the backups by name and still get a sorting by diff --git a/src/server/mod.rs b/src/server/mod.rs index ed5cb21..4c2beb2 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,5 +1,6 @@ mod backups; mod command; +mod path; mod process; pub use backups::BackupManager; diff --git a/src/server/path.rs b/src/server/path.rs new file mode 100644 index 0000000..d9df799 --- /dev/null +++ b/src/server/path.rs @@ -0,0 +1,19 @@ +use chrono::Utc; +use std::collections::HashSet; +use std::path::PathBuf; +use std::{fs, io}; + +struct ReadDirRecursive { + ignored_dirs: HashSet, + read_dir: Option, + stack: Vec, +} + +impl ReadDirRecursive { + // pub fn new() +} + +trait PathExt { + fn modified_since(timestamp: chrono::DateTime) -> bool; + fn read_dir_recusive() -> ReadDirRecursive; +} diff --git a/src/server/process.rs b/src/server/process.rs index 7555aa3..6edc484 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -89,7 +89,7 @@ impl ServerProcess { std::thread::sleep(std::time::Duration::from_secs(10)); let start_time = chrono::offset::Utc::now(); - let res = self.backups.create_archive(); + let res = self.backups.create_backup(); if res.is_ok() { self.backups.remove_old_backups()?; From fcc111b4efc75eae67092ec14c61952d2f02322c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 14 Jun 2023 21:47:59 +0200 Subject: [PATCH 18/20] feat: possible incremental backup implementation using new abstraction --- src/server/backups.rs | 232 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 204 insertions(+), 28 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index e0937f1..66c7192 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -5,6 +5,7 @@ use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io; use std::path::{Path, PathBuf}; +use std::sync::Arc; #[link(name = "c")] extern "C" { @@ -14,27 +15,6 @@ extern "C" { const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; -pub enum BackupType { - Full, - Incremental, -} - -/// Represents a successful backup -pub struct Backup { - previous: Option>, - /// When the backup was started (also corresponds to the name) - start_time: chrono::DateTime, - /// Type of the backup - type_: BackupType, - /// What files were added/modified in each part of the tarball. - pub added: HashMap>, - /// What files were removed in this backup, in comparison to the previous backup. For full - /// backups, this will always be empty, as they do not consider previous backups. - /// The map stores a separate list for each top-level directory, as the contents of these - /// directories can come for different source directories. - pub removed: HashMap>, -} - fn files(src_dir: PathBuf) -> io::Result> { let mut dirs = vec![src_dir.clone()]; let mut files: HashSet = HashSet::new(); @@ -58,8 +38,148 @@ fn files(src_dir: PathBuf) -> io::Result> { Ok(files) } +/// Return false only if we can say with certainty that the file wasn't modified since the given +/// timestamp, true otherwise. +fn modified_since>(time: chrono::DateTime, path: T) -> bool { + let path = path.as_ref(); + + if let Ok(metadata) = path.metadata() { + let last_modified = metadata.modified(); + + if let Ok(last_modified) = last_modified { + let t: chrono::DateTime = last_modified.into(); + let t = t.with_timezone(&Local); + + return t >= time; + } + } + + false +} + +#[derive(PartialEq)] +pub enum BackupType { + Full, + Incremental, +} + +#[derive(Debug)] +pub enum BackupError { + NoFullAncestor, +} + +type BackupResult = Result; + +/// Represents the changes relative to the previous backup +pub struct BackupDelta { + /// What files were added/modified in each part of the tarball. + pub added: HashMap>, + /// What files were removed in this backup, in comparison to the previous backup. For full + /// backups, this will always be empty, as they do not consider previous backups. + /// The map stores a separate list for each top-level directory, as the contents of these + /// directories can come for different source directories. + pub removed: HashMap>, +} + +impl BackupDelta { + pub fn new() -> Self { + BackupDelta { + added: HashMap::new(), + removed: HashMap::new(), + } + } + + /// Update the current state so that its result becomes the merge of itself and the other + /// state. + pub fn merge(&mut self, delta: &BackupDelta) { + for (dir, added) in delta.added.iter() { + // Files that were removed in the current state, but added in the new state, are no + // longer removed + if let Some(orig_removed) = self.removed.get_mut(dir) { + orig_removed.retain(|k| !added.contains(k)); + } + + // Newly added files are added to the state as well + if let Some(orig_added) = self.added.get_mut(dir) { + orig_added.extend(added.iter().cloned()); + } else { + self.added.insert(dir.clone(), added.clone()); + } + } + + for (dir, removed) in delta.removed.iter() { + // Files that were originally added, but now deleted are removed from the added list + if let Some(orig_added) = self.added.get_mut(dir) { + orig_added.retain(|k| !removed.contains(k)); + } + + // Newly removed files are added to the state as well + if let Some(orig_removed) = self.removed.get_mut(dir) { + orig_removed.extend(removed.iter().cloned()); + } else { + self.removed.insert(dir.clone(), removed.clone()); + } + } + } + + /// Modify the given state by applying this delta's changes to it + pub fn apply(&self, state: &mut HashMap>) { + // First we add new files, then we remove the old ones + for (dir, added) in self.added.iter() { + if let Some(current) = state.get_mut(dir) { + current.extend(added.iter().cloned()); + } else { + state.insert(dir.clone(), added.clone()); + } + } + + for (dir, removed) in self.removed.iter() { + if let Some(current) = state.get_mut(dir) { + current.retain(|k| !removed.contains(k)); + } + } + } +} + +/// Represents a successful backup +pub struct Backup { + previous: Option>, + /// When the backup was started (also corresponds to the name) + start_time: chrono::DateTime, + /// Type of the backup + type_: BackupType, + delta: BackupDelta, +} + impl Backup { - /// Create a new full backup + /// Calculate the full state of the backup by applying all its ancestors delta's in order, + /// starting from the last full ancestor. + pub fn state(&self) -> BackupResult>> { + if self.type_ == BackupType::Full { + let mut state = HashMap::new(); + self.delta.apply(&mut state); + + Ok(state) + } else if let Some(previous) = &self.previous { + let mut state = previous.state()?; + self.delta.apply(&mut state); + + Ok(state) + } else { + return Err(BackupError::NoFullAncestor); + } + } + /// Create a new Full backup, populated with the given directories. + /// + /// # Arguments + /// + /// * `backup_dir` - Directory to store archive in + /// * `dirs` - list of tuples `(path_in_tar, src_dir)` with `path_in_tar` the directory name + /// under which `src_dir`'s contents should be stored in the archive + /// + /// # Returns + /// + /// The `Backup` instance describing this new backup. pub fn create>( backup_dir: P, dirs: Vec<(PathBuf, PathBuf)>, @@ -79,7 +199,7 @@ impl Backup { let files = files(src_dir.clone())?; for path in &files { - ar.append_path_with_name(dir_in_tar.join(&path), src_dir.join(&path))?; + ar.append_path_with_name(dir_in_tar.join(path), src_dir.join(path))?; } added.insert(dir_in_tar, files); @@ -89,8 +209,58 @@ impl Backup { previous: None, type_: BackupType::Full, start_time, - added, - removed: HashMap::new(), + delta: BackupDelta { + added, + removed: HashMap::new(), + }, + }) + } + + /// Create a new incremental backup from a given previous backup + pub fn create_from>( + previous: Arc, + backup_dir: P, + dirs: Vec<(PathBuf, PathBuf)>, + ) -> io::Result { + let backup_dir = backup_dir.as_ref(); + let start_time = chrono::offset::Utc::now(); + + let filename = format!("{}", start_time.format(FILENAME_FORMAT)); + let path = backup_dir.join(filename); + let tar_gz = File::create(path)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut ar = tar::Builder::new(enc); + + let previous_state = previous.state().unwrap(); + let mut delta = BackupDelta::new(); + + for (dir_in_tar, src_dir) in dirs { + let files = files(src_dir.clone())?; + let added_files = files + .iter() + .filter(|p| modified_since(previous.start_time, p)) + .cloned() + .collect::>(); + + for path in added_files.iter() { + ar.append_path_with_name(dir_in_tar.join(path), src_dir.join(path))?; + } + + delta.added.insert(dir_in_tar.clone(), added_files); + + if let Some(previous_files) = previous_state.get(&dir_in_tar) { + delta.removed.insert( + dir_in_tar, + previous_files.difference(&files).cloned().collect(), + ); + } + } + + Ok(Backup { + previous: Some(previous), + type_: BackupType::Incremental, + start_time, + delta, }) } } @@ -100,7 +270,7 @@ pub struct BackupManager { config_dir: PathBuf, world_dir: PathBuf, max_backups: u64, - last_backup: Option, + last_backup: Option>, } impl BackupManager { @@ -124,10 +294,16 @@ impl BackupManager { (PathBuf::from("config"), self.config_dir.clone()), (PathBuf::from("worlds"), self.world_dir.clone()), ]; + if let Some(last_backup) = &self.last_backup { - todo!(); + let clone = last_backup.clone(); + self.last_backup = Some(Arc::new(Backup::create_from( + clone, + &self.backup_dir, + dirs, + )?)); } else { - self.last_backup = Some(Backup::create(&self.backup_dir, dirs)?); + self.last_backup = Some(Arc::new(Backup::create(&self.backup_dir, dirs)?)); } Ok(()) From a9e7b215d18f7fe9a9d778d878649ec29df8c09b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 14 Jun 2023 22:16:54 +0200 Subject: [PATCH 19/20] feat: move running server to subcommand --- .cargo/config.toml | 2 +- CHANGELOG.md | 4 ++ src/cli.rs | 91 +++++++++++++++++++++++++++++++++++++++ src/main.rs | 103 ++++++++++----------------------------------- 4 files changed, 119 insertions(+), 81 deletions(-) create mode 100644 src/cli.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 37bb90a..09b7896 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,3 @@ [alias] -runs = "run -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" +runs = "run -- run paper 1.19.4-550 --config data/config --backup data/backups --world data/worlds --jar paper-1.19.4-550.jar" runrs = "run --release -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b9570a..e431581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/alex/src/branch/dev) +### Changed + +* Running the server now uses the `run` CLI subcommand + ## [0.2.2](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.2) ### Fixed diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..1acbe36 --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,91 @@ +use crate::server::ServerType; +use clap::{Args, Parser, Subcommand}; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, + /// Directory where configs are stored, and where the server will run + #[arg( + long, + value_name = "CONFIG_DIR", + default_value = ".", + env = "ALEX_CONFIG_DIR", + global = true + )] + pub config: PathBuf, + /// Directory where world files will be saved + #[arg( + long, + value_name = "WORLD_DIR", + default_value = "../worlds", + env = "ALEX_WORLD_DIR", + global = true + )] + pub world: PathBuf, + /// Directory where backups will be stored + #[arg( + long, + value_name = "BACKUP_DIR", + default_value = "../backups", + env = "ALEX_BACKUP_DIR", + global = true + )] + pub backup: PathBuf, + + /// How many backups to keep + #[arg( + short = 'n', + long, + default_value_t = 7, + env = "ALEX_MAX_BACKUPS", + global = true + )] + pub max_backups: u64, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Run the server + Run(RunArgs), +} + +#[derive(Args)] +pub struct RunArgs { + /// Type of server + pub type_: ServerType, + /// Version string for the server, e.g. 1.19.4-545 + #[arg(env = "ALEX_SERVER_VERSION")] + pub server_version: String, + + /// Server jar to execute + #[arg( + long, + value_name = "JAR_PATH", + default_value = "server.jar", + env = "ALEX_JAR" + )] + pub jar: PathBuf, + + /// Java command to run the server jar with + #[arg(long, value_name = "JAVA_CMD", default_value_t = String::from("java"), env = "ALEX_JAVA")] + pub java: String, + + /// XMS value in megabytes for the server instance + #[arg(long, default_value_t = 1024, env = "ALEX_XMS")] + pub xms: u64, + /// XMX value in megabytes for the server instance + #[arg(long, default_value_t = 2048, env = "ALEX_XMX")] + pub xmx: u64, + + /// How frequently to perform a backup, in minutes; 0 to disable. + #[arg(short = 't', long, default_value_t = 0, env = "ALEX_FREQUENCY")] + pub frequency: u64, + + /// Don't actually run the server, but simply output the server configuration that would have + /// been ran + #[arg(short, long, default_value_t = false)] + pub dry: bool, +} diff --git a/src/main.rs b/src/main.rs index 2cd84ff..a1ae21c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,78 +1,13 @@ +mod cli; mod server; mod signals; mod stdin; use clap::Parser; -use server::ServerType; +use cli::{Cli, Commands, RunArgs}; use std::io; -use std::path::PathBuf; use std::sync::{Arc, Mutex}; -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct Cli { - /// Type of server - type_: ServerType, - /// Version string for the server, e.g. 1.19.4-545 - #[arg(env = "ALEX_SERVER_VERSION")] - server_version: String, - - /// Server jar to execute - #[arg( - long, - value_name = "JAR_PATH", - default_value = "server.jar", - env = "ALEX_JAR" - )] - jar: PathBuf, - /// Directory where configs are stored, and where the server will run - #[arg( - long, - value_name = "CONFIG_DIR", - default_value = ".", - env = "ALEX_CONFIG_DIR" - )] - config: PathBuf, - /// Directory where world files will be saved - #[arg( - long, - value_name = "WORLD_DIR", - default_value = "../worlds", - env = "ALEX_WORLD_DIR" - )] - world: PathBuf, - /// Directory where backups will be stored - #[arg( - long, - value_name = "BACKUP_DIR", - default_value = "../backups", - env = "ALEX_BACKUP_DIR" - )] - backup: PathBuf, - /// Java command to run the server jar with - #[arg(long, value_name = "JAVA_CMD", default_value_t = String::from("java"), env = "ALEX_JAVA")] - java: String, - - /// XMS value in megabytes for the server instance - #[arg(long, default_value_t = 1024, env = "ALEX_XMS")] - xms: u64, - /// XMX value in megabytes for the server instance - #[arg(long, default_value_t = 2048, env = "ALEX_XMX")] - xmx: u64, - - /// How many backups to keep - #[arg(short = 'n', long, default_value_t = 7, env = "ALEX_MAX_BACKUPS")] - max_backups: u64, - /// How frequently to perform a backup, in minutes; 0 to disable. - #[arg(short = 't', long, default_value_t = 0, env = "ALEX_FREQUENCY")] - frequency: u64, - - /// Don't actually run the server, but simply output the server configuration that would have - /// been ran - #[arg(short, long, default_value_t = false)] - dry: bool, -} - fn backups_thread(counter: Arc>, frequency: u64) { loop { std::thread::sleep(std::time::Duration::from_secs(frequency * 60)); @@ -86,22 +21,21 @@ fn backups_thread(counter: Arc>, frequency: u64) { } } -fn main() -> io::Result<()> { +fn command_run(cli: &Cli, args: &RunArgs) -> io::Result<()> { let (_, mut signals) = signals::install_signal_handlers()?; - let cli = Cli::parse(); - let mut cmd = server::ServerCommand::new(cli.type_, &cli.server_version) - .java(&cli.java) - .jar(cli.jar) - .config(cli.config) - .world(cli.world) - .backup(cli.backup) - .xms(cli.xms) - .xmx(cli.xmx) + let mut cmd = server::ServerCommand::new(args.type_, &args.server_version) + .java(&args.java) + .jar(args.jar.clone()) + .config(cli.config.clone()) + .world(cli.world.clone()) + .backup(cli.backup.clone()) + .xms(args.xms) + .xmx(args.xmx) .max_backups(cli.max_backups); cmd.canonicalize()?; - if cli.dry { + if args.dry { print!("{}", cmd); return Ok(()); @@ -109,9 +43,10 @@ fn main() -> io::Result<()> { let counter = Arc::new(Mutex::new(cmd.spawn()?)); - if cli.frequency > 0 { + if args.frequency > 0 { let clone = Arc::clone(&counter); - std::thread::spawn(move || backups_thread(clone, cli.frequency)); + let frequency = args.frequency; + std::thread::spawn(move || backups_thread(clone, frequency)); } // Spawn thread that handles the main stdin loop @@ -121,3 +56,11 @@ fn main() -> io::Result<()> { // Signal handler loop exits the process when necessary signals::handle_signals(&mut signals, counter) } + +fn main() -> io::Result<()> { + let cli = Cli::parse(); + + match &cli.command { + Commands::Run(args) => command_run(&cli, args), + } +} From d204c684008fc35e8e9e6aa619c1aa034328d9f7 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 15 Jun 2023 09:56:40 +0200 Subject: [PATCH 20/20] fix: actually working incremental backup --- src/server/backups.rs | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index 66c7192..2620e17 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -38,9 +38,12 @@ fn files(src_dir: PathBuf) -> io::Result> { Ok(files) } -/// Return false only if we can say with certainty that the file wasn't modified since the given -/// timestamp, true otherwise. -fn modified_since>(time: chrono::DateTime, path: T) -> bool { +/// Check whether a file has been modified since the given timestamp. +/// +/// Note that this function will *only* return true if it can determine with certainty that the +/// file has not been modified. If any errors occur while obtaining the required metadata (e.g. if +/// the file system does not support this metadata), this function will return false. +fn not_modified_since>(time: chrono::DateTime, path: T) -> bool { let path = path.as_ref(); if let Ok(metadata) = path.metadata() { @@ -50,14 +53,14 @@ fn modified_since>(time: chrono::DateTime, path: T) -> bool let t: chrono::DateTime = last_modified.into(); let t = t.with_timezone(&Local); - return t >= time; + return t < time; } } false } -#[derive(PartialEq)] +#[derive(Debug, PartialEq)] pub enum BackupType { Full, Incremental, @@ -71,6 +74,7 @@ pub enum BackupError { type BackupResult = Result; /// Represents the changes relative to the previous backup +#[derive(Debug)] pub struct BackupDelta { /// What files were added/modified in each part of the tarball. pub added: HashMap>, @@ -142,6 +146,7 @@ impl BackupDelta { } /// Represents a successful backup +#[derive(Debug)] pub struct Backup { previous: Option>, /// When the backup was started (also corresponds to the name) @@ -199,7 +204,7 @@ impl Backup { let files = files(src_dir.clone())?; for path in &files { - ar.append_path_with_name(dir_in_tar.join(path), src_dir.join(path))?; + ar.append_path_with_name(src_dir.join(path), dir_in_tar.join(path))?; } added.insert(dir_in_tar, files); @@ -231,6 +236,7 @@ impl Backup { let enc = GzEncoder::new(tar_gz, Compression::default()); let mut ar = tar::Builder::new(enc); + // TODO remove unwrap let previous_state = previous.state().unwrap(); let mut delta = BackupDelta::new(); @@ -238,12 +244,14 @@ impl Backup { let files = files(src_dir.clone())?; let added_files = files .iter() - .filter(|p| modified_since(previous.start_time, p)) + // This explicit negation is because we wish to also include files for which we + // couldn't determine the last modified time + .filter(|p| !not_modified_since(previous.start_time, src_dir.join(p))) .cloned() .collect::>(); for path in added_files.iter() { - ar.append_path_with_name(dir_in_tar.join(path), src_dir.join(path))?; + ar.append_path_with_name(src_dir.join(path), dir_in_tar.join(path))?; } delta.added.insert(dir_in_tar.clone(), added_files); @@ -295,16 +303,13 @@ impl BackupManager { (PathBuf::from("worlds"), self.world_dir.clone()), ]; - if let Some(last_backup) = &self.last_backup { - let clone = last_backup.clone(); - self.last_backup = Some(Arc::new(Backup::create_from( - clone, - &self.backup_dir, - dirs, - )?)); + let backup = if let Some(last_backup) = &self.last_backup { + Backup::create_from(Arc::clone(last_backup), &self.backup_dir, dirs)? } else { - self.last_backup = Some(Arc::new(Backup::create(&self.backup_dir, dirs)?)); - } + Backup::create(&self.backup_dir, dirs)? + }; + + self.last_backup = Some(Arc::new(backup)); Ok(()) }