From 90aa929b738b462aa2ebbb6b47da02830b7d58f6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 7 Jun 2023 21:15:15 +0200 Subject: [PATCH 01/19] feat: show backup time in message --- src/server/process.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/server/process.rs b/src/server/process.rs index a9a9f45..233a678 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -84,6 +84,7 @@ impl ServerProcess { // We wait some time to (hopefully) ensure the save-all call has completed std::thread::sleep(std::time::Duration::from_secs(10)); + let start_time = chrono::offset::Local::now(); let res = self.create_backup_archive(); if res.is_ok() { @@ -93,10 +94,20 @@ impl ServerProcess { // The server's save feature needs to be enabled again even if the archive failed to create self.custom("save-on")?; + let duration = chrono::offset::Local::now() - start_time; + let duration_str = format!( + "{}m{}s", + duration.num_seconds() / 60, + duration.num_seconds() % 60 + ); + if res.is_ok() { - self.custom("say server backed up successfully")?; + self.custom(&format!("say server backed up in {}", duration_str))?; } else { - self.custom("an error occured while backing up the server")?; + self.custom(&format!( + "an error occured after {} while backing up the server", + duration_str + ))?; } res From 4958257f6e28c179d4ac81e1c23d037093438fda Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 9 Jun 2023 09:25:51 +0200 Subject: [PATCH 02/19] refactor: move backup logic to separate module --- .gitignore | 2 +- src/server/backups.rs | 108 ++++++++++++++++++++++++++++++++++++++++++ src/server/mod.rs | 2 + src/server/process.rs | 93 +++++------------------------------- 4 files changed, 123 insertions(+), 82 deletions(-) create mode 100644 src/server/backups.rs diff --git a/.gitignore b/.gitignore index 4259b1b..3695da7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ target/ # testing files *.jar -data/ +data*/ diff --git a/src/server/backups.rs b/src/server/backups.rs new file mode 100644 index 0000000..37e6021 --- /dev/null +++ b/src/server/backups.rs @@ -0,0 +1,108 @@ +use flate2::write::GzEncoder; +use flate2::Compression; +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; + +#[link(name = "c")] +extern "C" { + fn geteuid() -> u32; + fn getegid() -> u32; +} + +static FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; + +pub struct BackupManager { + backup_dir: PathBuf, + config_dir: PathBuf, + world_dir: PathBuf, + max_backups: u64, + start_time: Option>, +} + +impl BackupManager { + pub fn open( + backup_dir: PathBuf, + config_dir: PathBuf, + world_dir: PathBuf, + max_backups: u64, + ) -> Self { + BackupManager { + backup_dir, + config_dir, + world_dir, + max_backups, + start_time: None, + } + } + + pub fn create_archive(&mut self) -> io::Result<()> { + let start_time = chrono::offset::Local::now(); + self.start_time = Some(start_time); + + let filename = format!("{}", start_time.format(FILENAME_FORMAT)); + let path = self.backup_dir.join(filename); + let tar_gz = File::create(path)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut tar = tar::Builder::new(enc); + + tar.append_dir_all("worlds", &self.world_dir)?; + + // Add all files from the config directory that aren't the cache + for entry in self + .config_dir + .read_dir()? + .filter_map(|e| e.ok()) + .filter(|e| e.file_name() != "cache") + { + let tar_path = Path::new("config").join(entry.file_name()); + + if entry.file_type()?.is_dir() { + tar.append_dir_all(tar_path, entry.path())?; + } else { + tar.append_path_with_name(entry.path(), tar_path)?; + } + } + + // TODO re-add this info file in some way + // We add a file to the backup describing for what version it was made + // let info = format!("{} {}", self.type_, self.version); + // let info_bytes = info.as_bytes(); + + // let mut header = tar::Header::new_gnu(); + // header.set_size(info_bytes.len().try_into().unwrap()); + // header.set_mode(0o100644); + // unsafe { + // header.set_gid(getegid().into()); + // header.set_uid(geteuid().into()); + // } + + // tar.append_data(&mut header, "info.txt", info_bytes)?; + + Ok(()) + } + + /// Remove the oldest backups + pub fn remove_old_backups(&mut self) -> std::io::Result<()> { + // The naming format used allows us to sort the backups by name and still get a sorting by + // creation time + let mut backups = self + .backup_dir + .read_dir()? + .filter_map(|res| res.map(|e| e.path()).ok()) + .collect::>(); + backups.sort(); + + let max_backups: usize = self.max_backups.try_into().unwrap(); + + if backups.len() > max_backups { + let excess_backups = backups.len() - max_backups; + + for backup in &backups[0..excess_backups] { + std::fs::remove_file(backup)?; + } + } + + Ok(()) + } +} diff --git a/src/server/mod.rs b/src/server/mod.rs index e3e3131..ed5cb21 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,5 +1,7 @@ +mod backups; mod command; mod process; +pub use backups::BackupManager; pub use command::{ServerCommand, ServerType}; pub use process::ServerProcess; diff --git a/src/server/process.rs b/src/server/process.rs index 233a678..f503c84 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -1,3 +1,4 @@ +use crate::server::BackupManager; use crate::server::ServerType; use flate2::write::GzEncoder; use flate2::Compression; @@ -5,12 +6,6 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Child; -#[link(name = "c")] -extern "C" { - fn geteuid() -> u32; - fn getegid() -> u32; -} - pub struct ServerProcess { type_: ServerType, version: String, @@ -19,6 +14,7 @@ pub struct ServerProcess { backup_dir: PathBuf, max_backups: u64, child: Child, + backups: BackupManager, } impl ServerProcess { @@ -31,6 +27,13 @@ impl ServerProcess { max_backups: u64, child: Child, ) -> ServerProcess { + let backup_manager = BackupManager::open( + backup_dir.clone(), + config_dir.clone(), + world_dir.clone(), + max_backups, + ); + ServerProcess { type_, version, @@ -39,6 +42,7 @@ impl ServerProcess { backup_dir, max_backups, child, + backups: backup_manager, } } @@ -85,10 +89,10 @@ impl ServerProcess { std::thread::sleep(std::time::Duration::from_secs(10)); let start_time = chrono::offset::Local::now(); - let res = self.create_backup_archive(); + let res = self.backups.create_archive(); if res.is_ok() { - self.remove_old_backups()?; + self.backups.remove_old_backups()?; } // The server's save feature needs to be enabled again even if the archive failed to create @@ -112,77 +116,4 @@ impl ServerProcess { res } - - /// Create a new compressed backup archive of the server's data. - fn create_backup_archive(&mut self) -> std::io::Result<()> { - // Create a gzip-compressed tarball of the worlds folder - let filename = format!( - "{}", - chrono::offset::Local::now().format("%Y-%m-%d_%H-%M-%S.tar.gz") - ); - let path = self.backup_dir.join(filename); - let tar_gz = std::fs::File::create(path)?; - let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut tar = tar::Builder::new(enc); - - tar.append_dir_all("worlds", &self.world_dir)?; - - // Add all files from the config directory that aren't the cache - for entry in self - .config_dir - .read_dir()? - .filter_map(|e| e.ok()) - .filter(|e| e.file_name() != "cache") - { - let tar_path = Path::new("config").join(entry.file_name()); - - if entry.file_type()?.is_dir() { - tar.append_dir_all(tar_path, entry.path())?; - } else { - tar.append_path_with_name(entry.path(), tar_path)?; - } - } - - // We add a file to the backup describing for what version it was made - let info = format!("{} {}", self.type_, self.version); - let info_bytes = info.as_bytes(); - - let mut header = tar::Header::new_gnu(); - header.set_size(info_bytes.len().try_into().unwrap()); - header.set_mode(0o100644); - unsafe { - header.set_gid(getegid().into()); - header.set_uid(geteuid().into()); - } - - tar.append_data(&mut header, "info.txt", info_bytes)?; - - // Backup file gets finalized in the drop - - Ok(()) - } - - /// Remove the oldest backups - fn remove_old_backups(&mut self) -> std::io::Result<()> { - // The naming format used allows us to sort the backups by name and still get a sorting by - // creation time - let mut backups = self - .backup_dir - .read_dir()? - .filter_map(|res| res.map(|e| e.path()).ok()) - .collect::>(); - backups.sort(); - - let max_backups: usize = self.max_backups.try_into().unwrap(); - - if backups.len() > max_backups { - let excess_backups = backups.len() - max_backups; - - for backup in &backups[0..excess_backups] { - std::fs::remove_file(backup)?; - } - } - - Ok(()) - } } From 29d6713486bae0bd38d8bbebe533c02fa93f8fbd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 9 Jun 2023 10:11:02 +0200 Subject: [PATCH 03/19] feat: implement own listing of files --- src/server/backups.rs | 50 ++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index 37e6021..e5fd4de 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -18,6 +18,7 @@ pub struct BackupManager { world_dir: PathBuf, max_backups: u64, start_time: Option>, + files: Vec<(PathBuf, PathBuf)> } impl BackupManager { @@ -33,9 +34,40 @@ impl BackupManager { world_dir, max_backups, start_time: None, + files: Vec::new() } } + fn set_files_to_backup(&mut self) -> io::Result<()> { + let mut dirs = vec![ + (PathBuf::from("worlds"), self.world_dir.clone()), + (PathBuf::from("config"), self.config_dir.clone()), + ]; + self.files.clear(); + + while let Some((path_in_tar, path)) = dirs.pop() { + for res in path.read_dir()? { + let entry = res?; + + if entry.file_name() == "cache" { + continue; + } + + let new_path_in_tar = path_in_tar.join(entry.file_name()); + + // All dirs get expanded recursively, while all files get returned as output + // NOTE: does this remove empty directories from backups? Is this a problem? + if entry.file_type()?.is_dir() { + dirs.push((new_path_in_tar, entry.path())); + } else { + self.files.push((new_path_in_tar, entry.path())); + } + } + } + + Ok(()) + } + pub fn create_archive(&mut self) -> io::Result<()> { let start_time = chrono::offset::Local::now(); self.start_time = Some(start_time); @@ -46,22 +78,10 @@ impl BackupManager { let enc = GzEncoder::new(tar_gz, Compression::default()); let mut tar = tar::Builder::new(enc); - tar.append_dir_all("worlds", &self.world_dir)?; + self.set_files_to_backup()?; - // Add all files from the config directory that aren't the cache - for entry in self - .config_dir - .read_dir()? - .filter_map(|e| e.ok()) - .filter(|e| e.file_name() != "cache") - { - let tar_path = Path::new("config").join(entry.file_name()); - - if entry.file_type()?.is_dir() { - tar.append_dir_all(tar_path, entry.path())?; - } else { - tar.append_path_with_name(entry.path(), tar_path)?; - } + for (path_in_tar, path) in &self.files { + tar.append_path_with_name(path, path_in_tar)?; } // TODO re-add this info file in some way From 703a25e8beb36e4546d588808a3f11d59f7c80bd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 9 Jun 2023 10:42:17 +0200 Subject: [PATCH 04/19] refactor: use utc time --- src/server/backups.rs | 4 ++-- src/server/process.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index e5fd4de..d567f42 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -17,7 +17,7 @@ pub struct BackupManager { config_dir: PathBuf, world_dir: PathBuf, max_backups: u64, - start_time: Option>, + start_time: Option>, files: Vec<(PathBuf, PathBuf)> } @@ -69,7 +69,7 @@ impl BackupManager { } pub fn create_archive(&mut self) -> io::Result<()> { - let start_time = chrono::offset::Local::now(); + let start_time = chrono::offset::Utc::now(); self.start_time = Some(start_time); let filename = format!("{}", start_time.format(FILENAME_FORMAT)); diff --git a/src/server/process.rs b/src/server/process.rs index f503c84..7555aa3 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -88,7 +88,7 @@ impl ServerProcess { // We wait some time to (hopefully) ensure the save-all call has completed std::thread::sleep(std::time::Duration::from_secs(10)); - let start_time = chrono::offset::Local::now(); + let start_time = chrono::offset::Utc::now(); let res = self.backups.create_archive(); if res.is_ok() { @@ -98,7 +98,7 @@ impl ServerProcess { // The server's save feature needs to be enabled again even if the archive failed to create self.custom("save-on")?; - let duration = chrono::offset::Local::now() - start_time; + let duration = chrono::offset::Utc::now() - start_time; let duration_str = format!( "{}m{}s", duration.num_seconds() / 60, From b7a678e32f5af5bdcb10f430771767db22a51431 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Jun 2023 15:09:07 +0200 Subject: [PATCH 05/19] feat: lots of backup stuff --- .cargo/config.toml | 4 +- src/server/backups.rs | 155 +++++++++++++++++++++++++----------------- src/server/mod.rs | 1 + src/server/path.rs | 19 ++++++ src/server/process.rs | 2 +- 5 files changed, 116 insertions(+), 65 deletions(-) create mode 100644 src/server/path.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index d1675c8..37bb90a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,3 @@ [alias] -runs = "run -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper.jar" -runrs = "run --release -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper.jar" +runs = "run -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" +runrs = "run --release -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" diff --git a/src/server/backups.rs b/src/server/backups.rs index d567f42..e0937f1 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -1,5 +1,7 @@ +use chrono::{Local, Utc}; use flate2::write::GzEncoder; use flate2::Compression; +use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io; use std::path::{Path, PathBuf}; @@ -10,15 +12,95 @@ extern "C" { fn getegid() -> u32; } -static FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; +const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; + +pub enum BackupType { + Full, + Incremental, +} + +/// Represents a successful backup +pub struct Backup { + previous: Option>, + /// When the backup was started (also corresponds to the name) + start_time: chrono::DateTime, + /// Type of the backup + type_: BackupType, + /// What files were added/modified in each part of the tarball. + pub added: HashMap>, + /// What files were removed in this backup, in comparison to the previous backup. For full + /// backups, this will always be empty, as they do not consider previous backups. + /// The map stores a separate list for each top-level directory, as the contents of these + /// directories can come for different source directories. + pub removed: HashMap>, +} + +fn files(src_dir: PathBuf) -> io::Result> { + let mut dirs = vec![src_dir.clone()]; + let mut files: HashSet = HashSet::new(); + + while let Some(dir) = dirs.pop() { + for res in dir.read_dir()? { + let entry = res?; + + if entry.file_name() == "cache" { + continue; + } + + if entry.file_type()?.is_dir() { + dirs.push(entry.path()); + } else { + files.insert(entry.path().strip_prefix(&src_dir).unwrap().to_path_buf()); + } + } + } + + Ok(files) +} + +impl Backup { + /// Create a new full backup + pub fn create>( + backup_dir: P, + dirs: Vec<(PathBuf, PathBuf)>, + ) -> io::Result { + let backup_dir = backup_dir.as_ref(); + let start_time = chrono::offset::Utc::now(); + + let filename = format!("{}", start_time.format(FILENAME_FORMAT)); + let path = backup_dir.join(filename); + let tar_gz = File::create(path)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut ar = tar::Builder::new(enc); + + let mut added: HashMap> = HashMap::new(); + + for (dir_in_tar, src_dir) in dirs { + let files = files(src_dir.clone())?; + + for path in &files { + ar.append_path_with_name(dir_in_tar.join(&path), src_dir.join(&path))?; + } + + added.insert(dir_in_tar, files); + } + + Ok(Backup { + previous: None, + type_: BackupType::Full, + start_time, + added, + removed: HashMap::new(), + }) + } +} pub struct BackupManager { backup_dir: PathBuf, config_dir: PathBuf, world_dir: PathBuf, max_backups: u64, - start_time: Option>, - files: Vec<(PathBuf, PathBuf)> + last_backup: Option, } impl BackupManager { @@ -33,75 +115,24 @@ impl BackupManager { config_dir, world_dir, max_backups, - start_time: None, - files: Vec::new() + last_backup: None, } } - fn set_files_to_backup(&mut self) -> io::Result<()> { - let mut dirs = vec![ - (PathBuf::from("worlds"), self.world_dir.clone()), + pub fn create_backup(&mut self) -> io::Result<()> { + let dirs = vec![ (PathBuf::from("config"), self.config_dir.clone()), + (PathBuf::from("worlds"), self.world_dir.clone()), ]; - self.files.clear(); - - while let Some((path_in_tar, path)) = dirs.pop() { - for res in path.read_dir()? { - let entry = res?; - - if entry.file_name() == "cache" { - continue; - } - - let new_path_in_tar = path_in_tar.join(entry.file_name()); - - // All dirs get expanded recursively, while all files get returned as output - // NOTE: does this remove empty directories from backups? Is this a problem? - if entry.file_type()?.is_dir() { - dirs.push((new_path_in_tar, entry.path())); - } else { - self.files.push((new_path_in_tar, entry.path())); - } - } + if let Some(last_backup) = &self.last_backup { + todo!(); + } else { + self.last_backup = Some(Backup::create(&self.backup_dir, dirs)?); } Ok(()) } - pub fn create_archive(&mut self) -> io::Result<()> { - let start_time = chrono::offset::Utc::now(); - self.start_time = Some(start_time); - - let filename = format!("{}", start_time.format(FILENAME_FORMAT)); - let path = self.backup_dir.join(filename); - let tar_gz = File::create(path)?; - let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut tar = tar::Builder::new(enc); - - self.set_files_to_backup()?; - - for (path_in_tar, path) in &self.files { - tar.append_path_with_name(path, path_in_tar)?; - } - - // TODO re-add this info file in some way - // We add a file to the backup describing for what version it was made - // let info = format!("{} {}", self.type_, self.version); - // let info_bytes = info.as_bytes(); - - // let mut header = tar::Header::new_gnu(); - // header.set_size(info_bytes.len().try_into().unwrap()); - // header.set_mode(0o100644); - // unsafe { - // header.set_gid(getegid().into()); - // header.set_uid(geteuid().into()); - // } - - // tar.append_data(&mut header, "info.txt", info_bytes)?; - - Ok(()) - } - /// Remove the oldest backups pub fn remove_old_backups(&mut self) -> std::io::Result<()> { // The naming format used allows us to sort the backups by name and still get a sorting by diff --git a/src/server/mod.rs b/src/server/mod.rs index ed5cb21..4c2beb2 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,5 +1,6 @@ mod backups; mod command; +mod path; mod process; pub use backups::BackupManager; diff --git a/src/server/path.rs b/src/server/path.rs new file mode 100644 index 0000000..d9df799 --- /dev/null +++ b/src/server/path.rs @@ -0,0 +1,19 @@ +use chrono::Utc; +use std::collections::HashSet; +use std::path::PathBuf; +use std::{fs, io}; + +struct ReadDirRecursive { + ignored_dirs: HashSet, + read_dir: Option, + stack: Vec, +} + +impl ReadDirRecursive { + // pub fn new() +} + +trait PathExt { + fn modified_since(timestamp: chrono::DateTime) -> bool; + fn read_dir_recusive() -> ReadDirRecursive; +} diff --git a/src/server/process.rs b/src/server/process.rs index 7555aa3..6edc484 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -89,7 +89,7 @@ impl ServerProcess { std::thread::sleep(std::time::Duration::from_secs(10)); let start_time = chrono::offset::Utc::now(); - let res = self.backups.create_archive(); + let res = self.backups.create_backup(); if res.is_ok() { self.backups.remove_old_backups()?; From fcc111b4efc75eae67092ec14c61952d2f02322c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 14 Jun 2023 21:47:59 +0200 Subject: [PATCH 06/19] feat: possible incremental backup implementation using new abstraction --- src/server/backups.rs | 232 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 204 insertions(+), 28 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index e0937f1..66c7192 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -5,6 +5,7 @@ use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io; use std::path::{Path, PathBuf}; +use std::sync::Arc; #[link(name = "c")] extern "C" { @@ -14,27 +15,6 @@ extern "C" { const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; -pub enum BackupType { - Full, - Incremental, -} - -/// Represents a successful backup -pub struct Backup { - previous: Option>, - /// When the backup was started (also corresponds to the name) - start_time: chrono::DateTime, - /// Type of the backup - type_: BackupType, - /// What files were added/modified in each part of the tarball. - pub added: HashMap>, - /// What files were removed in this backup, in comparison to the previous backup. For full - /// backups, this will always be empty, as they do not consider previous backups. - /// The map stores a separate list for each top-level directory, as the contents of these - /// directories can come for different source directories. - pub removed: HashMap>, -} - fn files(src_dir: PathBuf) -> io::Result> { let mut dirs = vec![src_dir.clone()]; let mut files: HashSet = HashSet::new(); @@ -58,8 +38,148 @@ fn files(src_dir: PathBuf) -> io::Result> { Ok(files) } +/// Return false only if we can say with certainty that the file wasn't modified since the given +/// timestamp, true otherwise. +fn modified_since>(time: chrono::DateTime, path: T) -> bool { + let path = path.as_ref(); + + if let Ok(metadata) = path.metadata() { + let last_modified = metadata.modified(); + + if let Ok(last_modified) = last_modified { + let t: chrono::DateTime = last_modified.into(); + let t = t.with_timezone(&Local); + + return t >= time; + } + } + + false +} + +#[derive(PartialEq)] +pub enum BackupType { + Full, + Incremental, +} + +#[derive(Debug)] +pub enum BackupError { + NoFullAncestor, +} + +type BackupResult = Result; + +/// Represents the changes relative to the previous backup +pub struct BackupDelta { + /// What files were added/modified in each part of the tarball. + pub added: HashMap>, + /// What files were removed in this backup, in comparison to the previous backup. For full + /// backups, this will always be empty, as they do not consider previous backups. + /// The map stores a separate list for each top-level directory, as the contents of these + /// directories can come for different source directories. + pub removed: HashMap>, +} + +impl BackupDelta { + pub fn new() -> Self { + BackupDelta { + added: HashMap::new(), + removed: HashMap::new(), + } + } + + /// Update the current state so that its result becomes the merge of itself and the other + /// state. + pub fn merge(&mut self, delta: &BackupDelta) { + for (dir, added) in delta.added.iter() { + // Files that were removed in the current state, but added in the new state, are no + // longer removed + if let Some(orig_removed) = self.removed.get_mut(dir) { + orig_removed.retain(|k| !added.contains(k)); + } + + // Newly added files are added to the state as well + if let Some(orig_added) = self.added.get_mut(dir) { + orig_added.extend(added.iter().cloned()); + } else { + self.added.insert(dir.clone(), added.clone()); + } + } + + for (dir, removed) in delta.removed.iter() { + // Files that were originally added, but now deleted are removed from the added list + if let Some(orig_added) = self.added.get_mut(dir) { + orig_added.retain(|k| !removed.contains(k)); + } + + // Newly removed files are added to the state as well + if let Some(orig_removed) = self.removed.get_mut(dir) { + orig_removed.extend(removed.iter().cloned()); + } else { + self.removed.insert(dir.clone(), removed.clone()); + } + } + } + + /// Modify the given state by applying this delta's changes to it + pub fn apply(&self, state: &mut HashMap>) { + // First we add new files, then we remove the old ones + for (dir, added) in self.added.iter() { + if let Some(current) = state.get_mut(dir) { + current.extend(added.iter().cloned()); + } else { + state.insert(dir.clone(), added.clone()); + } + } + + for (dir, removed) in self.removed.iter() { + if let Some(current) = state.get_mut(dir) { + current.retain(|k| !removed.contains(k)); + } + } + } +} + +/// Represents a successful backup +pub struct Backup { + previous: Option>, + /// When the backup was started (also corresponds to the name) + start_time: chrono::DateTime, + /// Type of the backup + type_: BackupType, + delta: BackupDelta, +} + impl Backup { - /// Create a new full backup + /// Calculate the full state of the backup by applying all its ancestors delta's in order, + /// starting from the last full ancestor. + pub fn state(&self) -> BackupResult>> { + if self.type_ == BackupType::Full { + let mut state = HashMap::new(); + self.delta.apply(&mut state); + + Ok(state) + } else if let Some(previous) = &self.previous { + let mut state = previous.state()?; + self.delta.apply(&mut state); + + Ok(state) + } else { + return Err(BackupError::NoFullAncestor); + } + } + /// Create a new Full backup, populated with the given directories. + /// + /// # Arguments + /// + /// * `backup_dir` - Directory to store archive in + /// * `dirs` - list of tuples `(path_in_tar, src_dir)` with `path_in_tar` the directory name + /// under which `src_dir`'s contents should be stored in the archive + /// + /// # Returns + /// + /// The `Backup` instance describing this new backup. pub fn create>( backup_dir: P, dirs: Vec<(PathBuf, PathBuf)>, @@ -79,7 +199,7 @@ impl Backup { let files = files(src_dir.clone())?; for path in &files { - ar.append_path_with_name(dir_in_tar.join(&path), src_dir.join(&path))?; + ar.append_path_with_name(dir_in_tar.join(path), src_dir.join(path))?; } added.insert(dir_in_tar, files); @@ -89,8 +209,58 @@ impl Backup { previous: None, type_: BackupType::Full, start_time, - added, - removed: HashMap::new(), + delta: BackupDelta { + added, + removed: HashMap::new(), + }, + }) + } + + /// Create a new incremental backup from a given previous backup + pub fn create_from>( + previous: Arc, + backup_dir: P, + dirs: Vec<(PathBuf, PathBuf)>, + ) -> io::Result { + let backup_dir = backup_dir.as_ref(); + let start_time = chrono::offset::Utc::now(); + + let filename = format!("{}", start_time.format(FILENAME_FORMAT)); + let path = backup_dir.join(filename); + let tar_gz = File::create(path)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut ar = tar::Builder::new(enc); + + let previous_state = previous.state().unwrap(); + let mut delta = BackupDelta::new(); + + for (dir_in_tar, src_dir) in dirs { + let files = files(src_dir.clone())?; + let added_files = files + .iter() + .filter(|p| modified_since(previous.start_time, p)) + .cloned() + .collect::>(); + + for path in added_files.iter() { + ar.append_path_with_name(dir_in_tar.join(path), src_dir.join(path))?; + } + + delta.added.insert(dir_in_tar.clone(), added_files); + + if let Some(previous_files) = previous_state.get(&dir_in_tar) { + delta.removed.insert( + dir_in_tar, + previous_files.difference(&files).cloned().collect(), + ); + } + } + + Ok(Backup { + previous: Some(previous), + type_: BackupType::Incremental, + start_time, + delta, }) } } @@ -100,7 +270,7 @@ pub struct BackupManager { config_dir: PathBuf, world_dir: PathBuf, max_backups: u64, - last_backup: Option, + last_backup: Option>, } impl BackupManager { @@ -124,10 +294,16 @@ impl BackupManager { (PathBuf::from("config"), self.config_dir.clone()), (PathBuf::from("worlds"), self.world_dir.clone()), ]; + if let Some(last_backup) = &self.last_backup { - todo!(); + let clone = last_backup.clone(); + self.last_backup = Some(Arc::new(Backup::create_from( + clone, + &self.backup_dir, + dirs, + )?)); } else { - self.last_backup = Some(Backup::create(&self.backup_dir, dirs)?); + self.last_backup = Some(Arc::new(Backup::create(&self.backup_dir, dirs)?)); } Ok(()) From a9e7b215d18f7fe9a9d778d878649ec29df8c09b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 14 Jun 2023 22:16:54 +0200 Subject: [PATCH 07/19] feat: move running server to subcommand --- .cargo/config.toml | 2 +- CHANGELOG.md | 4 ++ src/cli.rs | 91 +++++++++++++++++++++++++++++++++++++++ src/main.rs | 103 ++++++++++----------------------------------- 4 files changed, 119 insertions(+), 81 deletions(-) create mode 100644 src/cli.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 37bb90a..09b7896 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,3 @@ [alias] -runs = "run -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" +runs = "run -- run paper 1.19.4-550 --config data/config --backup data/backups --world data/worlds --jar paper-1.19.4-550.jar" runrs = "run --release -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper-1.19.4-525.jar" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b9570a..e431581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/alex/src/branch/dev) +### Changed + +* Running the server now uses the `run` CLI subcommand + ## [0.2.2](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.2) ### Fixed diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..1acbe36 --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,91 @@ +use crate::server::ServerType; +use clap::{Args, Parser, Subcommand}; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, + /// Directory where configs are stored, and where the server will run + #[arg( + long, + value_name = "CONFIG_DIR", + default_value = ".", + env = "ALEX_CONFIG_DIR", + global = true + )] + pub config: PathBuf, + /// Directory where world files will be saved + #[arg( + long, + value_name = "WORLD_DIR", + default_value = "../worlds", + env = "ALEX_WORLD_DIR", + global = true + )] + pub world: PathBuf, + /// Directory where backups will be stored + #[arg( + long, + value_name = "BACKUP_DIR", + default_value = "../backups", + env = "ALEX_BACKUP_DIR", + global = true + )] + pub backup: PathBuf, + + /// How many backups to keep + #[arg( + short = 'n', + long, + default_value_t = 7, + env = "ALEX_MAX_BACKUPS", + global = true + )] + pub max_backups: u64, +} + +#[derive(Subcommand)] +pub enum Commands { + /// Run the server + Run(RunArgs), +} + +#[derive(Args)] +pub struct RunArgs { + /// Type of server + pub type_: ServerType, + /// Version string for the server, e.g. 1.19.4-545 + #[arg(env = "ALEX_SERVER_VERSION")] + pub server_version: String, + + /// Server jar to execute + #[arg( + long, + value_name = "JAR_PATH", + default_value = "server.jar", + env = "ALEX_JAR" + )] + pub jar: PathBuf, + + /// Java command to run the server jar with + #[arg(long, value_name = "JAVA_CMD", default_value_t = String::from("java"), env = "ALEX_JAVA")] + pub java: String, + + /// XMS value in megabytes for the server instance + #[arg(long, default_value_t = 1024, env = "ALEX_XMS")] + pub xms: u64, + /// XMX value in megabytes for the server instance + #[arg(long, default_value_t = 2048, env = "ALEX_XMX")] + pub xmx: u64, + + /// How frequently to perform a backup, in minutes; 0 to disable. + #[arg(short = 't', long, default_value_t = 0, env = "ALEX_FREQUENCY")] + pub frequency: u64, + + /// Don't actually run the server, but simply output the server configuration that would have + /// been ran + #[arg(short, long, default_value_t = false)] + pub dry: bool, +} diff --git a/src/main.rs b/src/main.rs index 2cd84ff..a1ae21c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,78 +1,13 @@ +mod cli; mod server; mod signals; mod stdin; use clap::Parser; -use server::ServerType; +use cli::{Cli, Commands, RunArgs}; use std::io; -use std::path::PathBuf; use std::sync::{Arc, Mutex}; -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct Cli { - /// Type of server - type_: ServerType, - /// Version string for the server, e.g. 1.19.4-545 - #[arg(env = "ALEX_SERVER_VERSION")] - server_version: String, - - /// Server jar to execute - #[arg( - long, - value_name = "JAR_PATH", - default_value = "server.jar", - env = "ALEX_JAR" - )] - jar: PathBuf, - /// Directory where configs are stored, and where the server will run - #[arg( - long, - value_name = "CONFIG_DIR", - default_value = ".", - env = "ALEX_CONFIG_DIR" - )] - config: PathBuf, - /// Directory where world files will be saved - #[arg( - long, - value_name = "WORLD_DIR", - default_value = "../worlds", - env = "ALEX_WORLD_DIR" - )] - world: PathBuf, - /// Directory where backups will be stored - #[arg( - long, - value_name = "BACKUP_DIR", - default_value = "../backups", - env = "ALEX_BACKUP_DIR" - )] - backup: PathBuf, - /// Java command to run the server jar with - #[arg(long, value_name = "JAVA_CMD", default_value_t = String::from("java"), env = "ALEX_JAVA")] - java: String, - - /// XMS value in megabytes for the server instance - #[arg(long, default_value_t = 1024, env = "ALEX_XMS")] - xms: u64, - /// XMX value in megabytes for the server instance - #[arg(long, default_value_t = 2048, env = "ALEX_XMX")] - xmx: u64, - - /// How many backups to keep - #[arg(short = 'n', long, default_value_t = 7, env = "ALEX_MAX_BACKUPS")] - max_backups: u64, - /// How frequently to perform a backup, in minutes; 0 to disable. - #[arg(short = 't', long, default_value_t = 0, env = "ALEX_FREQUENCY")] - frequency: u64, - - /// Don't actually run the server, but simply output the server configuration that would have - /// been ran - #[arg(short, long, default_value_t = false)] - dry: bool, -} - fn backups_thread(counter: Arc>, frequency: u64) { loop { std::thread::sleep(std::time::Duration::from_secs(frequency * 60)); @@ -86,22 +21,21 @@ fn backups_thread(counter: Arc>, frequency: u64) { } } -fn main() -> io::Result<()> { +fn command_run(cli: &Cli, args: &RunArgs) -> io::Result<()> { let (_, mut signals) = signals::install_signal_handlers()?; - let cli = Cli::parse(); - let mut cmd = server::ServerCommand::new(cli.type_, &cli.server_version) - .java(&cli.java) - .jar(cli.jar) - .config(cli.config) - .world(cli.world) - .backup(cli.backup) - .xms(cli.xms) - .xmx(cli.xmx) + let mut cmd = server::ServerCommand::new(args.type_, &args.server_version) + .java(&args.java) + .jar(args.jar.clone()) + .config(cli.config.clone()) + .world(cli.world.clone()) + .backup(cli.backup.clone()) + .xms(args.xms) + .xmx(args.xmx) .max_backups(cli.max_backups); cmd.canonicalize()?; - if cli.dry { + if args.dry { print!("{}", cmd); return Ok(()); @@ -109,9 +43,10 @@ fn main() -> io::Result<()> { let counter = Arc::new(Mutex::new(cmd.spawn()?)); - if cli.frequency > 0 { + if args.frequency > 0 { let clone = Arc::clone(&counter); - std::thread::spawn(move || backups_thread(clone, cli.frequency)); + let frequency = args.frequency; + std::thread::spawn(move || backups_thread(clone, frequency)); } // Spawn thread that handles the main stdin loop @@ -121,3 +56,11 @@ fn main() -> io::Result<()> { // Signal handler loop exits the process when necessary signals::handle_signals(&mut signals, counter) } + +fn main() -> io::Result<()> { + let cli = Cli::parse(); + + match &cli.command { + Commands::Run(args) => command_run(&cli, args), + } +} From d204c684008fc35e8e9e6aa619c1aa034328d9f7 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 15 Jun 2023 09:56:40 +0200 Subject: [PATCH 08/19] fix: actually working incremental backup --- src/server/backups.rs | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index 66c7192..2620e17 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -38,9 +38,12 @@ fn files(src_dir: PathBuf) -> io::Result> { Ok(files) } -/// Return false only if we can say with certainty that the file wasn't modified since the given -/// timestamp, true otherwise. -fn modified_since>(time: chrono::DateTime, path: T) -> bool { +/// Check whether a file has been modified since the given timestamp. +/// +/// Note that this function will *only* return true if it can determine with certainty that the +/// file has not been modified. If any errors occur while obtaining the required metadata (e.g. if +/// the file system does not support this metadata), this function will return false. +fn not_modified_since>(time: chrono::DateTime, path: T) -> bool { let path = path.as_ref(); if let Ok(metadata) = path.metadata() { @@ -50,14 +53,14 @@ fn modified_since>(time: chrono::DateTime, path: T) -> bool let t: chrono::DateTime = last_modified.into(); let t = t.with_timezone(&Local); - return t >= time; + return t < time; } } false } -#[derive(PartialEq)] +#[derive(Debug, PartialEq)] pub enum BackupType { Full, Incremental, @@ -71,6 +74,7 @@ pub enum BackupError { type BackupResult = Result; /// Represents the changes relative to the previous backup +#[derive(Debug)] pub struct BackupDelta { /// What files were added/modified in each part of the tarball. pub added: HashMap>, @@ -142,6 +146,7 @@ impl BackupDelta { } /// Represents a successful backup +#[derive(Debug)] pub struct Backup { previous: Option>, /// When the backup was started (also corresponds to the name) @@ -199,7 +204,7 @@ impl Backup { let files = files(src_dir.clone())?; for path in &files { - ar.append_path_with_name(dir_in_tar.join(path), src_dir.join(path))?; + ar.append_path_with_name(src_dir.join(path), dir_in_tar.join(path))?; } added.insert(dir_in_tar, files); @@ -231,6 +236,7 @@ impl Backup { let enc = GzEncoder::new(tar_gz, Compression::default()); let mut ar = tar::Builder::new(enc); + // TODO remove unwrap let previous_state = previous.state().unwrap(); let mut delta = BackupDelta::new(); @@ -238,12 +244,14 @@ impl Backup { let files = files(src_dir.clone())?; let added_files = files .iter() - .filter(|p| modified_since(previous.start_time, p)) + // This explicit negation is because we wish to also include files for which we + // couldn't determine the last modified time + .filter(|p| !not_modified_since(previous.start_time, src_dir.join(p))) .cloned() .collect::>(); for path in added_files.iter() { - ar.append_path_with_name(dir_in_tar.join(path), src_dir.join(path))?; + ar.append_path_with_name(src_dir.join(path), dir_in_tar.join(path))?; } delta.added.insert(dir_in_tar.clone(), added_files); @@ -295,16 +303,13 @@ impl BackupManager { (PathBuf::from("worlds"), self.world_dir.clone()), ]; - if let Some(last_backup) = &self.last_backup { - let clone = last_backup.clone(); - self.last_backup = Some(Arc::new(Backup::create_from( - clone, - &self.backup_dir, - dirs, - )?)); + let backup = if let Some(last_backup) = &self.last_backup { + Backup::create_from(Arc::clone(last_backup), &self.backup_dir, dirs)? } else { - self.last_backup = Some(Arc::new(Backup::create(&self.backup_dir, dirs)?)); - } + Backup::create(&self.backup_dir, dirs)? + }; + + self.last_backup = Some(Arc::new(backup)); Ok(()) } From 8add96b39bb0050a21e4ed16d66f9781005e9479 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 15 Jun 2023 20:36:46 +0200 Subject: [PATCH 09/19] feat: persistently store backup state --- Cargo.lock | 46 ++++++++++++++++++++++++++++++ Cargo.toml | 5 ++-- src/server/backups.rs | 66 ++++++++++++++++++++++++++++++++++++++++--- src/server/command.rs | 13 +++++---- src/server/process.rs | 25 ++-------------- 5 files changed, 121 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e798dd..5ba48c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,8 @@ dependencies = [ "chrono", "clap", "flate2", + "serde", + "serde_json", "signal-hook", "tar", ] @@ -123,6 +125,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "time", "wasm-bindgen", "winapi", @@ -292,6 +295,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + [[package]] name = "js-sys" version = "0.3.63" @@ -384,6 +393,43 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "serde" +version = "1.0.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "signal-hook" version = "0.3.15" diff --git a/Cargo.toml b/Cargo.toml index 2c4045b..2ac6b39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,10 +12,11 @@ edition = "2021" tar = "0.4.38" # Used to compress said tarballs using gzip flate2 = "1.0.26" -# Used for backup filenames -chrono = "0.4.26" +chrono = { version = "0.4.26", features = ["serde"] } clap = { version = "4.3.1", features = ["derive", "env"] } signal-hook = "0.3.15" +serde = { version = "1.0.164", features = ["derive", "rc"] } +serde_json = "1.0.96" [profile.release] lto = "fat" diff --git a/src/server/backups.rs b/src/server/backups.rs index 2620e17..8012b5f 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -1,6 +1,7 @@ use chrono::{Local, Utc}; use flate2::write::GzEncoder; use flate2::Compression; +use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io; @@ -60,7 +61,7 @@ fn not_modified_since>(time: chrono::DateTime, path: T) -> b false } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Serialize, Deserialize)] pub enum BackupType { Full, Incremental, @@ -74,7 +75,7 @@ pub enum BackupError { type BackupResult = Result; /// Represents the changes relative to the previous backup -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct BackupDelta { /// What files were added/modified in each part of the tarball. pub added: HashMap>, @@ -146,8 +147,9 @@ impl BackupDelta { } /// Represents a successful backup -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct Backup { + #[serde(skip)] previous: Option>, /// When the backup was started (also corresponds to the name) start_time: chrono::DateTime, @@ -174,6 +176,11 @@ impl Backup { return Err(BackupError::NoFullAncestor); } } + + pub fn set_previous(&mut self, previous: Arc) { + self.previous = Some(previous); + } + /// Create a new Full backup, populated with the given directories. /// /// # Arguments @@ -282,7 +289,9 @@ pub struct BackupManager { } impl BackupManager { - pub fn open( + const METADATA_FILE: &str = "alex.json"; + + pub fn new( backup_dir: PathBuf, config_dir: PathBuf, world_dir: PathBuf, @@ -297,6 +306,18 @@ impl BackupManager { } } + pub fn open( + backup_dir: PathBuf, + config_dir: PathBuf, + world_dir: PathBuf, + max_backups: u64, + ) -> std::io::Result { + let mut manager = Self::new(backup_dir, config_dir, world_dir, max_backups); + manager.load_json()?; + + Ok(manager) + } + pub fn create_backup(&mut self) -> io::Result<()> { let dirs = vec![ (PathBuf::from("config"), self.config_dir.clone()), @@ -310,6 +331,7 @@ impl BackupManager { }; self.last_backup = Some(Arc::new(backup)); + self.write_json()?; Ok(()) } @@ -337,4 +359,40 @@ impl BackupManager { Ok(()) } + + pub fn write_json(&self) -> std::io::Result<()> { + // Put the backup chain into a list that can be serialized + let mut backups: Vec> = Vec::new(); + let mut backup_opt = &self.last_backup; + + while let Some(backup) = backup_opt { + backups.insert(0, Arc::clone(backup)); + backup_opt = &backup.previous; + } + + let json_file = File::create(self.backup_dir.join(Self::METADATA_FILE))?; + serde_json::to_writer(json_file, &backups)?; + + Ok(()) + } + + pub fn load_json(&mut self) -> std::io::Result<()> { + let json_file = File::open(self.backup_dir.join(Self::METADATA_FILE))?; + let mut backups: Vec> = serde_json::from_reader(json_file)?; + + if !backups.is_empty() { + for i in 1..backups.len() { + let previous = Arc::clone(&backups[i - 1]); + // We can unwrap here, as this function creates the first instance of each Arc, + // meaning we're definitely the only pointer. + Arc::get_mut(&mut backups[i]) + .unwrap() + .set_previous(previous); + } + + self.last_backup = Some(Arc::clone(backups.last().unwrap())); + } + + Ok(()) + } } diff --git a/src/server/command.rs b/src/server/command.rs index 641c6b8..1c44d5b 100644 --- a/src/server/command.rs +++ b/src/server/command.rs @@ -1,4 +1,4 @@ -use crate::server::ServerProcess; +use crate::server::{BackupManager, ServerProcess}; use clap::ValueEnum; use std::fmt; use std::fs::File; @@ -179,6 +179,12 @@ impl ServerCommand { } pub fn spawn(&mut self) -> std::io::Result { + let manager = BackupManager::open( + self.backup_dir.clone(), + self.config_dir.clone(), + self.world_dir.clone(), + self.max_backups, + )?; let mut cmd = self.create_cmd(); self.accept_eula()?; let child = cmd.spawn()?; @@ -186,10 +192,7 @@ impl ServerCommand { Ok(ServerProcess::new( self.type_, self.version.clone(), - self.config_dir.clone(), - self.world_dir.clone(), - self.backup_dir.clone(), - self.max_backups, + manager, child, )) } diff --git a/src/server/process.rs b/src/server/process.rs index 6edc484..fbd806a 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -1,18 +1,11 @@ use crate::server::BackupManager; use crate::server::ServerType; -use flate2::write::GzEncoder; -use flate2::Compression; use std::io::Write; -use std::path::{Path, PathBuf}; use std::process::Child; pub struct ServerProcess { type_: ServerType, version: String, - config_dir: PathBuf, - world_dir: PathBuf, - backup_dir: PathBuf, - max_backups: u64, child: Child, backups: BackupManager, } @@ -21,28 +14,14 @@ impl ServerProcess { pub fn new( type_: ServerType, version: String, - config_dir: PathBuf, - world_dir: PathBuf, - backup_dir: PathBuf, - max_backups: u64, + manager: BackupManager, child: Child, ) -> ServerProcess { - let backup_manager = BackupManager::open( - backup_dir.clone(), - config_dir.clone(), - world_dir.clone(), - max_backups, - ); - ServerProcess { type_, version, - config_dir, - world_dir, - backup_dir, - max_backups, child, - backups: backup_manager, + backups: manager, } } From 27d7e681c3d9d00e90b21bd7c9fd8985183ea54c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 15 Jun 2023 22:54:17 +0200 Subject: [PATCH 10/19] feat: temporarily disable "remove old backups" --- src/server/backups.rs | 102 ++++++++++++++++++++++++++++-------------- src/server/process.rs | 6 +-- 2 files changed, 71 insertions(+), 37 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index 8012b5f..e225aec 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -14,8 +14,7 @@ extern "C" { fn getegid() -> u32; } -const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; - +/// List all files in `src_dir` and all child directories. fn files(src_dir: PathBuf) -> io::Result> { let mut dirs = vec![src_dir.clone()]; let mut files: HashSet = HashSet::new(); @@ -159,7 +158,18 @@ pub struct Backup { } impl Backup { - /// Calculate the full state of the backup by applying all its ancestors delta's in order, + const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; + + /// Returns a pointer to this backup's previous backup by cloning the Arc pointer. + pub fn previous(&self) -> Option> { + if let Some(previous) = &self.previous { + Some(Arc::clone(&previous)) + } else { + None + } + } + + /// Calculate the full state of the backup by applying all its ancestors' delta's in order, /// starting from the last full ancestor. pub fn state(&self) -> BackupResult>> { if self.type_ == BackupType::Full { @@ -177,8 +187,27 @@ impl Backup { } } - pub fn set_previous(&mut self, previous: Arc) { - self.previous = Some(previous); + /// Returns the n'th ancestor of the given backup, if it exists. + pub fn ancestor(&self, n: u64) -> Option> { + if n == 0 { + None + } else if let Some(previous) = &self.previous { + if n == 1 { + Some(Arc::clone(&previous)) + } else { + previous.ancestor(n - 1) + } + } else { + None + } + } + + /// Return the path to a backup file by properly formatting the data. + pub fn path>(backup_dir: P, start_time: chrono::DateTime) -> PathBuf { + let backup_dir = backup_dir.as_ref(); + + let filename = format!("{}", start_time.format(Self::FILENAME_FORMAT)); + backup_dir.join(filename) } /// Create a new Full backup, populated with the given directories. @@ -196,11 +225,9 @@ impl Backup { backup_dir: P, dirs: Vec<(PathBuf, PathBuf)>, ) -> io::Result { - let backup_dir = backup_dir.as_ref(); let start_time = chrono::offset::Utc::now(); - let filename = format!("{}", start_time.format(FILENAME_FORMAT)); - let path = backup_dir.join(filename); + let path = Self::path(backup_dir, start_time); let tar_gz = File::create(path)?; let enc = GzEncoder::new(tar_gz, Compression::default()); let mut ar = tar::Builder::new(enc); @@ -234,17 +261,16 @@ impl Backup { backup_dir: P, dirs: Vec<(PathBuf, PathBuf)>, ) -> io::Result { - let backup_dir = backup_dir.as_ref(); let start_time = chrono::offset::Utc::now(); - let filename = format!("{}", start_time.format(FILENAME_FORMAT)); - let path = backup_dir.join(filename); + let path = Self::path(backup_dir, start_time); let tar_gz = File::create(path)?; let enc = GzEncoder::new(tar_gz, Compression::default()); let mut ar = tar::Builder::new(enc); - // TODO remove unwrap - let previous_state = previous.state().unwrap(); + let previous_state = previous + .state() + .map_err(|_| io::Error::new(io::ErrorKind::Other, "No Full ancestor"))?; let mut delta = BackupDelta::new(); for (dir_in_tar, src_dir) in dirs { @@ -337,28 +363,38 @@ impl BackupManager { } /// Remove the oldest backups - pub fn remove_old_backups(&mut self) -> std::io::Result<()> { - // The naming format used allows us to sort the backups by name and still get a sorting by - // creation time - let mut backups = self - .backup_dir - .read_dir()? - .filter_map(|res| res.map(|e| e.path()).ok()) - .collect::>(); - backups.sort(); + // pub fn remove_old_backups(&mut self) -> std::io::Result<()> { + // if let Some(last_backup) = &self.last_backup { + // let last_valid_ancestor = last_backup.ancestor(self.max_backups - 1); + // let ancestor = last_valid_ancestor.previous(); - let max_backups: usize = self.max_backups.try_into().unwrap(); + // while let Some(backup) = &ancestor { + // let path = Backup::path(&self.backup_dir, backup.start_time); + // std::fs::remove_file(path)?; + // } + // } - if backups.len() > max_backups { - let excess_backups = backups.len() - max_backups; + // // The naming format used allows us to sort the backups by name and still get a sorting by + // // creation time + // let mut backups = self + // .backup_dir + // .read_dir()? + // .filter_map(|res| res.map(|e| e.path()).ok()) + // .collect::>(); + // backups.sort(); - for backup in &backups[0..excess_backups] { - std::fs::remove_file(backup)?; - } - } + // let max_backups: usize = self.max_backups.try_into().unwrap(); - Ok(()) - } + // if backups.len() > max_backups { + // let excess_backups = backups.len() - max_backups; + + // for backup in &backups[0..excess_backups] { + // std::fs::remove_file(backup)?; + // } + // } + + // Ok(()) + // } pub fn write_json(&self) -> std::io::Result<()> { // Put the backup chain into a list that can be serialized @@ -385,9 +421,7 @@ impl BackupManager { let previous = Arc::clone(&backups[i - 1]); // We can unwrap here, as this function creates the first instance of each Arc, // meaning we're definitely the only pointer. - Arc::get_mut(&mut backups[i]) - .unwrap() - .set_previous(previous); + Arc::get_mut(&mut backups[i]).unwrap().previous = Some(previous); } self.last_backup = Some(Arc::clone(backups.last().unwrap())); diff --git a/src/server/process.rs b/src/server/process.rs index fbd806a..d784061 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -70,9 +70,9 @@ impl ServerProcess { let start_time = chrono::offset::Utc::now(); let res = self.backups.create_backup(); - if res.is_ok() { - self.backups.remove_old_backups()?; - } + // if res.is_ok() { + // self.backups.remove_old_backups()?; + // } // The server's save feature needs to be enabled again even if the archive failed to create self.custom("save-on")?; From 527535635357d574b1d6e042a5a78271b6a9aecf Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 16 Jun 2023 17:23:36 +0200 Subject: [PATCH 11/19] feat: added backup cli command --- CHANGELOG.md | 4 ++++ src/cli.rs | 12 ++++++++++++ src/main.rs | 14 +++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e431581..e2ec666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased](https://git.rustybever.be/Chewing_Bever/alex/src/branch/dev) +### Added + +* Added `backup` CLI command + ### Changed * Running the server now uses the `run` CLI subcommand diff --git a/src/cli.rs b/src/cli.rs index 1acbe36..3c588aa 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -50,6 +50,9 @@ pub struct Cli { pub enum Commands { /// Run the server Run(RunArgs), + /// Create a new backup of the server. This command should only be used when the server is not + /// running. + Backup(BackupArgs), } #[derive(Args)] @@ -89,3 +92,12 @@ pub struct RunArgs { #[arg(short, long, default_value_t = false)] pub dry: bool, } + +#[derive(Args)] +pub struct BackupArgs { + /// Type of server + pub type_: ServerType, + /// Version string for the server, e.g. 1.19.4-545 + #[arg(env = "ALEX_SERVER_VERSION")] + pub server_version: String, +} diff --git a/src/main.rs b/src/main.rs index a1ae21c..3d82d11 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ mod signals; mod stdin; use clap::Parser; -use cli::{Cli, Commands, RunArgs}; +use cli::{BackupArgs, Cli, Commands, RunArgs}; use std::io; use std::sync::{Arc, Mutex}; @@ -57,10 +57,22 @@ fn command_run(cli: &Cli, args: &RunArgs) -> io::Result<()> { signals::handle_signals(&mut signals, counter) } +fn commands_backup(cli: &Cli, _args: &BackupArgs) -> io::Result<()> { + let mut manager = server::BackupManager::open( + cli.backup.clone(), + cli.config.clone(), + cli.world.clone(), + cli.max_backups, + )?; + + manager.create_backup() +} + fn main() -> io::Result<()> { let cli = Cli::parse(); match &cli.command { Commands::Run(args) => command_run(&cli, args), + Commands::Backup(args) => commands_backup(&cli, args), } } From f7235fb34243767711d42767c912055a47ce7452 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 17 Jun 2023 12:08:46 +0200 Subject: [PATCH 12/19] refactor: move iterating over files to Path extension trait --- src/server/backups.rs | 98 ++++++++-------------------- src/server/path.rs | 148 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 164 insertions(+), 82 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index e225aec..cdb0409 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -1,3 +1,4 @@ +use crate::server::path::PathExt; use chrono::{Local, Utc}; use flate2::write::GzEncoder; use flate2::Compression; @@ -14,52 +15,6 @@ extern "C" { fn getegid() -> u32; } -/// List all files in `src_dir` and all child directories. -fn files(src_dir: PathBuf) -> io::Result> { - let mut dirs = vec![src_dir.clone()]; - let mut files: HashSet = HashSet::new(); - - while let Some(dir) = dirs.pop() { - for res in dir.read_dir()? { - let entry = res?; - - if entry.file_name() == "cache" { - continue; - } - - if entry.file_type()?.is_dir() { - dirs.push(entry.path()); - } else { - files.insert(entry.path().strip_prefix(&src_dir).unwrap().to_path_buf()); - } - } - } - - Ok(files) -} - -/// Check whether a file has been modified since the given timestamp. -/// -/// Note that this function will *only* return true if it can determine with certainty that the -/// file has not been modified. If any errors occur while obtaining the required metadata (e.g. if -/// the file system does not support this metadata), this function will return false. -fn not_modified_since>(time: chrono::DateTime, path: T) -> bool { - let path = path.as_ref(); - - if let Ok(metadata) = path.metadata() { - let last_modified = metadata.modified(); - - if let Ok(last_modified) = last_modified { - let t: chrono::DateTime = last_modified.into(); - let t = t.with_timezone(&Local); - - return t < time; - } - } - - false -} - #[derive(Debug, PartialEq, Serialize, Deserialize)] pub enum BackupType { Full, @@ -162,11 +117,7 @@ impl Backup { /// Returns a pointer to this backup's previous backup by cloning the Arc pointer. pub fn previous(&self) -> Option> { - if let Some(previous) = &self.previous { - Some(Arc::clone(&previous)) - } else { - None - } + self.previous.as_ref().map(Arc::clone) } /// Calculate the full state of the backup by applying all its ancestors' delta's in order, @@ -193,7 +144,7 @@ impl Backup { None } else if let Some(previous) = &self.previous { if n == 1 { - Some(Arc::clone(&previous)) + Some(Arc::clone(previous)) } else { previous.ancestor(n - 1) } @@ -232,26 +183,27 @@ impl Backup { let enc = GzEncoder::new(tar_gz, Compression::default()); let mut ar = tar::Builder::new(enc); - let mut added: HashMap> = HashMap::new(); + let mut delta = BackupDelta::new(); for (dir_in_tar, src_dir) in dirs { - let files = files(src_dir.clone())?; + let mut added_files: HashSet = HashSet::new(); - for path in &files { - ar.append_path_with_name(src_dir.join(path), dir_in_tar.join(path))?; + for entry in src_dir.read_dir_recursive()?.ignored("cache").files() { + let path = entry?.path(); + let stripped = path.strip_prefix(&src_dir).unwrap(); + + ar.append_path_with_name(&path, dir_in_tar.join(stripped))?; + added_files.insert(stripped.to_path_buf()); } - added.insert(dir_in_tar, files); + delta.added.insert(dir_in_tar, added_files); } Ok(Backup { previous: None, type_: BackupType::Full, start_time, - delta: BackupDelta { - added, - removed: HashMap::new(), - }, + delta, }) } @@ -274,17 +226,19 @@ impl Backup { let mut delta = BackupDelta::new(); for (dir_in_tar, src_dir) in dirs { - let files = files(src_dir.clone())?; - let added_files = files - .iter() - // This explicit negation is because we wish to also include files for which we - // couldn't determine the last modified time - .filter(|p| !not_modified_since(previous.start_time, src_dir.join(p))) - .cloned() - .collect::>(); + let mut all_files: HashSet = HashSet::new(); + let mut added_files: HashSet = HashSet::new(); - for path in added_files.iter() { - ar.append_path_with_name(src_dir.join(path), dir_in_tar.join(path))?; + for entry in src_dir.read_dir_recursive()?.ignored("cache").files() { + let path = entry?.path(); + let stripped = path.strip_prefix(&src_dir).unwrap(); + + if !path.not_modified_since(previous.start_time) { + ar.append_path_with_name(&path, dir_in_tar.join(stripped))?; + added_files.insert(stripped.to_path_buf()); + } + + all_files.insert(stripped.to_path_buf()); } delta.added.insert(dir_in_tar.clone(), added_files); @@ -292,7 +246,7 @@ impl Backup { if let Some(previous_files) = previous_state.get(&dir_in_tar) { delta.removed.insert( dir_in_tar, - previous_files.difference(&files).cloned().collect(), + previous_files.difference(&all_files).cloned().collect(), ); } } diff --git a/src/server/path.rs b/src/server/path.rs index d9df799..b8b5ae9 100644 --- a/src/server/path.rs +++ b/src/server/path.rs @@ -1,19 +1,147 @@ -use chrono::Utc; +use chrono::{Local, Utc}; use std::collections::HashSet; -use std::path::PathBuf; +use std::ffi::OsString; +use std::fs::DirEntry; +use std::path::{Path, PathBuf}; use std::{fs, io}; -struct ReadDirRecursive { - ignored_dirs: HashSet, - read_dir: Option, - stack: Vec, +pub struct ReadDirRecursive { + ignored: HashSet, + read_dir: fs::ReadDir, + dir_stack: Vec, + files_only: bool, } impl ReadDirRecursive { - // pub fn new() + /// Start the iterator for a new directory + pub fn start>(path: P) -> io::Result { + let path = path.as_ref(); + let read_dir = path.read_dir()?; + + Ok(ReadDirRecursive { + ignored: HashSet::new(), + read_dir, + dir_stack: Vec::new(), + files_only: false, + }) + } + + pub fn ignored>(mut self, s: S) -> Self { + self.ignored.insert(s.into()); + + self + } + + pub fn files(mut self) -> Self { + self.files_only = true; + + self + } + + /// Tries to populate the `read_dir` field with a new `ReadDir` instance to consume. + fn next_read_dir(&mut self) -> io::Result { + if let Some(path) = self.dir_stack.pop() { + self.read_dir = path.read_dir()?; + + Ok(true) + } else { + Ok(false) + } + } + + /// Convenience method to add a new directory to the stack. + fn push_entry(&mut self, entry: &io::Result) { + if let Ok(entry) = entry { + if entry.path().is_dir() { + self.dir_stack.push(entry.path()); + } + } + } + + /// Determine whether an entry should be returned by the iterator. + fn should_return(&self, entry: &io::Result) -> bool { + if let Ok(entry) = entry { + let mut res = !self.ignored.contains(&entry.file_name()); + + // Please just let me combine these already + if self.files_only { + if let Ok(file_type) = entry.file_type() { + res = res && file_type.is_file(); + } + // We couldn't determine if it's a file, so we don't return it + else { + res = false; + } + } + + res + } else { + true + } + } } -trait PathExt { - fn modified_since(timestamp: chrono::DateTime) -> bool; - fn read_dir_recusive() -> ReadDirRecursive; +impl Iterator for ReadDirRecursive { + type Item = io::Result; + + fn next(&mut self) -> Option { + loop { + // First, we try to consume the current directory's items + while let Some(entry) = self.read_dir.next() { + self.push_entry(&entry); + + if self.should_return(&entry) { + return Some(entry); + } + } + + // If we get an error while setting up a new directory, we return this, otherwise we + // keep trying to consume the directories + match self.next_read_dir() { + Ok(true) => (), + // There's no more directories to traverse, so the iterator is done + Ok(false) => return None, + Err(e) => return Some(Err(e)), + } + } + } +} + +pub trait PathExt { + /// Confirm whether the file has not been modified since the given timestamp. + /// + /// This function will only return true if it can determine with certainty that the file hasn't + /// been modified. + /// + /// # Args + /// + /// * `timestamp` - Timestamp to compare modified time with + /// + /// # Returns + /// + /// True if the file has not been modified for sure, false otherwise. + fn not_modified_since(&self, timestamp: chrono::DateTime) -> bool; + + /// An extension of the `read_dir` command that runs through the entire underlying directory + /// structure using breadth-first search + fn read_dir_recursive(&self) -> io::Result; +} + +impl PathExt for Path { + fn not_modified_since(&self, timestamp: chrono::DateTime) -> bool { + if let Ok(metadata) = self.metadata() { + if let Ok(last_modified) = metadata.modified() { + let t: chrono::DateTime = last_modified.into(); + let t = t.with_timezone(&Local); + + return t < timestamp; + } + } + + false + } + + fn read_dir_recursive(&self) -> io::Result { + ReadDirRecursive::start(self) + } } From bb7b57899b9ab6bcdfa72642c05b7e6a23123c67 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 18 Jun 2023 21:15:05 +0200 Subject: [PATCH 13/19] refactor: store backups in nested vecs instead; introduce concept of chains --- src/server/backups.rs | 114 +++++++++++++++++------------------------- 1 file changed, 47 insertions(+), 67 deletions(-) diff --git a/src/server/backups.rs b/src/server/backups.rs index cdb0409..3fcf4b1 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -103,8 +103,6 @@ impl BackupDelta { /// Represents a successful backup #[derive(Debug, Serialize, Deserialize)] pub struct Backup { - #[serde(skip)] - previous: Option>, /// When the backup was started (also corresponds to the name) start_time: chrono::DateTime, /// Type of the backup @@ -115,42 +113,14 @@ pub struct Backup { impl Backup { const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; - /// Returns a pointer to this backup's previous backup by cloning the Arc pointer. - pub fn previous(&self) -> Option> { - self.previous.as_ref().map(Arc::clone) - } + pub fn state(backups: &Vec) -> HashMap> { + let mut state: HashMap> = HashMap::new(); - /// Calculate the full state of the backup by applying all its ancestors' delta's in order, - /// starting from the last full ancestor. - pub fn state(&self) -> BackupResult>> { - if self.type_ == BackupType::Full { - let mut state = HashMap::new(); - self.delta.apply(&mut state); - - Ok(state) - } else if let Some(previous) = &self.previous { - let mut state = previous.state()?; - self.delta.apply(&mut state); - - Ok(state) - } else { - return Err(BackupError::NoFullAncestor); + for backup in backups { + backup.delta.apply(&mut state); } - } - /// Returns the n'th ancestor of the given backup, if it exists. - pub fn ancestor(&self, n: u64) -> Option> { - if n == 0 { - None - } else if let Some(previous) = &self.previous { - if n == 1 { - Some(Arc::clone(previous)) - } else { - previous.ancestor(n - 1) - } - } else { - None - } + state } /// Return the path to a backup file by properly formatting the data. @@ -200,7 +170,6 @@ impl Backup { } Ok(Backup { - previous: None, type_: BackupType::Full, start_time, delta, @@ -209,7 +178,8 @@ impl Backup { /// Create a new incremental backup from a given previous backup pub fn create_from>( - previous: Arc, + previous_state: HashMap>, + previous_start_time: chrono::DateTime, backup_dir: P, dirs: Vec<(PathBuf, PathBuf)>, ) -> io::Result { @@ -220,9 +190,6 @@ impl Backup { let enc = GzEncoder::new(tar_gz, Compression::default()); let mut ar = tar::Builder::new(enc); - let previous_state = previous - .state() - .map_err(|_| io::Error::new(io::ErrorKind::Other, "No Full ancestor"))?; let mut delta = BackupDelta::new(); for (dir_in_tar, src_dir) in dirs { @@ -233,7 +200,7 @@ impl Backup { let path = entry?.path(); let stripped = path.strip_prefix(&src_dir).unwrap(); - if !path.not_modified_since(previous.start_time) { + if !path.not_modified_since(previous_start_time) { ar.append_path_with_name(&path, dir_in_tar.join(stripped))?; added_files.insert(stripped.to_path_buf()); } @@ -252,7 +219,6 @@ impl Backup { } Ok(Backup { - previous: Some(previous), type_: BackupType::Incremental, start_time, delta, @@ -264,8 +230,10 @@ pub struct BackupManager { backup_dir: PathBuf, config_dir: PathBuf, world_dir: PathBuf, + max_chain_length: u64, max_backups: u64, last_backup: Option>, + chains: Vec>, } impl BackupManager { @@ -282,6 +250,8 @@ impl BackupManager { config_dir, world_dir, max_backups, + max_chain_length: 2, + chains: Vec::new(), last_backup: None, } } @@ -304,13 +274,33 @@ impl BackupManager { (PathBuf::from("worlds"), self.world_dir.clone()), ]; - let backup = if let Some(last_backup) = &self.last_backup { - Backup::create_from(Arc::clone(last_backup), &self.backup_dir, dirs)? + // I kinda hate this statement, please just let me combine let statements in if statements + // already + let backup = if let Some(current_chain) = self.chains.last() { + let current_chain_len: u64 = current_chain.len().try_into().unwrap(); + + if current_chain_len < self.max_chain_length { + if let Some(previous_backup) = current_chain.last() { + let state = Backup::state(current_chain); + + Backup::create_from(state, previous_backup.start_time, &self.backup_dir, dirs)? + } else { + Backup::create(&self.backup_dir, dirs)? + } + } else { + self.chains.push(Vec::new()); + + Backup::create(&self.backup_dir, dirs)? + } } else { + self.chains.push(Vec::new()); + Backup::create(&self.backup_dir, dirs)? }; - self.last_backup = Some(Arc::new(backup)); + // The above statement always creates this element, so this unwrap is safe + self.chains.last_mut().unwrap().push(backup); + self.write_json()?; Ok(()) @@ -351,35 +341,25 @@ impl BackupManager { // } pub fn write_json(&self) -> std::io::Result<()> { - // Put the backup chain into a list that can be serialized - let mut backups: Vec> = Vec::new(); - let mut backup_opt = &self.last_backup; - - while let Some(backup) = backup_opt { - backups.insert(0, Arc::clone(backup)); - backup_opt = &backup.previous; - } - let json_file = File::create(self.backup_dir.join(Self::METADATA_FILE))?; - serde_json::to_writer(json_file, &backups)?; + serde_json::to_writer(json_file, &self.chains)?; Ok(()) } pub fn load_json(&mut self) -> std::io::Result<()> { - let json_file = File::open(self.backup_dir.join(Self::METADATA_FILE))?; - let mut backups: Vec> = serde_json::from_reader(json_file)?; - - if !backups.is_empty() { - for i in 1..backups.len() { - let previous = Arc::clone(&backups[i - 1]); - // We can unwrap here, as this function creates the first instance of each Arc, - // meaning we're definitely the only pointer. - Arc::get_mut(&mut backups[i]).unwrap().previous = Some(previous); + let json_file = match File::open(self.backup_dir.join(Self::METADATA_FILE)) { + Ok(f) => f, + Err(e) => { + // Don't error out if the file isn't there, it will be created when necessary + if e.kind() == io::ErrorKind::NotFound { + return Ok(()); + } else { + return Err(e); + } } - - self.last_backup = Some(Arc::clone(backups.last().unwrap())); - } + }; + self.chains = serde_json::from_reader(json_file)?; Ok(()) } From b51d951688684da09b9535b5abb719de019a34f5 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 18 Jun 2023 21:56:43 +0200 Subject: [PATCH 14/19] feat: re-implement remove old backups --- src/main.rs | 3 ++- src/server/backups.rs | 54 +++++++++++++++++++------------------------ src/server/process.rs | 9 ++++---- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3d82d11..c057db2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,7 +65,8 @@ fn commands_backup(cli: &Cli, _args: &BackupArgs) -> io::Result<()> { cli.max_backups, )?; - manager.create_backup() + manager.create_backup()?; + manager.remove_old_backups() } fn main() -> io::Result<()> { diff --git a/src/server/backups.rs b/src/server/backups.rs index 3fcf4b1..82d56a7 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -230,7 +230,8 @@ pub struct BackupManager { backup_dir: PathBuf, config_dir: PathBuf, world_dir: PathBuf, - max_chain_length: u64, + chain_len: u64, + chains_to_keep: u64, max_backups: u64, last_backup: Option>, chains: Vec>, @@ -250,7 +251,8 @@ impl BackupManager { config_dir, world_dir, max_backups, - max_chain_length: 2, + chain_len: 2, + chains_to_keep: 1, chains: Vec::new(), last_backup: None, } @@ -279,7 +281,7 @@ impl BackupManager { let backup = if let Some(current_chain) = self.chains.last() { let current_chain_len: u64 = current_chain.len().try_into().unwrap(); - if current_chain_len < self.max_chain_length { + if current_chain_len < self.chain_len { if let Some(previous_backup) = current_chain.last() { let state = Backup::state(current_chain); @@ -307,38 +309,30 @@ impl BackupManager { } /// Remove the oldest backups - // pub fn remove_old_backups(&mut self) -> std::io::Result<()> { - // if let Some(last_backup) = &self.last_backup { - // let last_valid_ancestor = last_backup.ancestor(self.max_backups - 1); - // let ancestor = last_valid_ancestor.previous(); + pub fn remove_old_backups(&mut self) -> std::io::Result<()> { + let chains_to_store: usize = self.chains_to_keep.try_into().unwrap(); - // while let Some(backup) = &ancestor { - // let path = Backup::path(&self.backup_dir, backup.start_time); - // std::fs::remove_file(path)?; - // } - // } + if chains_to_store < self.chains.len() { + let mut remove_count: usize = self.chains.len() - chains_to_store; - // // The naming format used allows us to sort the backups by name and still get a sorting by - // // creation time - // let mut backups = self - // .backup_dir - // .read_dir()? - // .filter_map(|res| res.map(|e| e.path()).ok()) - // .collect::>(); - // backups.sort(); + // We only count finished chains towards the list of stored chains + let chain_len: usize = self.chain_len.try_into().unwrap(); + if self.chains.last().unwrap().len() < chain_len { + remove_count -= 1; + } - // let max_backups: usize = self.max_backups.try_into().unwrap(); + for chain in self.chains.drain(..remove_count) { + for backup in chain { + let path = Backup::path(&self.backup_dir, backup.start_time); + std::fs::remove_file(path)?; + } + } + } - // if backups.len() > max_backups { - // let excess_backups = backups.len() - max_backups; + self.write_json()?; - // for backup in &backups[0..excess_backups] { - // std::fs::remove_file(backup)?; - // } - // } - - // Ok(()) - // } + Ok(()) + } pub fn write_json(&self) -> std::io::Result<()> { let json_file = File::create(self.backup_dir.join(Self::METADATA_FILE))?; diff --git a/src/server/process.rs b/src/server/process.rs index d784061..1e24373 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -70,12 +70,13 @@ impl ServerProcess { let start_time = chrono::offset::Utc::now(); let res = self.backups.create_backup(); - // if res.is_ok() { - // self.backups.remove_old_backups()?; - // } - // The server's save feature needs to be enabled again even if the archive failed to create self.custom("save-on")?; + self.custom("save-all")?; + + if res.is_ok() { + self.backups.remove_old_backups()?; + } let duration = chrono::offset::Utc::now() - start_time; let duration_str = format!( From b48c531d80d08b0772e264bf1249fc653fe90d48 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 18 Jun 2023 22:45:35 +0200 Subject: [PATCH 15/19] feat: configurable parameters for incremental backups --- CHANGELOG.md | 10 +++++++++- src/cli.rs | 15 ++++++++++++--- src/main.rs | 10 ++++++---- src/server/backups.rs | 29 +++++++++++------------------ src/server/command.rs | 18 +++++++++++++----- 5 files changed, 51 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2ec666..8157f9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -* Added `backup` CLI command +* `backup` CLI command +* Incremental backups + * Chain length descibres how many incremental backups to create from the + same full backup + * "backups to keep" has been replaced by "chains to keep" ### Changed * Running the server now uses the `run` CLI subcommand +### Removed + +* `max_backups` setting + ## [0.2.2](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.2) ### Fixed diff --git a/src/cli.rs b/src/cli.rs index 3c588aa..613dfa1 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -35,15 +35,24 @@ pub struct Cli { )] pub backup: PathBuf, - /// How many backups to keep + /// Length of a backup chain + #[arg( + short = 'l', + long, + default_value_t = 4, + env = "ALEX_CHAIN_LEN", + global = true + )] + pub chain_len: u64, + /// How many backup chains to keep #[arg( short = 'n', long, default_value_t = 7, - env = "ALEX_MAX_BACKUPS", + env = "ALEX_CHAINS", global = true )] - pub max_backups: u64, + pub chains: u64, } #[derive(Subcommand)] diff --git a/src/main.rs b/src/main.rs index c057db2..1898e90 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,7 +32,8 @@ fn command_run(cli: &Cli, args: &RunArgs) -> io::Result<()> { .backup(cli.backup.clone()) .xms(args.xms) .xmx(args.xmx) - .max_backups(cli.max_backups); + .chain_len(cli.chain_len) + .chains_to_keep(cli.chains); cmd.canonicalize()?; if args.dry { @@ -58,12 +59,13 @@ fn command_run(cli: &Cli, args: &RunArgs) -> io::Result<()> { } fn commands_backup(cli: &Cli, _args: &BackupArgs) -> io::Result<()> { - let mut manager = server::BackupManager::open( + let mut manager = server::BackupManager::new( cli.backup.clone(), cli.config.clone(), cli.world.clone(), - cli.max_backups, - )?; + cli.chain_len, + cli.chains, + ); manager.create_backup()?; manager.remove_old_backups() diff --git a/src/server/backups.rs b/src/server/backups.rs index 82d56a7..c8f235a 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -1,5 +1,5 @@ use crate::server::path::PathExt; -use chrono::{Local, Utc}; +use chrono::Utc; use flate2::write::GzEncoder; use flate2::Compression; use serde::{Deserialize, Serialize}; @@ -7,7 +7,6 @@ use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io; use std::path::{Path, PathBuf}; -use std::sync::Arc; #[link(name = "c")] extern "C" { @@ -21,13 +20,6 @@ pub enum BackupType { Incremental, } -#[derive(Debug)] -pub enum BackupError { - NoFullAncestor, -} - -type BackupResult = Result; - /// Represents the changes relative to the previous backup #[derive(Debug, Serialize, Deserialize)] pub struct BackupDelta { @@ -113,6 +105,8 @@ pub struct Backup { impl Backup { const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; + /// Resolve the state of the list of backups by applying their deltas in-order to an initially + /// empty state. pub fn state(backups: &Vec) -> HashMap> { let mut state: HashMap> = HashMap::new(); @@ -232,29 +226,27 @@ pub struct BackupManager { world_dir: PathBuf, chain_len: u64, chains_to_keep: u64, - max_backups: u64, - last_backup: Option>, chains: Vec>, } impl BackupManager { const METADATA_FILE: &str = "alex.json"; + /// Initialize a new instance of a `BackupManager`. pub fn new( backup_dir: PathBuf, config_dir: PathBuf, world_dir: PathBuf, - max_backups: u64, + chain_len: u64, + chains_to_keep: u64, ) -> Self { BackupManager { backup_dir, config_dir, world_dir, - max_backups, - chain_len: 2, - chains_to_keep: 1, + chain_len, + chains_to_keep, chains: Vec::new(), - last_backup: None, } } @@ -262,9 +254,10 @@ impl BackupManager { backup_dir: PathBuf, config_dir: PathBuf, world_dir: PathBuf, - max_backups: u64, + chain_len: u64, + chains_to_keep: u64, ) -> std::io::Result { - let mut manager = Self::new(backup_dir, config_dir, world_dir, max_backups); + let mut manager = Self::new(backup_dir, config_dir, world_dir, chain_len, chains_to_keep); manager.load_json()?; Ok(manager) diff --git a/src/server/command.rs b/src/server/command.rs index 1c44d5b..fa92804 100644 --- a/src/server/command.rs +++ b/src/server/command.rs @@ -35,7 +35,8 @@ pub struct ServerCommand { backup_dir: PathBuf, xms: u64, xmx: u64, - max_backups: u64, + chain_len: u64, + chains_to_keep: u64, } impl ServerCommand { @@ -50,7 +51,8 @@ impl ServerCommand { backup_dir: PathBuf::from("backups"), xms: 1024, xmx: 2048, - max_backups: 7, + chain_len: 4, + chains_to_keep: 7, } } @@ -91,8 +93,13 @@ impl ServerCommand { self } - pub fn max_backups(mut self, v: u64) -> Self { - self.max_backups = v; + pub fn chain_len(mut self, v: u64) -> Self { + self.chain_len = v; + self + } + + pub fn chains_to_keep(mut self, v: u64) -> Self { + self.chains_to_keep = v; self } @@ -183,7 +190,8 @@ impl ServerCommand { self.backup_dir.clone(), self.config_dir.clone(), self.world_dir.clone(), - self.max_backups, + self.chain_len, + self.chains_to_keep, )?; let mut cmd = self.create_cmd(); self.accept_eula()?; From 74a0b91fd13257e662e680c7047a3e0e3c002c41 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 18 Jun 2023 23:33:56 +0200 Subject: [PATCH 16/19] refactor: remove open function --- src/main.rs | 1 + src/server/backups.rs | 28 ++++++++++------------------ src/server/command.rs | 6 ++++-- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1898e90..0aef676 100644 --- a/src/main.rs +++ b/src/main.rs @@ -66,6 +66,7 @@ fn commands_backup(cli: &Cli, _args: &BackupArgs) -> io::Result<()> { cli.chain_len, cli.chains, ); + manager.load()?; manager.create_backup()?; manager.remove_old_backups() diff --git a/src/server/backups.rs b/src/server/backups.rs index c8f235a..5097746 100644 --- a/src/server/backups.rs +++ b/src/server/backups.rs @@ -250,19 +250,7 @@ impl BackupManager { } } - pub fn open( - backup_dir: PathBuf, - config_dir: PathBuf, - world_dir: PathBuf, - chain_len: u64, - chains_to_keep: u64, - ) -> std::io::Result { - let mut manager = Self::new(backup_dir, config_dir, world_dir, chain_len, chains_to_keep); - manager.load_json()?; - - Ok(manager) - } - + /// Create a new backup with the expected type. pub fn create_backup(&mut self) -> io::Result<()> { let dirs = vec![ (PathBuf::from("config"), self.config_dir.clone()), @@ -296,12 +284,12 @@ impl BackupManager { // The above statement always creates this element, so this unwrap is safe self.chains.last_mut().unwrap().push(backup); - self.write_json()?; + self.save()?; Ok(()) } - /// Remove the oldest backups + /// Delete all backups associated with outdated chains, and forget those chains. pub fn remove_old_backups(&mut self) -> std::io::Result<()> { let chains_to_store: usize = self.chains_to_keep.try_into().unwrap(); @@ -322,24 +310,28 @@ impl BackupManager { } } - self.write_json()?; + self.save()?; Ok(()) } - pub fn write_json(&self) -> std::io::Result<()> { + /// Write the in-memory state to disk. + pub fn save(&self) -> std::io::Result<()> { let json_file = File::create(self.backup_dir.join(Self::METADATA_FILE))?; serde_json::to_writer(json_file, &self.chains)?; Ok(()) } - pub fn load_json(&mut self) -> std::io::Result<()> { + /// Overwrite the in-memory state with the on-disk state. + pub fn load(&mut self) -> std::io::Result<()> { let json_file = match File::open(self.backup_dir.join(Self::METADATA_FILE)) { Ok(f) => f, Err(e) => { // Don't error out if the file isn't there, it will be created when necessary if e.kind() == io::ErrorKind::NotFound { + self.chains = Vec::new(); + return Ok(()); } else { return Err(e); diff --git a/src/server/command.rs b/src/server/command.rs index fa92804..bd1f0e6 100644 --- a/src/server/command.rs +++ b/src/server/command.rs @@ -186,13 +186,15 @@ impl ServerCommand { } pub fn spawn(&mut self) -> std::io::Result { - let manager = BackupManager::open( + let mut manager = BackupManager::new( self.backup_dir.clone(), self.config_dir.clone(), self.world_dir.clone(), self.chain_len, self.chains_to_keep, - )?; + ); + manager.load()?; + let mut cmd = self.create_cmd(); self.accept_eula()?; let child = cmd.spawn()?; From ef631fab1d6c35998cfedbf8c4714bd41deff981 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 19 Jun 2023 14:04:38 +0200 Subject: [PATCH 17/19] refactor: separate backup logic into own module --- src/backup/delta.rs | 75 +++++++ src/backup/manager.rs | 128 ++++++++++++ src/backup/mod.rs | 150 ++++++++++++++ src/{server => backup}/path.rs | 0 src/main.rs | 3 +- src/server/backups.rs | 345 --------------------------------- src/server/command.rs | 3 +- src/server/mod.rs | 3 - src/server/process.rs | 2 +- 9 files changed, 358 insertions(+), 351 deletions(-) create mode 100644 src/backup/delta.rs create mode 100644 src/backup/manager.rs create mode 100644 src/backup/mod.rs rename src/{server => backup}/path.rs (100%) delete mode 100644 src/server/backups.rs diff --git a/src/backup/delta.rs b/src/backup/delta.rs new file mode 100644 index 0000000..15f233b --- /dev/null +++ b/src/backup/delta.rs @@ -0,0 +1,75 @@ +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +/// Represents the changes relative to the previous backup +#[derive(Debug, Serialize, Deserialize)] +pub struct Delta { + /// What files were added/modified in each part of the tarball. + pub added: HashMap>, + /// What files were removed in this backup, in comparison to the previous backup. For full + /// backups, this will always be empty, as they do not consider previous backups. + /// The map stores a separate list for each top-level directory, as the contents of these + /// directories can come for different source directories. + pub removed: HashMap>, +} + +impl Delta { + pub fn new() -> Self { + Self { + added: HashMap::new(), + removed: HashMap::new(), + } + } + + /// Update the current state so that its result becomes the merge of itself and the other + /// state. + pub fn merge(&mut self, delta: &Self) { + for (dir, added) in delta.added.iter() { + // Files that were removed in the current state, but added in the new state, are no + // longer removed + if let Some(orig_removed) = self.removed.get_mut(dir) { + orig_removed.retain(|k| !added.contains(k)); + } + + // Newly added files are added to the state as well + if let Some(orig_added) = self.added.get_mut(dir) { + orig_added.extend(added.iter().cloned()); + } else { + self.added.insert(dir.clone(), added.clone()); + } + } + + for (dir, removed) in delta.removed.iter() { + // Files that were originally added, but now deleted are removed from the added list + if let Some(orig_added) = self.added.get_mut(dir) { + orig_added.retain(|k| !removed.contains(k)); + } + + // Newly removed files are added to the state as well + if let Some(orig_removed) = self.removed.get_mut(dir) { + orig_removed.extend(removed.iter().cloned()); + } else { + self.removed.insert(dir.clone(), removed.clone()); + } + } + } + + /// Modify the given state by applying this delta's changes to it + pub fn apply(&self, state: &mut HashMap>) { + // First we add new files, then we remove the old ones + for (dir, added) in self.added.iter() { + if let Some(current) = state.get_mut(dir) { + current.extend(added.iter().cloned()); + } else { + state.insert(dir.clone(), added.clone()); + } + } + + for (dir, removed) in self.removed.iter() { + if let Some(current) = state.get_mut(dir) { + current.retain(|k| !removed.contains(k)); + } + } + } +} diff --git a/src/backup/manager.rs b/src/backup/manager.rs new file mode 100644 index 0000000..71633a3 --- /dev/null +++ b/src/backup/manager.rs @@ -0,0 +1,128 @@ +use super::Backup; +use std::fs::File; +use std::io; +use std::path::PathBuf; + +pub struct Manager { + backup_dir: PathBuf, + config_dir: PathBuf, + world_dir: PathBuf, + chain_len: u64, + chains_to_keep: u64, + chains: Vec>, +} + +impl Manager { + const METADATA_FILE: &str = "alex.json"; + + /// Initialize a new instance of a `BackupManager`. + pub fn new( + backup_dir: PathBuf, + config_dir: PathBuf, + world_dir: PathBuf, + chain_len: u64, + chains_to_keep: u64, + ) -> Self { + Self { + backup_dir, + config_dir, + world_dir, + chain_len, + chains_to_keep, + chains: Vec::new(), + } + } + + /// Create a new backup with the expected type. + pub fn create_backup(&mut self) -> io::Result<()> { + let dirs = vec![ + (PathBuf::from("config"), self.config_dir.clone()), + (PathBuf::from("worlds"), self.world_dir.clone()), + ]; + + // I kinda hate this statement, please just let me combine let statements in if statements + // already + let backup = if let Some(current_chain) = self.chains.last() { + let current_chain_len: u64 = current_chain.len().try_into().unwrap(); + + if current_chain_len < self.chain_len { + if let Some(previous_backup) = current_chain.last() { + let state = Backup::state(current_chain); + + Backup::create_from(state, previous_backup.start_time, &self.backup_dir, dirs)? + } else { + Backup::create(&self.backup_dir, dirs)? + } + } else { + self.chains.push(Vec::new()); + + Backup::create(&self.backup_dir, dirs)? + } + } else { + self.chains.push(Vec::new()); + + Backup::create(&self.backup_dir, dirs)? + }; + + // The above statement always creates this element, so this unwrap is safe + self.chains.last_mut().unwrap().push(backup); + + self.save()?; + + Ok(()) + } + + /// Delete all backups associated with outdated chains, and forget those chains. + pub fn remove_old_backups(&mut self) -> std::io::Result<()> { + let chains_to_store: usize = self.chains_to_keep.try_into().unwrap(); + + if chains_to_store < self.chains.len() { + let mut remove_count: usize = self.chains.len() - chains_to_store; + + // We only count finished chains towards the list of stored chains + let chain_len: usize = self.chain_len.try_into().unwrap(); + if self.chains.last().unwrap().len() < chain_len { + remove_count -= 1; + } + + for chain in self.chains.drain(..remove_count) { + for backup in chain { + let path = Backup::path(&self.backup_dir, backup.start_time); + std::fs::remove_file(path)?; + } + } + } + + self.save()?; + + Ok(()) + } + + /// Write the in-memory state to disk. + pub fn save(&self) -> std::io::Result<()> { + let json_file = File::create(self.backup_dir.join(Self::METADATA_FILE))?; + serde_json::to_writer(json_file, &self.chains)?; + + Ok(()) + } + + /// Overwrite the in-memory state with the on-disk state. + pub fn load(&mut self) -> std::io::Result<()> { + let json_file = match File::open(self.backup_dir.join(Self::METADATA_FILE)) { + Ok(f) => f, + Err(e) => { + // Don't error out if the file isn't there, it will be created when necessary + if e.kind() == io::ErrorKind::NotFound { + self.chains = Vec::new(); + + return Ok(()); + } else { + return Err(e); + } + } + }; + self.chains = serde_json::from_reader(json_file)?; + + Ok(()) + } +} diff --git a/src/backup/mod.rs b/src/backup/mod.rs new file mode 100644 index 0000000..600ef6d --- /dev/null +++ b/src/backup/mod.rs @@ -0,0 +1,150 @@ +mod delta; +mod manager; +mod path; + +use delta::Delta; +pub use manager::Manager; + +use chrono::Utc; +use flate2::write::GzEncoder; +use flate2::Compression; +use path::PathExt; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub enum BackupType { + Full, + Incremental, +} + +/// Represents a successful backup +#[derive(Debug, Serialize, Deserialize)] +pub struct Backup { + /// When the backup was started (also corresponds to the name) + pub start_time: chrono::DateTime, + /// Type of the backup + pub type_: BackupType, + pub delta: Delta, +} + +impl Backup { + const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; + + /// Resolve the state of the list of backups by applying their deltas in-order to an initially + /// empty state. + pub fn state(backups: &Vec) -> HashMap> { + let mut state: HashMap> = HashMap::new(); + + for backup in backups { + backup.delta.apply(&mut state); + } + + state + } + + /// Return the path to a backup file by properly formatting the data. + pub fn path>(backup_dir: P, start_time: chrono::DateTime) -> PathBuf { + let backup_dir = backup_dir.as_ref(); + + let filename = format!("{}", start_time.format(Self::FILENAME_FORMAT)); + backup_dir.join(filename) + } + + /// Create a new Full backup, populated with the given directories. + /// + /// # Arguments + /// + /// * `backup_dir` - Directory to store archive in + /// * `dirs` - list of tuples `(path_in_tar, src_dir)` with `path_in_tar` the directory name + /// under which `src_dir`'s contents should be stored in the archive + /// + /// # Returns + /// + /// The `Backup` instance describing this new backup. + pub fn create>( + backup_dir: P, + dirs: Vec<(PathBuf, PathBuf)>, + ) -> io::Result { + let start_time = chrono::offset::Utc::now(); + + let path = Self::path(backup_dir, start_time); + let tar_gz = File::create(path)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut ar = tar::Builder::new(enc); + + let mut delta = Delta::new(); + + for (dir_in_tar, src_dir) in dirs { + let mut added_files: HashSet = HashSet::new(); + + for entry in src_dir.read_dir_recursive()?.ignored("cache").files() { + let path = entry?.path(); + let stripped = path.strip_prefix(&src_dir).unwrap(); + + ar.append_path_with_name(&path, dir_in_tar.join(stripped))?; + added_files.insert(stripped.to_path_buf()); + } + + delta.added.insert(dir_in_tar, added_files); + } + + Ok(Backup { + type_: BackupType::Full, + start_time, + delta, + }) + } + + /// Create a new incremental backup from a given previous backup + pub fn create_from>( + previous_state: HashMap>, + previous_start_time: chrono::DateTime, + backup_dir: P, + dirs: Vec<(PathBuf, PathBuf)>, + ) -> io::Result { + let start_time = chrono::offset::Utc::now(); + + let path = Self::path(backup_dir, start_time); + let tar_gz = File::create(path)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut ar = tar::Builder::new(enc); + + let mut delta = Delta::new(); + + for (dir_in_tar, src_dir) in dirs { + let mut all_files: HashSet = HashSet::new(); + let mut added_files: HashSet = HashSet::new(); + + for entry in src_dir.read_dir_recursive()?.ignored("cache").files() { + let path = entry?.path(); + let stripped = path.strip_prefix(&src_dir).unwrap(); + + if !path.not_modified_since(previous_start_time) { + ar.append_path_with_name(&path, dir_in_tar.join(stripped))?; + added_files.insert(stripped.to_path_buf()); + } + + all_files.insert(stripped.to_path_buf()); + } + + delta.added.insert(dir_in_tar.clone(), added_files); + + if let Some(previous_files) = previous_state.get(&dir_in_tar) { + delta.removed.insert( + dir_in_tar, + previous_files.difference(&all_files).cloned().collect(), + ); + } + } + + Ok(Backup { + type_: BackupType::Incremental, + start_time, + delta, + }) + } +} diff --git a/src/server/path.rs b/src/backup/path.rs similarity index 100% rename from src/server/path.rs rename to src/backup/path.rs diff --git a/src/main.rs b/src/main.rs index 0aef676..40d112c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +mod backup; mod cli; mod server; mod signals; @@ -59,7 +60,7 @@ fn command_run(cli: &Cli, args: &RunArgs) -> io::Result<()> { } fn commands_backup(cli: &Cli, _args: &BackupArgs) -> io::Result<()> { - let mut manager = server::BackupManager::new( + let mut manager = backup::Manager::new( cli.backup.clone(), cli.config.clone(), cli.world.clone(), diff --git a/src/server/backups.rs b/src/server/backups.rs deleted file mode 100644 index 5097746..0000000 --- a/src/server/backups.rs +++ /dev/null @@ -1,345 +0,0 @@ -use crate::server::path::PathExt; -use chrono::Utc; -use flate2::write::GzEncoder; -use flate2::Compression; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::fs::File; -use std::io; -use std::path::{Path, PathBuf}; - -#[link(name = "c")] -extern "C" { - fn geteuid() -> u32; - fn getegid() -> u32; -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -pub enum BackupType { - Full, - Incremental, -} - -/// Represents the changes relative to the previous backup -#[derive(Debug, Serialize, Deserialize)] -pub struct BackupDelta { - /// What files were added/modified in each part of the tarball. - pub added: HashMap>, - /// What files were removed in this backup, in comparison to the previous backup. For full - /// backups, this will always be empty, as they do not consider previous backups. - /// The map stores a separate list for each top-level directory, as the contents of these - /// directories can come for different source directories. - pub removed: HashMap>, -} - -impl BackupDelta { - pub fn new() -> Self { - BackupDelta { - added: HashMap::new(), - removed: HashMap::new(), - } - } - - /// Update the current state so that its result becomes the merge of itself and the other - /// state. - pub fn merge(&mut self, delta: &BackupDelta) { - for (dir, added) in delta.added.iter() { - // Files that were removed in the current state, but added in the new state, are no - // longer removed - if let Some(orig_removed) = self.removed.get_mut(dir) { - orig_removed.retain(|k| !added.contains(k)); - } - - // Newly added files are added to the state as well - if let Some(orig_added) = self.added.get_mut(dir) { - orig_added.extend(added.iter().cloned()); - } else { - self.added.insert(dir.clone(), added.clone()); - } - } - - for (dir, removed) in delta.removed.iter() { - // Files that were originally added, but now deleted are removed from the added list - if let Some(orig_added) = self.added.get_mut(dir) { - orig_added.retain(|k| !removed.contains(k)); - } - - // Newly removed files are added to the state as well - if let Some(orig_removed) = self.removed.get_mut(dir) { - orig_removed.extend(removed.iter().cloned()); - } else { - self.removed.insert(dir.clone(), removed.clone()); - } - } - } - - /// Modify the given state by applying this delta's changes to it - pub fn apply(&self, state: &mut HashMap>) { - // First we add new files, then we remove the old ones - for (dir, added) in self.added.iter() { - if let Some(current) = state.get_mut(dir) { - current.extend(added.iter().cloned()); - } else { - state.insert(dir.clone(), added.clone()); - } - } - - for (dir, removed) in self.removed.iter() { - if let Some(current) = state.get_mut(dir) { - current.retain(|k| !removed.contains(k)); - } - } - } -} - -/// Represents a successful backup -#[derive(Debug, Serialize, Deserialize)] -pub struct Backup { - /// When the backup was started (also corresponds to the name) - start_time: chrono::DateTime, - /// Type of the backup - type_: BackupType, - delta: BackupDelta, -} - -impl Backup { - const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; - - /// Resolve the state of the list of backups by applying their deltas in-order to an initially - /// empty state. - pub fn state(backups: &Vec) -> HashMap> { - let mut state: HashMap> = HashMap::new(); - - for backup in backups { - backup.delta.apply(&mut state); - } - - state - } - - /// Return the path to a backup file by properly formatting the data. - pub fn path>(backup_dir: P, start_time: chrono::DateTime) -> PathBuf { - let backup_dir = backup_dir.as_ref(); - - let filename = format!("{}", start_time.format(Self::FILENAME_FORMAT)); - backup_dir.join(filename) - } - - /// Create a new Full backup, populated with the given directories. - /// - /// # Arguments - /// - /// * `backup_dir` - Directory to store archive in - /// * `dirs` - list of tuples `(path_in_tar, src_dir)` with `path_in_tar` the directory name - /// under which `src_dir`'s contents should be stored in the archive - /// - /// # Returns - /// - /// The `Backup` instance describing this new backup. - pub fn create>( - backup_dir: P, - dirs: Vec<(PathBuf, PathBuf)>, - ) -> io::Result { - let start_time = chrono::offset::Utc::now(); - - let path = Self::path(backup_dir, start_time); - let tar_gz = File::create(path)?; - let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut ar = tar::Builder::new(enc); - - let mut delta = BackupDelta::new(); - - for (dir_in_tar, src_dir) in dirs { - let mut added_files: HashSet = HashSet::new(); - - for entry in src_dir.read_dir_recursive()?.ignored("cache").files() { - let path = entry?.path(); - let stripped = path.strip_prefix(&src_dir).unwrap(); - - ar.append_path_with_name(&path, dir_in_tar.join(stripped))?; - added_files.insert(stripped.to_path_buf()); - } - - delta.added.insert(dir_in_tar, added_files); - } - - Ok(Backup { - type_: BackupType::Full, - start_time, - delta, - }) - } - - /// Create a new incremental backup from a given previous backup - pub fn create_from>( - previous_state: HashMap>, - previous_start_time: chrono::DateTime, - backup_dir: P, - dirs: Vec<(PathBuf, PathBuf)>, - ) -> io::Result { - let start_time = chrono::offset::Utc::now(); - - let path = Self::path(backup_dir, start_time); - let tar_gz = File::create(path)?; - let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut ar = tar::Builder::new(enc); - - let mut delta = BackupDelta::new(); - - for (dir_in_tar, src_dir) in dirs { - let mut all_files: HashSet = HashSet::new(); - let mut added_files: HashSet = HashSet::new(); - - for entry in src_dir.read_dir_recursive()?.ignored("cache").files() { - let path = entry?.path(); - let stripped = path.strip_prefix(&src_dir).unwrap(); - - if !path.not_modified_since(previous_start_time) { - ar.append_path_with_name(&path, dir_in_tar.join(stripped))?; - added_files.insert(stripped.to_path_buf()); - } - - all_files.insert(stripped.to_path_buf()); - } - - delta.added.insert(dir_in_tar.clone(), added_files); - - if let Some(previous_files) = previous_state.get(&dir_in_tar) { - delta.removed.insert( - dir_in_tar, - previous_files.difference(&all_files).cloned().collect(), - ); - } - } - - Ok(Backup { - type_: BackupType::Incremental, - start_time, - delta, - }) - } -} - -pub struct BackupManager { - backup_dir: PathBuf, - config_dir: PathBuf, - world_dir: PathBuf, - chain_len: u64, - chains_to_keep: u64, - chains: Vec>, -} - -impl BackupManager { - const METADATA_FILE: &str = "alex.json"; - - /// Initialize a new instance of a `BackupManager`. - pub fn new( - backup_dir: PathBuf, - config_dir: PathBuf, - world_dir: PathBuf, - chain_len: u64, - chains_to_keep: u64, - ) -> Self { - BackupManager { - backup_dir, - config_dir, - world_dir, - chain_len, - chains_to_keep, - chains: Vec::new(), - } - } - - /// Create a new backup with the expected type. - pub fn create_backup(&mut self) -> io::Result<()> { - let dirs = vec![ - (PathBuf::from("config"), self.config_dir.clone()), - (PathBuf::from("worlds"), self.world_dir.clone()), - ]; - - // I kinda hate this statement, please just let me combine let statements in if statements - // already - let backup = if let Some(current_chain) = self.chains.last() { - let current_chain_len: u64 = current_chain.len().try_into().unwrap(); - - if current_chain_len < self.chain_len { - if let Some(previous_backup) = current_chain.last() { - let state = Backup::state(current_chain); - - Backup::create_from(state, previous_backup.start_time, &self.backup_dir, dirs)? - } else { - Backup::create(&self.backup_dir, dirs)? - } - } else { - self.chains.push(Vec::new()); - - Backup::create(&self.backup_dir, dirs)? - } - } else { - self.chains.push(Vec::new()); - - Backup::create(&self.backup_dir, dirs)? - }; - - // The above statement always creates this element, so this unwrap is safe - self.chains.last_mut().unwrap().push(backup); - - self.save()?; - - Ok(()) - } - - /// Delete all backups associated with outdated chains, and forget those chains. - pub fn remove_old_backups(&mut self) -> std::io::Result<()> { - let chains_to_store: usize = self.chains_to_keep.try_into().unwrap(); - - if chains_to_store < self.chains.len() { - let mut remove_count: usize = self.chains.len() - chains_to_store; - - // We only count finished chains towards the list of stored chains - let chain_len: usize = self.chain_len.try_into().unwrap(); - if self.chains.last().unwrap().len() < chain_len { - remove_count -= 1; - } - - for chain in self.chains.drain(..remove_count) { - for backup in chain { - let path = Backup::path(&self.backup_dir, backup.start_time); - std::fs::remove_file(path)?; - } - } - } - - self.save()?; - - Ok(()) - } - - /// Write the in-memory state to disk. - pub fn save(&self) -> std::io::Result<()> { - let json_file = File::create(self.backup_dir.join(Self::METADATA_FILE))?; - serde_json::to_writer(json_file, &self.chains)?; - - Ok(()) - } - - /// Overwrite the in-memory state with the on-disk state. - pub fn load(&mut self) -> std::io::Result<()> { - let json_file = match File::open(self.backup_dir.join(Self::METADATA_FILE)) { - Ok(f) => f, - Err(e) => { - // Don't error out if the file isn't there, it will be created when necessary - if e.kind() == io::ErrorKind::NotFound { - self.chains = Vec::new(); - - return Ok(()); - } else { - return Err(e); - } - } - }; - self.chains = serde_json::from_reader(json_file)?; - - Ok(()) - } -} diff --git a/src/server/command.rs b/src/server/command.rs index bd1f0e6..808d6bf 100644 --- a/src/server/command.rs +++ b/src/server/command.rs @@ -1,4 +1,5 @@ -use crate::server::{BackupManager, ServerProcess}; +use crate::backup::Manager as BackupManager; +use crate::server::ServerProcess; use clap::ValueEnum; use std::fmt; use std::fs::File; diff --git a/src/server/mod.rs b/src/server/mod.rs index 4c2beb2..e3e3131 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,8 +1,5 @@ -mod backups; mod command; -mod path; mod process; -pub use backups::BackupManager; pub use command::{ServerCommand, ServerType}; pub use process::ServerProcess; diff --git a/src/server/process.rs b/src/server/process.rs index 1e24373..3ac7beb 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -1,4 +1,4 @@ -use crate::server::BackupManager; +use crate::backup::Manager as BackupManager; use crate::server::ServerType; use std::io::Write; use std::process::Child; From 53dc3783ca2696e35f5dbd956c3b9632dd043d7d Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 20 Jun 2023 19:31:50 +0200 Subject: [PATCH 18/19] feat: store server info in metadata file; change cli flags --- CHANGELOG.md | 2 ++ src/backup/delta.rs | 1 + src/backup/manager.rs | 60 ++++++++++++++++++++++++++----------------- src/backup/mod.rs | 38 +++++++++++++++++---------- src/cli.rs | 20 +++++---------- src/main.rs | 8 +++++- src/server/command.rs | 23 ++++++++++++----- src/server/mod.rs | 2 +- src/server/process.rs | 15 +++-------- 9 files changed, 97 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8157f9e..9d35929 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Chain length descibres how many incremental backups to create from the same full backup * "backups to keep" has been replaced by "chains to keep" +* Server type & version is now stored as metadata in the metadata file ### Changed * Running the server now uses the `run` CLI subcommand +* `server_type` and `server_version` arguments are now optional flags ### Removed diff --git a/src/backup/delta.rs b/src/backup/delta.rs index 15f233b..34971f0 100644 --- a/src/backup/delta.rs +++ b/src/backup/delta.rs @@ -24,6 +24,7 @@ impl Delta { /// Update the current state so that its result becomes the merge of itself and the other /// state. + #[allow(dead_code)] pub fn merge(&mut self, delta: &Self) { for (dir, added) in delta.added.iter() { // Files that were removed in the current state, but added in the new state, are no diff --git a/src/backup/manager.rs b/src/backup/manager.rs index 71633a3..ce3dbe1 100644 --- a/src/backup/manager.rs +++ b/src/backup/manager.rs @@ -1,18 +1,27 @@ use super::Backup; +use serde::de::DeserializeOwned; +use serde::Serialize; use std::fs::File; use std::io; use std::path::PathBuf; -pub struct Manager { +pub struct Manager +where + T: Clone + Serialize + DeserializeOwned, +{ backup_dir: PathBuf, config_dir: PathBuf, world_dir: PathBuf, + default_metadata: T, chain_len: u64, chains_to_keep: u64, - chains: Vec>, + chains: Vec>>, } -impl Manager { +impl Manager +where + T: Clone + Serialize + DeserializeOwned, +{ const METADATA_FILE: &str = "alex.json"; /// Initialize a new instance of a `BackupManager`. @@ -20,6 +29,7 @@ impl Manager { backup_dir: PathBuf, config_dir: PathBuf, world_dir: PathBuf, + metadata: T, chain_len: u64, chains_to_keep: u64, ) -> Self { @@ -27,6 +37,7 @@ impl Manager { backup_dir, config_dir, world_dir, + default_metadata: metadata, chain_len, chains_to_keep, chains: Vec::new(), @@ -40,32 +51,32 @@ impl Manager { (PathBuf::from("worlds"), self.world_dir.clone()), ]; - // I kinda hate this statement, please just let me combine let statements in if statements - // already - let backup = if let Some(current_chain) = self.chains.last() { + // We start a new chain if the current chain is complete, or if there isn't a first chain + // yet + if let Some(current_chain) = self.chains.last() { let current_chain_len: u64 = current_chain.len().try_into().unwrap(); - if current_chain_len < self.chain_len { - if let Some(previous_backup) = current_chain.last() { - let state = Backup::state(current_chain); - - Backup::create_from(state, previous_backup.start_time, &self.backup_dir, dirs)? - } else { - Backup::create(&self.backup_dir, dirs)? - } - } else { + if current_chain_len >= self.chain_len { self.chains.push(Vec::new()); - - Backup::create(&self.backup_dir, dirs)? } } else { self.chains.push(Vec::new()); + } + let current_chain = self.chains.last_mut().unwrap(); + + let mut backup = if !current_chain.is_empty() { + let previous_backup = current_chain.last().unwrap(); + let state = Backup::state(current_chain); + + Backup::create_from(state, previous_backup.start_time, &self.backup_dir, dirs)? + } else { Backup::create(&self.backup_dir, dirs)? }; - // The above statement always creates this element, so this unwrap is safe - self.chains.last_mut().unwrap().push(backup); + backup.set_metadata(self.default_metadata.clone()); + + current_chain.push(backup); self.save()?; @@ -73,7 +84,7 @@ impl Manager { } /// Delete all backups associated with outdated chains, and forget those chains. - pub fn remove_old_backups(&mut self) -> std::io::Result<()> { + pub fn remove_old_backups(&mut self) -> io::Result<()> { let chains_to_store: usize = self.chains_to_keep.try_into().unwrap(); if chains_to_store < self.chains.len() { @@ -91,15 +102,15 @@ impl Manager { std::fs::remove_file(path)?; } } - } - self.save()?; + self.save()?; + } Ok(()) } /// Write the in-memory state to disk. - pub fn save(&self) -> std::io::Result<()> { + pub fn save(&self) -> io::Result<()> { let json_file = File::create(self.backup_dir.join(Self::METADATA_FILE))?; serde_json::to_writer(json_file, &self.chains)?; @@ -107,7 +118,7 @@ impl Manager { } /// Overwrite the in-memory state with the on-disk state. - pub fn load(&mut self) -> std::io::Result<()> { + pub fn load(&mut self) -> io::Result<()> { let json_file = match File::open(self.backup_dir.join(Self::METADATA_FILE)) { Ok(f) => f, Err(e) => { @@ -121,6 +132,7 @@ impl Manager { } } }; + self.chains = serde_json::from_reader(json_file)?; Ok(()) diff --git a/src/backup/mod.rs b/src/backup/mod.rs index 600ef6d..5b12b8c 100644 --- a/src/backup/mod.rs +++ b/src/backup/mod.rs @@ -22,21 +22,37 @@ pub enum BackupType { } /// Represents a successful backup -#[derive(Debug, Serialize, Deserialize)] -pub struct Backup { +#[derive(Serialize, Deserialize)] +pub struct Backup { /// When the backup was started (also corresponds to the name) pub start_time: chrono::DateTime, /// Type of the backup pub type_: BackupType, pub delta: Delta, + /// Additional metadata that can be associated with a given backup + pub metadata: Option, } -impl Backup { +impl Backup<()> { + /// Return the path to a backup file by properly formatting the data. + pub fn path>(backup_dir: P, start_time: chrono::DateTime) -> PathBuf { + let backup_dir = backup_dir.as_ref(); + + let filename = format!("{}", start_time.format(Self::FILENAME_FORMAT)); + backup_dir.join(filename) + } +} + +impl Backup { const FILENAME_FORMAT: &str = "%Y-%m-%d_%H-%M-%S.tar.gz"; + pub fn set_metadata(&mut self, metadata: T) { + self.metadata = Some(metadata); + } + /// Resolve the state of the list of backups by applying their deltas in-order to an initially /// empty state. - pub fn state(backups: &Vec) -> HashMap> { + pub fn state(backups: &Vec) -> HashMap> { let mut state: HashMap> = HashMap::new(); for backup in backups { @@ -46,14 +62,6 @@ impl Backup { state } - /// Return the path to a backup file by properly formatting the data. - pub fn path>(backup_dir: P, start_time: chrono::DateTime) -> PathBuf { - let backup_dir = backup_dir.as_ref(); - - let filename = format!("{}", start_time.format(Self::FILENAME_FORMAT)); - backup_dir.join(filename) - } - /// Create a new Full backup, populated with the given directories. /// /// # Arguments @@ -71,7 +79,7 @@ impl Backup { ) -> io::Result { let start_time = chrono::offset::Utc::now(); - let path = Self::path(backup_dir, start_time); + let path = Backup::path(backup_dir, start_time); let tar_gz = File::create(path)?; let enc = GzEncoder::new(tar_gz, Compression::default()); let mut ar = tar::Builder::new(enc); @@ -96,6 +104,7 @@ impl Backup { type_: BackupType::Full, start_time, delta, + metadata: None, }) } @@ -108,7 +117,7 @@ impl Backup { ) -> io::Result { let start_time = chrono::offset::Utc::now(); - let path = Self::path(backup_dir, start_time); + let path = Backup::path(backup_dir, start_time); let tar_gz = File::create(path)?; let enc = GzEncoder::new(tar_gz, Compression::default()); let mut ar = tar::Builder::new(enc); @@ -145,6 +154,7 @@ impl Backup { type_: BackupType::Incremental, start_time, delta, + metadata: None, }) } } diff --git a/src/cli.rs b/src/cli.rs index 613dfa1..166fa82 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -53,6 +53,12 @@ pub struct Cli { global = true )] pub chains: u64, + /// Type of server + #[arg(long, default_value = "unknown", env = "ALEX_SERVER")] + pub server: ServerType, + /// Version string for the server, e.g. 1.19.4-545 + #[arg(long, default_value = "", env = "ALEX_SERVER_VERSION")] + pub server_version: String, } #[derive(Subcommand)] @@ -66,12 +72,6 @@ pub enum Commands { #[derive(Args)] pub struct RunArgs { - /// Type of server - pub type_: ServerType, - /// Version string for the server, e.g. 1.19.4-545 - #[arg(env = "ALEX_SERVER_VERSION")] - pub server_version: String, - /// Server jar to execute #[arg( long, @@ -103,10 +103,4 @@ pub struct RunArgs { } #[derive(Args)] -pub struct BackupArgs { - /// Type of server - pub type_: ServerType, - /// Version string for the server, e.g. 1.19.4-545 - #[arg(env = "ALEX_SERVER_VERSION")] - pub server_version: String, -} +pub struct BackupArgs {} diff --git a/src/main.rs b/src/main.rs index 40d112c..ffb0c93 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,7 +25,7 @@ fn backups_thread(counter: Arc>, frequency: u64) { fn command_run(cli: &Cli, args: &RunArgs) -> io::Result<()> { let (_, mut signals) = signals::install_signal_handlers()?; - let mut cmd = server::ServerCommand::new(args.type_, &args.server_version) + let mut cmd = server::ServerCommand::new(cli.server, &cli.server_version) .java(&args.java) .jar(args.jar.clone()) .config(cli.config.clone()) @@ -60,10 +60,16 @@ fn command_run(cli: &Cli, args: &RunArgs) -> io::Result<()> { } fn commands_backup(cli: &Cli, _args: &BackupArgs) -> io::Result<()> { + let metadata = server::Metadata { + server_type: cli.server, + server_version: cli.server_version.clone(), + }; + let mut manager = backup::Manager::new( cli.backup.clone(), cli.config.clone(), cli.world.clone(), + metadata, cli.chain_len, cli.chains, ); diff --git a/src/server/command.rs b/src/server/command.rs index 808d6bf..d06f8bc 100644 --- a/src/server/command.rs +++ b/src/server/command.rs @@ -1,14 +1,16 @@ use crate::backup::Manager as BackupManager; use crate::server::ServerProcess; use clap::ValueEnum; +use serde::{Deserialize, Serialize}; use std::fmt; use std::fs::File; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Serialize, Deserialize)] pub enum ServerType { + Unknown, Paper, Forge, Vanilla, @@ -17,6 +19,7 @@ pub enum ServerType { impl fmt::Display for ServerType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let s = match self { + ServerType::Unknown => "Unknown", ServerType::Paper => "PaperMC", ServerType::Forge => "Forge", ServerType::Vanilla => "Vanilla", @@ -26,6 +29,12 @@ impl fmt::Display for ServerType { } } +#[derive(Clone, Serialize, Deserialize)] +pub struct Metadata { + pub server_type: ServerType, + pub server_version: String, +} + pub struct ServerCommand { type_: ServerType, version: String, @@ -187,10 +196,15 @@ impl ServerCommand { } pub fn spawn(&mut self) -> std::io::Result { + let metadata = Metadata { + server_type: self.type_, + server_version: self.version.clone(), + }; let mut manager = BackupManager::new( self.backup_dir.clone(), self.config_dir.clone(), self.world_dir.clone(), + metadata, self.chain_len, self.chains_to_keep, ); @@ -200,12 +214,7 @@ impl ServerCommand { self.accept_eula()?; let child = cmd.spawn()?; - Ok(ServerProcess::new( - self.type_, - self.version.clone(), - manager, - child, - )) + Ok(ServerProcess::new(manager, child)) } } diff --git a/src/server/mod.rs b/src/server/mod.rs index e3e3131..0f01c9a 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,5 +1,5 @@ mod command; mod process; -pub use command::{ServerCommand, ServerType}; +pub use command::{Metadata, ServerCommand, ServerType}; pub use process::ServerProcess; diff --git a/src/server/process.rs b/src/server/process.rs index 3ac7beb..0a09ab0 100644 --- a/src/server/process.rs +++ b/src/server/process.rs @@ -1,25 +1,16 @@ use crate::backup::Manager as BackupManager; -use crate::server::ServerType; +use crate::server::Metadata; use std::io::Write; use std::process::Child; pub struct ServerProcess { - type_: ServerType, - version: String, child: Child, - backups: BackupManager, + backups: BackupManager, } impl ServerProcess { - pub fn new( - type_: ServerType, - version: String, - manager: BackupManager, - child: Child, - ) -> ServerProcess { + pub fn new(manager: BackupManager, child: Child) -> ServerProcess { ServerProcess { - type_, - version, child, backups: manager, } From 188fb3034335a4244fd754cc668090de90d8d5f2 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 22 Jun 2023 20:10:37 +0200 Subject: [PATCH 19/19] fix: better serde bounds --- CHANGELOG.md | 2 +- src/backup/manager.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d35929..c6009a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `backup` CLI command * Incremental backups - * Chain length descibres how many incremental backups to create from the + * Chain length describes how many incremental backups to create from the same full backup * "backups to keep" has been replaced by "chains to keep" * Server type & version is now stored as metadata in the metadata file diff --git a/src/backup/manager.rs b/src/backup/manager.rs index ce3dbe1..1dfec16 100644 --- a/src/backup/manager.rs +++ b/src/backup/manager.rs @@ -1,5 +1,5 @@ use super::Backup; -use serde::de::DeserializeOwned; +use serde::Deserialize; use serde::Serialize; use std::fs::File; use std::io; @@ -7,7 +7,7 @@ use std::path::PathBuf; pub struct Manager where - T: Clone + Serialize + DeserializeOwned, + T: Clone + Serialize + for<'de> Deserialize<'de>, { backup_dir: PathBuf, config_dir: PathBuf, @@ -20,7 +20,7 @@ where impl Manager where - T: Clone + Serialize + DeserializeOwned, + T: Clone + Serialize + for<'de> Deserialize<'de>, { const METADATA_FILE: &str = "alex.json";