Compare commits
6 Commits
c32741bbc9
...
45d736d1bb
| Author | SHA1 | Date |
|---|---|---|
|
|
45d736d1bb | |
|
|
69ce8616d5 | |
|
|
50cdd3115f | |
|
|
0faa6a8578 | |
|
|
f5fc8b588f | |
|
|
640364405f |
|
|
@ -1,2 +1,3 @@
|
||||||
[alias]
|
[alias]
|
||||||
runs = "run -- paper --config data/config --backup data/backups --world data/worlds --jar data/paper.jar"
|
runs = "run -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper.jar"
|
||||||
|
runrs = "run --release -- paper 1.19.4-545 --config data/config --backup data/backups --world data/worlds --jar data/paper.jar"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
*
|
||||||
|
|
||||||
|
!Cargo.toml
|
||||||
|
!Cargo.lock
|
||||||
|
!src/
|
||||||
10
CHANGELOG.md
10
CHANGELOG.md
|
|
@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||||
|
|
||||||
## [Unreleased](https://git.rustybever.be/Chewing_Bever/alex/src/branch/dev)
|
## [Unreleased](https://git.rustybever.be/Chewing_Bever/alex/src/branch/dev)
|
||||||
|
|
||||||
|
## [0.2.0](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.2.0)
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
* Rudimentary signal handling for gently stopping server
|
||||||
|
* A single stop signal will trigger the Java process to shut down, but Alex
|
||||||
|
still expects to be run from a utility such as dumb-init
|
||||||
|
* Properly back up entire config directory
|
||||||
|
* Inject Java optimisation flags
|
||||||
|
|
||||||
## [0.1.1](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.1.1)
|
## [0.1.1](https://git.rustybever.be/Chewing_Bever/alex/src/tag/0.1.1)
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
|
||||||
|
|
@ -10,11 +10,12 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "alex"
|
name = "alex"
|
||||||
version = "0.1.0"
|
version = "0.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
"clap",
|
"clap",
|
||||||
"flate2",
|
"flate2",
|
||||||
|
"signal-hook",
|
||||||
"tar",
|
"tar",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -383,6 +384,25 @@ dependencies = [
|
||||||
"windows-sys",
|
"windows-sys",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "signal-hook"
|
||||||
|
version = "0.3.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"signal-hook-registry",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "signal-hook-registry"
|
||||||
|
version = "1.4.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "strsim"
|
name = "strsim"
|
||||||
version = "0.10.0"
|
version = "0.10.0"
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "alex"
|
name = "alex"
|
||||||
version = "0.1.0"
|
version = "0.2.0"
|
||||||
description = "Wrapper around Minecraft server processes, designed to complement Docker image installations."
|
description = "Wrapper around Minecraft server processes, designed to complement Docker image installations."
|
||||||
authors = ["Jef Roosens"]
|
authors = ["Jef Roosens"]
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
@ -15,6 +15,7 @@ flate2 = "1.0.26"
|
||||||
# Used for backup filenames
|
# Used for backup filenames
|
||||||
chrono = "0.4.26"
|
chrono = "0.4.26"
|
||||||
clap = { version = "4.3.1", features = ["derive", "env"] }
|
clap = { version = "4.3.1", features = ["derive", "env"] }
|
||||||
|
signal-hook = "0.3.15"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
lto = "fat"
|
lto = "fat"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
FROM rust:1.70-alpine3.18 AS builder
|
||||||
|
|
||||||
|
ARG DI_VER=1.2.5
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . ./
|
||||||
|
|
||||||
|
RUN apk add --no-cache build-base unzip curl && \
|
||||||
|
curl -Lo - "https://github.com/Yelp/dumb-init/archive/refs/tags/v${DI_VER}.tar.gz" | tar -xzf - && \
|
||||||
|
cd "dumb-init-${DI_VER}" && \
|
||||||
|
make SHELL=/bin/sh && \
|
||||||
|
mv dumb-init ..
|
||||||
|
|
||||||
|
RUN cargo build && \
|
||||||
|
[ "$(readelf -d target/debug/alex | grep NEEDED | wc -l)" = 0 ]
|
||||||
|
|
||||||
|
|
||||||
|
# We use ${:-} instead of a default value because the argument is always passed
|
||||||
|
# to the build, it'll just be blank most likely
|
||||||
|
FROM eclipse-temurin:17-jre-alpine
|
||||||
|
|
||||||
|
# Build arguments
|
||||||
|
ARG MC_VERSION=1.19.4
|
||||||
|
ARG PAPERMC_VERSION=545
|
||||||
|
|
||||||
|
RUN addgroup -Sg 1000 paper && \
|
||||||
|
adduser -SHG paper -u 1000 paper
|
||||||
|
|
||||||
|
# Create worlds and config directory
|
||||||
|
WORKDIR /app
|
||||||
|
RUN mkdir -p worlds config/cache backups
|
||||||
|
|
||||||
|
# Download server file
|
||||||
|
ADD "https://papermc.io/api/v2/projects/paper/versions/$MC_VERSION/builds/$PAPERMC_VERSION/downloads/paper-$MC_VERSION-$PAPERMC_VERSION.jar" server.jar
|
||||||
|
|
||||||
|
# Make sure the server user can access all necessary folders
|
||||||
|
RUN chown -R paper:paper /app
|
||||||
|
|
||||||
|
# Store the cache in an anonymous volume, which means it won't get stored in the other volumes
|
||||||
|
VOLUME /app/config/cache
|
||||||
|
VOLUME /app/backups
|
||||||
|
|
||||||
|
COPY --from=builder /app/dumb-init /bin/dumb-init
|
||||||
|
COPY --from=builder /app/target/debug/alex /bin/alex
|
||||||
|
|
||||||
|
RUN chmod +x /bin/alex
|
||||||
|
|
||||||
|
# Default value to keep users from eating up all ram accidentally
|
||||||
|
ENV ALEX_XMS=1024 \
|
||||||
|
ALEX_XMX=2048 \
|
||||||
|
ALEX_JAR=/app/server.jar \
|
||||||
|
ALEX_CONFIG_DIR=/app/config \
|
||||||
|
ALEX_WORLD_DIR=/app/worlds \
|
||||||
|
ALEX_BACKUPS_DIR=/app/backups \
|
||||||
|
ALEX_SERVER_VERSION="${MC_VERSION}-${PAPERMC_VERSION}"
|
||||||
|
|
||||||
|
# Document exposed ports
|
||||||
|
EXPOSE 25565
|
||||||
|
|
||||||
|
# Switch to non-root user
|
||||||
|
USER paper:paper
|
||||||
|
|
||||||
|
ENTRYPOINT ["/bin/alex", "paper"]
|
||||||
33
src/main.rs
33
src/main.rs
|
|
@ -1,4 +1,6 @@
|
||||||
mod server;
|
mod server;
|
||||||
|
mod signals;
|
||||||
|
mod stdin;
|
||||||
|
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use server::ServerType;
|
use server::ServerType;
|
||||||
|
|
@ -79,7 +81,8 @@ fn backups_thread(counter: Arc<Mutex<server::ServerProcess>>, frequency: u64) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() -> io::Result<()> {
|
||||||
|
let (_, mut signals) = signals::install_signal_handlers()?;
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
|
|
||||||
let cmd = server::ServerCommand::new(cli.type_, &cli.server_version)
|
let cmd = server::ServerCommand::new(cli.type_, &cli.server_version)
|
||||||
|
|
@ -91,33 +94,17 @@ fn main() {
|
||||||
.xms(cli.xms)
|
.xms(cli.xms)
|
||||||
.xmx(cli.xmx)
|
.xmx(cli.xmx)
|
||||||
.max_backups(cli.max_backups);
|
.max_backups(cli.max_backups);
|
||||||
let counter = Arc::new(Mutex::new(cmd.spawn().expect("Failed to start server.")));
|
let counter = Arc::new(Mutex::new(cmd.spawn()?));
|
||||||
|
|
||||||
if cli.frequency > 0 {
|
if cli.frequency > 0 {
|
||||||
let clone = Arc::clone(&counter);
|
let clone = Arc::clone(&counter);
|
||||||
std::thread::spawn(move || backups_thread(clone, cli.frequency));
|
std::thread::spawn(move || backups_thread(clone, cli.frequency));
|
||||||
}
|
}
|
||||||
|
|
||||||
let stdin = io::stdin();
|
// Spawn thread that handles the main stdin loop
|
||||||
let input = &mut String::new();
|
let clone = Arc::clone(&counter);
|
||||||
|
std::thread::spawn(move || stdin::handle_stdin(clone));
|
||||||
|
|
||||||
loop {
|
// Signal handler loop exits the process when necessary
|
||||||
input.clear();
|
signals::handle_signals(&mut signals, counter)
|
||||||
|
|
||||||
if stdin.read_line(input).is_err() {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
|
|
||||||
{
|
|
||||||
let mut server = counter.lock().unwrap();
|
|
||||||
|
|
||||||
if let Err(e) = server.send_command(input) {
|
|
||||||
println!("{}", e);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
if input.trim() == "stop" {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -113,16 +113,57 @@ impl ServerCommand {
|
||||||
let backup_dir = self.backup_dir.canonicalize()?;
|
let backup_dir = self.backup_dir.canonicalize()?;
|
||||||
|
|
||||||
self.accept_eula()?;
|
self.accept_eula()?;
|
||||||
|
let mut cmd = Command::new(&self.java);
|
||||||
|
|
||||||
let child = Command::new(&self.java)
|
// Apply JVM optimisation flags
|
||||||
.current_dir(&config_dir)
|
// https://aikar.co/2018/07/02/tuning-the-jvm-g1gc-garbage-collector-flags-for-minecraft/
|
||||||
|
cmd.arg(format!("-Xms{}M", self.xms))
|
||||||
|
.arg(format!("-Xmx{}M", self.xmx))
|
||||||
|
.args([
|
||||||
|
"-XX:+UseG1GC",
|
||||||
|
"-XX:+ParallelRefProcEnabled",
|
||||||
|
"-XX:MaxGCPauseMillis=200",
|
||||||
|
"-XX:+UnlockExperimentalVMOptions",
|
||||||
|
"-XX:+DisableExplicitGC",
|
||||||
|
"-XX:+AlwaysPreTouch",
|
||||||
|
"-XX:G1HeapWastePercent=5",
|
||||||
|
"-XX:G1MixedGCCountTarget=4",
|
||||||
|
"-XX:G1MixedGCLiveThresholdPercent=90",
|
||||||
|
"-XX:G1RSetUpdatingPauseTimePercent=5",
|
||||||
|
"-XX:SurvivorRatio=32",
|
||||||
|
"-XX:+PerfDisableSharedMem",
|
||||||
|
"-XX:MaxTenuringThreshold=1",
|
||||||
|
"-Dusing.aikars.flags=https://mcflags.emc.gs",
|
||||||
|
"-Daikars.new.flags=true",
|
||||||
|
]);
|
||||||
|
|
||||||
|
if self.xms > 12 * 1024 {
|
||||||
|
cmd.args([
|
||||||
|
"-XX:G1NewSizePercent=40",
|
||||||
|
"-XX:G1MaxNewSizePercent=50",
|
||||||
|
"-XX:G1HeapRegionSize=16M",
|
||||||
|
"-XX:G1ReservePercent=15",
|
||||||
|
"-XX:InitiatingHeapOccupancyPercent=20",
|
||||||
|
]);
|
||||||
|
} else {
|
||||||
|
cmd.args([
|
||||||
|
"-XX:G1NewSizePercent=30",
|
||||||
|
"-XX:G1MaxNewSizePercent=40",
|
||||||
|
"-XX:G1HeapRegionSize=8M",
|
||||||
|
"-XX:G1ReservePercent=15",
|
||||||
|
"-XX:InitiatingHeapOccupancyPercent=15",
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.current_dir(&config_dir)
|
||||||
.arg("-jar")
|
.arg("-jar")
|
||||||
.arg(&jar)
|
.arg(&jar)
|
||||||
.arg("--universe")
|
.arg("--universe")
|
||||||
.arg(&world_dir)
|
.arg(&world_dir)
|
||||||
.arg("--nogui")
|
.arg("--nogui")
|
||||||
.stdin(Stdio::piped())
|
.stdin(Stdio::piped());
|
||||||
.spawn()?;
|
|
||||||
|
let child = cmd.spawn()?;
|
||||||
|
|
||||||
Ok(ServerProcess::new(
|
Ok(ServerProcess::new(
|
||||||
self.type_,
|
self.type_,
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ use crate::server::ServerType;
|
||||||
use flate2::write::GzEncoder;
|
use flate2::write::GzEncoder;
|
||||||
use flate2::Compression;
|
use flate2::Compression;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use std::path::PathBuf;
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::Child;
|
use std::process::Child;
|
||||||
|
|
||||||
#[link(name = "c")]
|
#[link(name = "c")]
|
||||||
|
|
@ -67,6 +67,10 @@ impl ServerProcess {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn kill(&mut self) -> std::io::Result<()> {
|
||||||
|
self.child.kill()
|
||||||
|
}
|
||||||
|
|
||||||
/// Perform a backup by disabling the server's save feature and flushing its data, before
|
/// Perform a backup by disabling the server's save feature and flushing its data, before
|
||||||
/// creating an archive file.
|
/// creating an archive file.
|
||||||
pub fn backup(&mut self) -> std::io::Result<()> {
|
pub fn backup(&mut self) -> std::io::Result<()> {
|
||||||
|
|
@ -89,7 +93,11 @@ impl ServerProcess {
|
||||||
// The server's save feature needs to be enabled again even if the archive failed to create
|
// The server's save feature needs to be enabled again even if the archive failed to create
|
||||||
self.custom("save-on")?;
|
self.custom("save-on")?;
|
||||||
|
|
||||||
self.custom("say server backed up successfully")?;
|
if res.is_ok() {
|
||||||
|
self.custom("say server backed up successfully")?;
|
||||||
|
} else {
|
||||||
|
self.custom("an error occured while backing up the server")?;
|
||||||
|
}
|
||||||
|
|
||||||
res
|
res
|
||||||
}
|
}
|
||||||
|
|
@ -108,11 +116,21 @@ impl ServerProcess {
|
||||||
|
|
||||||
tar.append_dir_all("worlds", &self.world_dir)?;
|
tar.append_dir_all("worlds", &self.world_dir)?;
|
||||||
|
|
||||||
// We don't store all files in the config, as this would include caches
|
// Add all files from the config directory that aren't the cache
|
||||||
tar.append_path_with_name(
|
for entry in self
|
||||||
self.config_dir.join("server.properties"),
|
.config_dir
|
||||||
"config/server.properties",
|
.read_dir()?
|
||||||
)?;
|
.filter_map(|e| e.ok())
|
||||||
|
.filter(|e| e.file_name() != "cache")
|
||||||
|
{
|
||||||
|
let tar_path = Path::new("config").join(entry.file_name());
|
||||||
|
|
||||||
|
if entry.file_type()?.is_dir() {
|
||||||
|
tar.append_dir_all(tar_path, entry.path())?;
|
||||||
|
} else {
|
||||||
|
tar.append_path_with_name(entry.path(), tar_path)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// We add a file to the backup describing for what version it was made
|
// We add a file to the backup describing for what version it was made
|
||||||
let info = format!("{} {}", self.type_, self.version);
|
let info = format!("{} {}", self.type_, self.version);
|
||||||
|
|
@ -128,8 +146,6 @@ impl ServerProcess {
|
||||||
|
|
||||||
tar.append_data(&mut header, "info.txt", info_bytes)?;
|
tar.append_data(&mut header, "info.txt", info_bytes)?;
|
||||||
|
|
||||||
// tar.append_dir_all("config", &self.config_dir)?;
|
|
||||||
|
|
||||||
// Backup file gets finalized in the drop
|
// Backup file gets finalized in the drop
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
@ -139,7 +155,9 @@ impl ServerProcess {
|
||||||
fn remove_old_backups(&mut self) -> std::io::Result<()> {
|
fn remove_old_backups(&mut self) -> std::io::Result<()> {
|
||||||
// The naming format used allows us to sort the backups by name and still get a sorting by
|
// The naming format used allows us to sort the backups by name and still get a sorting by
|
||||||
// creation time
|
// creation time
|
||||||
let mut backups = std::fs::read_dir(&self.backup_dir)?
|
let mut backups = self
|
||||||
|
.backup_dir
|
||||||
|
.read_dir()?
|
||||||
.filter_map(|res| res.map(|e| e.path()).ok())
|
.filter_map(|res| res.map(|e| e.path()).ok())
|
||||||
.collect::<Vec<PathBuf>>();
|
.collect::<Vec<PathBuf>>();
|
||||||
backups.sort();
|
backups.sort();
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
use std::io;
|
||||||
|
use std::sync::atomic::AtomicBool;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use signal_hook::consts::TERM_SIGNALS;
|
||||||
|
use signal_hook::flag;
|
||||||
|
use signal_hook::iterator::{Signals, SignalsInfo};
|
||||||
|
|
||||||
|
use crate::server;
|
||||||
|
|
||||||
|
/// Install the required signal handlers for terminating signals.
|
||||||
|
pub fn install_signal_handlers() -> io::Result<(Arc<AtomicBool>, SignalsInfo)> {
|
||||||
|
let term = Arc::new(AtomicBool::new(false));
|
||||||
|
|
||||||
|
// For each terminating signal, we register both a shutdown handler and a handler that sets an
|
||||||
|
// atomic bool. With this, the process will get killed immediately once it receives a second
|
||||||
|
// termination signal (e.g. a double ctrl-c).
|
||||||
|
// https://docs.rs/signal-hook/0.3.15/signal_hook/#a-complex-signal-handling-with-a-background-thread
|
||||||
|
for sig in TERM_SIGNALS {
|
||||||
|
// When terminated by a second term signal, exit with exit code 1.
|
||||||
|
// This will do nothing the first time (because term_now is false).
|
||||||
|
flag::register_conditional_shutdown(*sig, 1, Arc::clone(&term))?;
|
||||||
|
// But this will "arm" the above for the second time, by setting it to true.
|
||||||
|
// The order of registering these is important, if you put this one first, it will
|
||||||
|
// first arm and then terminate ‒ all in the first round.
|
||||||
|
flag::register(*sig, Arc::clone(&term))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let signals = TERM_SIGNALS;
|
||||||
|
|
||||||
|
Ok((term, Signals::new(signals)?))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Loop that handles terminating signals as they come in.
|
||||||
|
pub fn handle_signals(
|
||||||
|
signals: &mut SignalsInfo,
|
||||||
|
counter: Arc<Mutex<server::ServerProcess>>,
|
||||||
|
) -> io::Result<()> {
|
||||||
|
let mut force = false;
|
||||||
|
|
||||||
|
// We only register terminating signals, so we don't need to differentiate between what kind of
|
||||||
|
// signal came in
|
||||||
|
for _ in signals {
|
||||||
|
// If term is already true, this is the second signal, meaning we kill the process
|
||||||
|
// immediately.
|
||||||
|
// This will currently not work, as the initial stop command will block the kill from
|
||||||
|
// happening.
|
||||||
|
if force {
|
||||||
|
let mut server = counter.lock().unwrap();
|
||||||
|
return server.kill();
|
||||||
|
}
|
||||||
|
// The stop command runs in a separate thread to avoid blocking the signal handling loop.
|
||||||
|
// After stopping the server, the thread terminates the process.
|
||||||
|
else {
|
||||||
|
let clone = Arc::clone(&counter);
|
||||||
|
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
let mut server = clone.lock().unwrap();
|
||||||
|
let _ = server.stop();
|
||||||
|
std::process::exit(0);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
force = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
use std::io;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use crate::server;
|
||||||
|
|
||||||
|
pub fn handle_stdin(counter: Arc<Mutex<server::ServerProcess>>) {
|
||||||
|
let stdin = io::stdin();
|
||||||
|
let input = &mut String::new();
|
||||||
|
|
||||||
|
loop {
|
||||||
|
input.clear();
|
||||||
|
|
||||||
|
if stdin.read_line(input).is_err() {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
{
|
||||||
|
let mut server = counter.lock().unwrap();
|
||||||
|
|
||||||
|
if let Err(e) = server.send_command(input) {
|
||||||
|
println!("{}", e);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if input.trim() == "stop" {
|
||||||
|
std::process::exit(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue