[#24] Merge branch 'develop' into 24-diesel

master^2
Jef Roosens 2021-04-16 18:57:08 +02:00
commit 915ab9ea25
Signed by: Jef Roosens
GPG Key ID: 955C0660072F691F
20 changed files with 227 additions and 243 deletions

View File

@ -1,8 +1,8 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# This hook lints the code, and if we're on develop or master, also forces the tests to pass. # This hook lints the code, and if we're on develop or master, also forces the tests to pass.
make lint &> /dev/null 2>&1 || { ./fejctl lint &> /dev/null 2>&1 || {
>&2 echo "Format check failed, use 'make lint' for more information."; >&2 echo "Format check failed, use './fejctl lint' for more information.";
exit 1; exit 1;
} }

View File

@ -5,19 +5,20 @@ authors = ["Jef Roosens <roosensjef@gmail.com>"]
edition = "2018" edition = "2018"
[lib] [lib]
name = "fej_lib" name = "fej"
src = "src/lib.rs" path = "src/fej/lib.rs"
test = true test = true
bench = true bench = true
doc = true doc = true
doctest = true doctest = true
[[bin]] [[bin]]
name = "fej" name = "server"
src = "src/main.rs" path = "src/server/main.rs"
test = false test = true
bench = false bench = true
doc = false doc = true
doctest = true
[dependencies] [dependencies]
rocket = "0.4.7" rocket = "0.4.7"

View File

@ -1,76 +0,0 @@
all: debug
.PHONY: all
# Builds the debug release inside the Alpine container. For build caching, two
# volumes are used named `fej_build-cache` and `fej_registry-cache`. These
# images are automatically created for you if they don't exist. If you
# encounter any strange build errors, you can try removing this volumes to
# start a completely fresh build.
debug:
@ ./build -m dev -a run build
.PHONY: debug
# Builds the release version. In contrary to the debug version, this build
# doesn't use volumes for caching, as this would require the build to happen
# during runtime instead of during the building of the image. Instead, it uses
# the new `--mount` feature from Buildkit. This does mean that only very recent
# Docker engines can build the release version (in my case, at the time of
# writing this, 20.10.5).
release:
@ ./build -m rel
.PHONY: release
# This builds the release version, and pushes all relevant tags to my Docker
# Hub repository, namely chewingbever/fej
push:
@ ./build -m rel -a push
.PHONY: push
# This builds the debug release, and runs it detached. The reason we detach the
# container is because Rocket has a tendency to ignore ctlr-c when inside a
# container, which gets annoying really fast.
run:
@ ./build -m dev -a run
.PHONY: run
# As a workaround, we just have a stop command that stops the container.
stop:
@ docker stop -t 2 fej
@ docker stop fej_db
.PHONY: stop
# This attaches to the running container, essentially giving the same result as
# just running `cargo run` locally.
logs:
@ docker logs -f fej
.PHONY: logs
# This just starts up a shell inside the fej container
sh:
@ docker exec -it fej sh
.PHONY: sh
# Starts a psql session in the database container
dbsh:
@ docker exec -it fej_db psql -U fej -d fej
# Builds the debug version, and runs the tests (but doesn't detach).
test:
@ ./build -m dev -a run -l -- test --no-fail-fast
.PHONY: test
# Runs the cargo code formatter on your code.
format:
@ cargo fmt
.PHONY: format
# Lints your code. This also gets run in the pre-commit hook, basically
# preventing you from committing badly-formatted code.
lint:
@ cargo fmt -- --check
.PHONY: lint
# This builds the documentation for the project, excluding the documentation.
docs:
@ cargo doc --no-deps
.PHONY: docs

View File

@ -1,25 +1,17 @@
# Fej # Fej
Fej is an API written in Rust. I started this project to learn the language, Fej is a RESTful API that does lots of different things. It started as an
and really just have some fun. experiment to learn Rust, but has grown into a full-on passion project.
## Project Structure ## Project Structure
The folder structure follows the structure of the URLs, e.g. the route for The `src` folder contains subfolders for the various binaries and the main
`/hello/world` is found in the module `src/hello`. library, called `fej`. The biggest binary is called `server`, which is the
binary that actually runs the Rocket.rs web server. All the others are utility
programs, mostly consisting of scrapers for various services.
Each module contains the following base files: Version 1.1 also introduces the use of a database, namely
[PostgreSQL 13](https://www.postgresql.org/).
* `mod.rs`: defines the modules' content, and contains the route definitions.
The route functions themselves only contain the functionality needed to
represent the data, not acquire it.
* `controller.rs`: this file contains the actual logic of each route. If the
logic becomes too complicated to be contained inside a single file,
`controller.rs` becomes its own module folder named `controller`.
* `tests.rs`: this contains tests for the specific module. This can also be a
module directory if need be.
Every module has a `routes` function that returns its route macros.
## Roadmap ## Roadmap
@ -27,19 +19,33 @@ See [roadmap.md](roadmap.md).
## Development ## Development
The entire toolchain runs on Alpine inside Docker. This makes building easier, To make development more consistent (and keep my computer a bit cleaner) I've
and (hopefully) eliminates any system-specific bugs. decided to run the entire toolchain using Docker, with Alpine Linux base
images. This also allows me to make really small final images. Technically,
Docker is the only dependency you need to contribute to this project
(not accounting for language servers etc.).
A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as A [Bash script](fejctl) is provided to speed up development on the various
all the commands are documented for your understanding ;) binaries. It aids in starting up the containers, choosing which binary to run
etc. A quick rundown of its most important features:
There's also the `build` script. This script does all the "heavy" lifting. It ```bash
chooses which Dockerfile to build according to the given arguments, and # By default, it compiles the 'server' binary (but doesn't run it)
generates tags for the images (useful when pushing releases). The Makefile is ./fejctl
really just a wrapper around this build script, allowing you to write
`make test` instead of `./build -m dev -a run test`.
tl;dr run `make run` to run your build, and `make test` to run the tests. # The first argument is the action you wish to perform, the second on which
# binary (not all commands need a binary as input, so they just ignore it)
# For example, this will run the binary called server
./fejctl r server
# This runs the tests (all commands also have their full name if you want)
./fejctl t
./fejctl test
# These attach to the two containers
./fejctl sh # Opens a new root shell inside the 'fej' container
./fejctl psql # Open a new psql shell in the fej database
```
## Docker images ## Docker images

108
build
View File

@ -1,108 +0,0 @@
#!/usr/bin/env bash
image="chewingbever/fej"
# Should be either dev or rel
mode="dev"
action=""
attach="--detach"
while getopts ":i:m:a:l" c; do
case $c in
i ) image="$OPTARG" ;;
m ) mode="$OPTARG" ;;
a ) action="$OPTARG" ;;
l ) attach="" ;;
? ) exit 1 ;;
esac
done
shift $((OPTIND-1))
# Extract current version from Cargo.toml & get current branch
patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1`
major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'`
minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'`
branch=`git rev-parse --abbrev-ref HEAD`
if [[ "$branch" = "master" ]]; then
tags=("$patch_version" "$minor_version" "$major_version" "latest")
elif [[ "$branch" = "develop" ]]; then
tags=("$patch_version-dev" "$minor_version-dev" "$major_version-dev" "dev")
else
tags=("$branch")
fi
# First, we build the builder
DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" .
# Run the actual build command
if [ "$mode" = "rel" ]; then
DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel .
elif [[ "$mode" = "dev" ]]; then
DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev .
else
>&2 echo "Invalid mode."
exit 1
fi
if [[ "$action" = push ]]; then
[[ "$branch" =~ ^develop|master$ ]] || {
>&2 echo "You can only push from develop or master."
exit 2
}
[[ "$mode" = "rel" ]] || {
>&2 echo "You can only push release builds."
exit 3
}
for tag in "${tags[@]}"; do
# Create the tag
docker tag "$image:$tags" "$image:$tag"
# Push the tag
docker push "$image:$tag"
# Remove the tag again, if it's not the main tag
[[ "$tag" != "$tags" ]] && docker rmi "$image:$tag"
done
elif [[ "$action" = run ]]; then
# Create the network & start the database container
docker network create fej
docker volume create fej_db-data
docker run --rm \
--detach \
--name fej_db \
--network fej \
-p 5432:5432 \
-e "POSTGRES_DB=fej" \
-e "POSTGRES_USER=fej" \
-e "POSTGRES_PASSWORD=fej" \
-v 'fej_db-data:/var/lib/postgresql/data' \
postgres:13-alpine
if [[ "$mode" = "dev" ]]; then
# Create caching volumes if needed (they need to be named)
docker volume create fej_build-cache
docker volume create fej_registry-cache
flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry"
fi
docker run $attach $flags \
--rm \
--interactive \
--tty \
--publish 8000:8000 \
--name fej \
--env-file .env.container \
--network fej \
"$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@"
fi

View File

@ -3,4 +3,4 @@ FROM chewingbever/fej-builder:latest
ENV RUST_BACKTRACE 1 ENV RUST_BACKTRACE 1
ENTRYPOINT ["cargo"] ENTRYPOINT ["cargo"]
CMD ["run"] CMD ["run", "--bin", "server"]

View File

@ -7,7 +7,7 @@ FROM chewingbever/fej-builder:latest AS builder
# NOTE: cargo install auto-appends bin to the path # NOTE: cargo install auto-appends bin to the path
RUN --mount=type=cache,target=/usr/src/app/target \ RUN --mount=type=cache,target=/usr/src/app/target \
--mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/root/.cargo/registry \
cargo install --path . --bin fej --root /usr/local cargo install --path . --root /usr/local
# Now, we create the actual image # Now, we create the actual image
@ -17,7 +17,7 @@ FROM alpine:latest
RUN apk update && apk add --no-cache openssl libgcc curl RUN apk update && apk add --no-cache openssl libgcc curl
# Copy binary over to final image # Copy binary over to final image
COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej COPY --from=builder /usr/local/bin/server /usr/local/bin/server
HEALTHCHECK \ HEALTHCHECK \
--interval=10s \ --interval=10s \
@ -26,4 +26,4 @@ HEALTHCHECK \
--retries=3 \ --retries=3 \
CMD curl -q localhost:8000 CMD curl -q localhost:8000
CMD ["/usr/local/bin/fej"] CMD ["/usr/local/bin/server"]

159
fejctl 100755
View File

@ -0,0 +1,159 @@
#!/usr/bin/env bash
image='chewingbever/fej'
# Creates the needed images
#
# $1: wether to build the debug or the release image (default debug)
function create_images() {
# First, we build the builder
DOCKER_BUILDKIT=1 docker build \
-f docker/Dockerfile.builder \
-t "$image-builder:latest" .
if [[ "$1" = "rel" ]]; then
DOCKER_BUILDKIT=1 docker build \
-t "$image:latest" \
-f docker/Dockerfile.rel .
else
# Then, we create the debug image
DOCKER_BUILDKIT=1 docker build \
-t "$image:dev" \
-f docker/Dockerfile.dev .
fi
}
# Execute the debug image (must be built first)
#
# $@: the arguments to pass to the image (passed as arguments to cargo)
function run_image() {
# Run the database image
docker run --rm \
--detach \
--name fej_db \
--network fej \
-p 5432:5432 \
-e 'POSTGRES_DB=fej' \
-e 'POSTGRES_USER=fej' \
-e 'POSTGRES_PASSWORD=fej' \
-v 'fej_db-data:/var/lib/postgresql/data' \
postgres:13-alpine
# Run the binary image
docker run \
--detach \
--rm \
--interactive \
--tty \
--publish 8000:8000 \
--name fej \
--env-file .env.container \
--network fej \
-v 'fej_build-cache:/usr/src/app/target' \
-v 'fej_registry-cache:/root/.cargo/registry' \
"$image:dev" "$@"
}
# Attach to the fej container
function logs() {
docker logs -f fej
}
# Builds the given binary
#
# $1: the binary to build
function build() {
create_images
run_image build --bin "$1"
logs
}
# Runs the given binary
#
# $1: the binary to run
function run() {
create_images
run_image run --bin "$1"
logs
}
# Runs the tests
function tests() {
create_images
run_image test --no-fail-fast
logs
}
# Stops both containers
function stop() {
docker stop fej_db
docker stop -t 0 fej
}
function run_release() {
echo "Not implemented"
}
# Tags & pushes the release version to Docker Hub
function publish() {
branch=`git rev-parse --abbrev-ref HEAD`
if [[ "$branch" != master ]]; then
>&2 echo "You can only publish from master."
exit 2
fi
patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1`
major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'`
minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'`
tags=("latest" "$patch_version" "$minor_version" "$major_version")
for tag in "${tags[@]}"; do
# Create the tag
docker tag "$image:$tags" "$image:$tag"
# Push the tag
docker push "$image:$tag"
# Remove the tag again, if it's not the main tag
[[ "$tag" != "$tags" ]] && docker rmi "$image:$tag"
done
}
# Entrypoint to the script
#
# $1: action to perform, defaults to 'build'
# $2: binary to use, defaults to 'server'
function main() {
# Default values
cmd="${1:-build}"
bin="${2:-server}"
case $cmd in
# Building
b | build ) build "$bin" ;;
br | build-release ) create_images rel ;;
# Running
r | run ) run "$bin" ;;
rr | run-release ) run_release ;;
s | stop ) stop ;;
# Ease of life
psql ) docker exec -it fej_db psql -U fej -d fej ;;
sh ) docker exec -it fej sh ;;
# Misc
docs ) cargo doc --no-deps ;;
format ) cargo fmt ;;
l | logs ) logs ;;
lint ) cargo fmt -- --check ;;
p | push | publish ) publish ;;
t | test ) tests ;;
* ) >&2 echo "Invalid command."; exit 1 ;;
esac
}
main "$@"

View File

@ -69,8 +69,8 @@ setting up a database for this version anyways.
## Kissanime ## Kissanime
I like watching anime from time to time, and I've always used Kissanime for I like watching anime from time to time, and I've always used Kissanime for
this. However, their sites can be quite slow, and riddled with ads from time to this. However, their sites can be quite slow, and riddled with ads. That's why
time. That's why I'd like to create a high-speed wrapper that extracts all the I'd like to create a high-speed wrapper that extracts all the needed info from
needed info from their sites, removing the need for the user to ever actually their sites, removing the need for the user to ever actually visit their site.
visit their site. The API can just act as a fast search index, complete with The API can just act as a fast search index, complete with indexing of the
indexing of the links to the videos and everything. links to the videos and everything.

View File

@ -10,6 +10,8 @@ pub enum FejError {
FailedRequest, FailedRequest,
} }
// I'd love to move this over to the server binary, but right now, error E0117 is making that
// imopssible
impl From<FejError> for Status { impl From<FejError> for Status {
fn from(err: FejError) -> Status { fn from(err: FejError) -> Status {
match err { match err {

View File

@ -1,11 +1,7 @@
#![feature(proc_macro_hygiene, decl_macro)] #![feature(proc_macro_hygiene, decl_macro)]
#[macro_use]
extern crate rocket;
// Route modules // Route modules
pub mod ivago; pub mod ivago;
// Helper modules // Helper modules
pub mod catchers;
pub mod errors; pub mod errors;

View File

@ -1,7 +1,13 @@
#![feature(proc_macro_hygiene, decl_macro)]
#[macro_use] #[macro_use]
extern crate rocket; extern crate rocket;
use fej_lib::{catchers, ivago}; #[cfg(test)]
mod tests;
mod catchers;
mod routes;
// Very temporary solution for CORS // Very temporary solution for CORS
// https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs
@ -33,7 +39,7 @@ impl Fairing for CORS {
fn rocket() -> rocket::Rocket { fn rocket() -> rocket::Rocket {
rocket::ignite() rocket::ignite()
.attach(CORS) .attach(CORS)
.mount("/ivago", ivago::routes()) .mount("/ivago", routes::ivago())
.register(catchers![catchers::not_found]) .register(catchers![catchers::not_found])
} }

View File

@ -1,14 +1,7 @@
mod controller; use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street};
use controller::{get_pickup_times, search_streets};
use controller::{BasicDate, PickupTime, Street};
use rocket::http::Status; use rocket::http::Status;
use rocket_contrib::json::Json; use rocket_contrib::json::Json;
pub fn routes() -> Vec<rocket::Route> {
routes![route_search_streets, route_get_pickup_times]
}
/// This route handles the Ivago search endpoint. It returns a list of streets, /// This route handles the Ivago search endpoint. It returns a list of streets,
/// consisting of a street name & a city. /// consisting of a street name & a city.
/// ///

View File

@ -0,0 +1,5 @@
mod ivago;
pub fn ivago() -> Vec<rocket::Route> {
routes![ivago::route_search_streets, ivago::route_get_pickup_times]
}

View File

@ -3,7 +3,7 @@ use rocket::http::Status;
use rocket::local::Client; use rocket::local::Client;
fn rocket() -> rocket::Rocket { fn rocket() -> rocket::Rocket {
rocket::ignite().mount("/", fej_lib::ivago::routes()) rocket::ignite().mount("/", super::routes::ivago())
} }
/// Test 404 response /// Test 404 response