diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 709fb95..fe55771 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -1,8 +1,8 @@ #!/usr/bin/env bash # This hook lints the code, and if we're on develop or master, also forces the tests to pass. -make lint &> /dev/null 2>&1 || { - >&2 echo "Format check failed, use 'make lint' for more information."; +./fejctl lint &> /dev/null 2>&1 || { + >&2 echo "Format check failed, use './fejctl lint' for more information."; exit 1; } diff --git a/Cargo.toml b/Cargo.toml index f359a51..4081b85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,19 +5,20 @@ authors = ["Jef Roosens "] edition = "2018" [lib] -name = "fej_lib" -src = "src/lib.rs" +name = "fej" +path = "src/fej/lib.rs" test = true bench = true doc = true doctest = true [[bin]] -name = "fej" -src = "src/main.rs" -test = false -bench = false -doc = false +name = "server" +path = "src/server/main.rs" +test = true +bench = true +doc = true +doctest = true [dependencies] rocket = "0.4.7" diff --git a/Makefile b/Makefile deleted file mode 100644 index 5d8c22e..0000000 --- a/Makefile +++ /dev/null @@ -1,76 +0,0 @@ -all: debug -.PHONY: all - -# Builds the debug release inside the Alpine container. For build caching, two -# volumes are used named `fej_build-cache` and `fej_registry-cache`. These -# images are automatically created for you if they don't exist. If you -# encounter any strange build errors, you can try removing this volumes to -# start a completely fresh build. -debug: - @ ./build -m dev -a run build -.PHONY: debug - -# Builds the release version. In contrary to the debug version, this build -# doesn't use volumes for caching, as this would require the build to happen -# during runtime instead of during the building of the image. Instead, it uses -# the new `--mount` feature from Buildkit. This does mean that only very recent -# Docker engines can build the release version (in my case, at the time of -# writing this, 20.10.5). -release: - @ ./build -m rel -.PHONY: release - -# This builds the release version, and pushes all relevant tags to my Docker -# Hub repository, namely chewingbever/fej -push: - @ ./build -m rel -a push -.PHONY: push - -# This builds the debug release, and runs it detached. The reason we detach the -# container is because Rocket has a tendency to ignore ctlr-c when inside a -# container, which gets annoying really fast. -run: - @ ./build -m dev -a run -.PHONY: run - -# As a workaround, we just have a stop command that stops the container. -stop: - @ docker stop -t 2 fej - @ docker stop fej_db -.PHONY: stop - -# This attaches to the running container, essentially giving the same result as -# just running `cargo run` locally. -logs: - @ docker logs -f fej -.PHONY: logs - -# This just starts up a shell inside the fej container -sh: - @ docker exec -it fej sh -.PHONY: sh - -# Starts a psql session in the database container -dbsh: - @ docker exec -it fej_db psql -U fej -d fej - -# Builds the debug version, and runs the tests (but doesn't detach). -test: - @ ./build -m dev -a run -l -- test --no-fail-fast -.PHONY: test - -# Runs the cargo code formatter on your code. -format: - @ cargo fmt -.PHONY: format - -# Lints your code. This also gets run in the pre-commit hook, basically -# preventing you from committing badly-formatted code. -lint: - @ cargo fmt -- --check -.PHONY: lint - -# This builds the documentation for the project, excluding the documentation. -docs: - @ cargo doc --no-deps -.PHONY: docs diff --git a/README.md b/README.md index 55d11e3..e2573af 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,17 @@ # Fej -Fej is an API written in Rust. I started this project to learn the language, -and really just have some fun. +Fej is a RESTful API that does lots of different things. It started as an +experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The folder structure follows the structure of the URLs, e.g. the route for -`/hello/world` is found in the module `src/hello`. +The `src` folder contains subfolders for the various binaries and the main +library, called `fej`. The biggest binary is called `server`, which is the +binary that actually runs the Rocket.rs web server. All the others are utility +programs, mostly consisting of scrapers for various services. -Each module contains the following base files: - -* `mod.rs`: defines the modules' content, and contains the route definitions. - The route functions themselves only contain the functionality needed to - represent the data, not acquire it. -* `controller.rs`: this file contains the actual logic of each route. If the - logic becomes too complicated to be contained inside a single file, - `controller.rs` becomes its own module folder named `controller`. -* `tests.rs`: this contains tests for the specific module. This can also be a - module directory if need be. - -Every module has a `routes` function that returns its route macros. +Version 1.1 also introduces the use of a database, namely +[PostgreSQL 13](https://www.postgresql.org/). ## Roadmap @@ -27,19 +19,33 @@ See [roadmap.md](roadmap.md). ## Development -The entire toolchain runs on Alpine inside Docker. This makes building easier, -and (hopefully) eliminates any system-specific bugs. +To make development more consistent (and keep my computer a bit cleaner) I've +decided to run the entire toolchain using Docker, with Alpine Linux base +images. This also allows me to make really small final images. Technically, +Docker is the only dependency you need to contribute to this project +(not accounting for language servers etc.). -A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as -all the commands are documented for your understanding ;) +A [Bash script](fejctl) is provided to speed up development on the various +binaries. It aids in starting up the containers, choosing which binary to run +etc. A quick rundown of its most important features: -There's also the `build` script. This script does all the "heavy" lifting. It -chooses which Dockerfile to build according to the given arguments, and -generates tags for the images (useful when pushing releases). The Makefile is -really just a wrapper around this build script, allowing you to write -`make test` instead of `./build -m dev -a run test`. +```bash +# By default, it compiles the 'server' binary (but doesn't run it) +./fejctl -tl;dr run `make run` to run your build, and `make test` to run the tests. +# The first argument is the action you wish to perform, the second on which +# binary (not all commands need a binary as input, so they just ignore it) +# For example, this will run the binary called server +./fejctl r server + +# This runs the tests (all commands also have their full name if you want) +./fejctl t +./fejctl test + +# These attach to the two containers +./fejctl sh # Opens a new root shell inside the 'fej' container +./fejctl psql # Open a new psql shell in the fej database +``` ## Docker images diff --git a/build b/build deleted file mode 100755 index 271a3d9..0000000 --- a/build +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash - -image="chewingbever/fej" -# Should be either dev or rel -mode="dev" -action="" -attach="--detach" - -while getopts ":i:m:a:l" c; do - case $c in - i ) image="$OPTARG" ;; - m ) mode="$OPTARG" ;; - a ) action="$OPTARG" ;; - l ) attach="" ;; - ? ) exit 1 ;; - esac -done -shift $((OPTIND-1)) - -# Extract current version from Cargo.toml & get current branch -patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` -major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` -minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` -branch=`git rev-parse --abbrev-ref HEAD` - -if [[ "$branch" = "master" ]]; then - tags=("$patch_version" "$minor_version" "$major_version" "latest") - -elif [[ "$branch" = "develop" ]]; then - tags=("$patch_version-dev" "$minor_version-dev" "$major_version-dev" "dev") - -else - tags=("$branch") - -fi - -# First, we build the builder -DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" . - -# Run the actual build command -if [ "$mode" = "rel" ]; then - DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel . - -elif [[ "$mode" = "dev" ]]; then - DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev . - -else - >&2 echo "Invalid mode." - exit 1 - -fi - -if [[ "$action" = push ]]; then - [[ "$branch" =~ ^develop|master$ ]] || { - >&2 echo "You can only push from develop or master." - exit 2 - } - - [[ "$mode" = "rel" ]] || { - >&2 echo "You can only push release builds." - exit 3 - } - - for tag in "${tags[@]}"; do - # Create the tag - docker tag "$image:$tags" "$image:$tag" - - # Push the tag - docker push "$image:$tag" - - # Remove the tag again, if it's not the main tag - [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" - done - -elif [[ "$action" = run ]]; then - # Create the network & start the database container - docker network create fej - docker volume create fej_db-data - - docker run --rm \ - --detach \ - --name fej_db \ - --network fej \ - -p 5432:5432 \ - -e "POSTGRES_DB=fej" \ - -e "POSTGRES_USER=fej" \ - -e "POSTGRES_PASSWORD=fej" \ - -v 'fej_db-data:/var/lib/postgresql/data' \ - postgres:13-alpine - - if [[ "$mode" = "dev" ]]; then - # Create caching volumes if needed (they need to be named) - docker volume create fej_build-cache - docker volume create fej_registry-cache - - flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry" - fi - - docker run $attach $flags \ - --rm \ - --interactive \ - --tty \ - --publish 8000:8000 \ - --name fej \ - --env-file .env.container \ - --network fej \ - "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" -fi diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 9205fd4..d2029af 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -3,4 +3,4 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 ENTRYPOINT ["cargo"] -CMD ["run"] +CMD ["run", "--bin", "server"] diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 6b4e438..b64916a 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -7,7 +7,7 @@ FROM chewingbever/fej-builder:latest AS builder # NOTE: cargo install auto-appends bin to the path RUN --mount=type=cache,target=/usr/src/app/target \ --mount=type=cache,target=/root/.cargo/registry \ - cargo install --path . --bin fej --root /usr/local + cargo install --path . --root /usr/local # Now, we create the actual image @@ -17,7 +17,7 @@ FROM alpine:latest RUN apk update && apk add --no-cache openssl libgcc curl # Copy binary over to final image -COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej +COPY --from=builder /usr/local/bin/server /usr/local/bin/server HEALTHCHECK \ --interval=10s \ @@ -26,4 +26,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -CMD ["/usr/local/bin/fej"] +CMD ["/usr/local/bin/server"] diff --git a/fejctl b/fejctl new file mode 100755 index 0000000..c977f79 --- /dev/null +++ b/fejctl @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +image='chewingbever/fej' + +# Creates the needed images +# +# $1: wether to build the debug or the release image (default debug) +function create_images() { + # First, we build the builder + DOCKER_BUILDKIT=1 docker build \ + -f docker/Dockerfile.builder \ + -t "$image-builder:latest" . + + if [[ "$1" = "rel" ]]; then + DOCKER_BUILDKIT=1 docker build \ + -t "$image:latest" \ + -f docker/Dockerfile.rel . + + else + # Then, we create the debug image + DOCKER_BUILDKIT=1 docker build \ + -t "$image:dev" \ + -f docker/Dockerfile.dev . + fi +} + +# Execute the debug image (must be built first) +# +# $@: the arguments to pass to the image (passed as arguments to cargo) +function run_image() { + # Run the database image + docker run --rm \ + --detach \ + --name fej_db \ + --network fej \ + -p 5432:5432 \ + -e 'POSTGRES_DB=fej' \ + -e 'POSTGRES_USER=fej' \ + -e 'POSTGRES_PASSWORD=fej' \ + -v 'fej_db-data:/var/lib/postgresql/data' \ + postgres:13-alpine + + # Run the binary image + docker run \ + --detach \ + --rm \ + --interactive \ + --tty \ + --publish 8000:8000 \ + --name fej \ + --env-file .env.container \ + --network fej \ + -v 'fej_build-cache:/usr/src/app/target' \ + -v 'fej_registry-cache:/root/.cargo/registry' \ + "$image:dev" "$@" +} + +# Attach to the fej container +function logs() { + docker logs -f fej +} + +# Builds the given binary +# +# $1: the binary to build +function build() { + create_images + run_image build --bin "$1" + logs +} + +# Runs the given binary +# +# $1: the binary to run +function run() { + create_images + run_image run --bin "$1" + logs +} + +# Runs the tests +function tests() { + create_images + run_image test --no-fail-fast + logs +} + +# Stops both containers +function stop() { + docker stop fej_db + docker stop -t 0 fej +} + +function run_release() { + echo "Not implemented" +} + +# Tags & pushes the release version to Docker Hub +function publish() { + branch=`git rev-parse --abbrev-ref HEAD` + + if [[ "$branch" != master ]]; then + >&2 echo "You can only publish from master." + exit 2 + fi + + patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` + major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` + minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` + tags=("latest" "$patch_version" "$minor_version" "$major_version") + + for tag in "${tags[@]}"; do + # Create the tag + docker tag "$image:$tags" "$image:$tag" + + # Push the tag + docker push "$image:$tag" + + # Remove the tag again, if it's not the main tag + [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" + done + +} + +# Entrypoint to the script +# +# $1: action to perform, defaults to 'build' +# $2: binary to use, defaults to 'server' +function main() { + # Default values + cmd="${1:-build}" + bin="${2:-server}" + + case $cmd in + # Building + b | build ) build "$bin" ;; + br | build-release ) create_images rel ;; + + # Running + r | run ) run "$bin" ;; + rr | run-release ) run_release ;; + s | stop ) stop ;; + + # Ease of life + psql ) docker exec -it fej_db psql -U fej -d fej ;; + sh ) docker exec -it fej sh ;; + + # Misc + docs ) cargo doc --no-deps ;; + format ) cargo fmt ;; + l | logs ) logs ;; + lint ) cargo fmt -- --check ;; + p | push | publish ) publish ;; + t | test ) tests ;; + * ) >&2 echo "Invalid command."; exit 1 ;; + esac +} + +main "$@" diff --git a/roadmap.md b/roadmap.md index f03e4b5..bfdd622 100644 --- a/roadmap.md +++ b/roadmap.md @@ -69,8 +69,8 @@ setting up a database for this version anyways. ## Kissanime I like watching anime from time to time, and I've always used Kissanime for -this. However, their sites can be quite slow, and riddled with ads from time to -time. That's why I'd like to create a high-speed wrapper that extracts all the -needed info from their sites, removing the need for the user to ever actually -visit their site. The API can just act as a fast search index, complete with -indexing of the links to the videos and everything. +this. However, their sites can be quite slow, and riddled with ads. That's why +I'd like to create a high-speed wrapper that extracts all the needed info from +their sites, removing the need for the user to ever actually visit their site. +The API can just act as a fast search index, complete with indexing of the +links to the videos and everything. diff --git a/src/errors.rs b/src/fej/errors.rs similarity index 89% rename from src/errors.rs rename to src/fej/errors.rs index afcf04a..730714c 100644 --- a/src/errors.rs +++ b/src/fej/errors.rs @@ -10,6 +10,8 @@ pub enum FejError { FailedRequest, } +// I'd love to move this over to the server binary, but right now, error E0117 is making that +// imopssible impl From for Status { fn from(err: FejError) -> Status { match err { diff --git a/src/ivago/controller/basic_date.rs b/src/fej/ivago/basic_date.rs similarity index 100% rename from src/ivago/controller/basic_date.rs rename to src/fej/ivago/basic_date.rs diff --git a/src/ivago/controller/mod.rs b/src/fej/ivago/mod.rs similarity index 100% rename from src/ivago/controller/mod.rs rename to src/fej/ivago/mod.rs diff --git a/src/ivago/controller/pickup_time.rs b/src/fej/ivago/pickup_time.rs similarity index 100% rename from src/ivago/controller/pickup_time.rs rename to src/fej/ivago/pickup_time.rs diff --git a/src/ivago/controller/street.rs b/src/fej/ivago/street.rs similarity index 100% rename from src/ivago/controller/street.rs rename to src/fej/ivago/street.rs diff --git a/src/lib.rs b/src/fej/lib.rs similarity index 67% rename from src/lib.rs rename to src/fej/lib.rs index b1e2733..761c91b 100644 --- a/src/lib.rs +++ b/src/fej/lib.rs @@ -1,11 +1,7 @@ #![feature(proc_macro_hygiene, decl_macro)] -#[macro_use] -extern crate rocket; - // Route modules pub mod ivago; // Helper modules -pub mod catchers; pub mod errors; diff --git a/src/catchers.rs b/src/server/catchers.rs similarity index 100% rename from src/catchers.rs rename to src/server/catchers.rs diff --git a/src/main.rs b/src/server/main.rs similarity index 88% rename from src/main.rs rename to src/server/main.rs index 8b8edc7..58b0300 100644 --- a/src/main.rs +++ b/src/server/main.rs @@ -1,7 +1,13 @@ +#![feature(proc_macro_hygiene, decl_macro)] + #[macro_use] extern crate rocket; -use fej_lib::{catchers, ivago}; +#[cfg(test)] +mod tests; + +mod catchers; +mod routes; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs @@ -33,7 +39,7 @@ impl Fairing for CORS { fn rocket() -> rocket::Rocket { rocket::ignite() .attach(CORS) - .mount("/ivago", ivago::routes()) + .mount("/ivago", routes::ivago()) .register(catchers![catchers::not_found]) } diff --git a/src/ivago/mod.rs b/src/server/routes/ivago.rs similarity index 83% rename from src/ivago/mod.rs rename to src/server/routes/ivago.rs index 627e89b..d8318bd 100644 --- a/src/ivago/mod.rs +++ b/src/server/routes/ivago.rs @@ -1,14 +1,7 @@ -mod controller; - -use controller::{get_pickup_times, search_streets}; -use controller::{BasicDate, PickupTime, Street}; +use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street}; use rocket::http::Status; use rocket_contrib::json::Json; -pub fn routes() -> Vec { - routes![route_search_streets, route_get_pickup_times] -} - /// This route handles the Ivago search endpoint. It returns a list of streets, /// consisting of a street name & a city. /// diff --git a/src/server/routes/mod.rs b/src/server/routes/mod.rs new file mode 100644 index 0000000..449f82f --- /dev/null +++ b/src/server/routes/mod.rs @@ -0,0 +1,5 @@ +mod ivago; + +pub fn ivago() -> Vec { + routes![ivago::route_search_streets, ivago::route_get_pickup_times] +} diff --git a/tests/ivago.rs b/src/server/tests.rs similarity index 93% rename from tests/ivago.rs rename to src/server/tests.rs index d78501a..8871762 100644 --- a/tests/ivago.rs +++ b/src/server/tests.rs @@ -3,7 +3,7 @@ use rocket::http::Status; use rocket::local::Client; fn rocket() -> rocket::Rocket { - rocket::ignite().mount("/", fej_lib::ivago::routes()) + rocket::ignite().mount("/", super::routes::ivago()) } /// Test 404 response