[#24] Merge branch 'develop' into 24-diesel

.hooks/pre-commit

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
 # This hook lints the code, and if we're on develop or master, also forces the tests to pass.
-make lint &> /dev/null 2>&1 || {
-    >&2 echo "Format check failed, use 'make lint' for more information.";
+./fejctl lint &> /dev/null 2>&1 || {
+    >&2 echo "Format check failed, use './fejctl lint' for more information.";
     exit 1;
 }
 

Cargo.toml

@@ -5,19 +5,20 @@ authors = ["Jef Roosens <roosensjef@gmail.com>"]
 edition = "2018"
 
 [lib]
-name = "fej_lib"
-src = "src/lib.rs"
+name = "fej"
+path = "src/fej/lib.rs"
 test = true
 bench = true
 doc = true
 doctest = true
 
 [[bin]]
-name = "fej"
-src = "src/main.rs"
-test = false
-bench = false
-doc = false
+name = "server"
+path = "src/server/main.rs"
+test = true
+bench = true
+doc = true
+doctest = true
  
 [dependencies]
 rocket = "0.4.7"

Makefile

@@ -1,76 +0,0 @@
-all: debug
-.PHONY: all
-
-# Builds the debug release inside the Alpine container. For build caching, two
-# volumes are used named `fej_build-cache` and `fej_registry-cache`. These
-# images are automatically created for you if they don't exist. If you
-# encounter any strange build errors, you can try removing this volumes to
-# start a completely fresh build.
-debug:
-	@ ./build -m dev -a run build
-.PHONY: debug
-
-# Builds the release version. In contrary to the debug version, this build
-# doesn't use volumes for caching, as this would require the build to happen
-# during runtime instead of during the building of the image. Instead, it uses
-# the new `--mount` feature from Buildkit. This does mean that only very recent
-# Docker engines can build the release version (in my case, at the time of
-# writing this, 20.10.5).
-release:
-	@ ./build -m rel
-.PHONY: release
-
-# This builds the release version, and pushes all relevant tags to my Docker
-# Hub repository, namely chewingbever/fej
-push:
-	@ ./build -m rel -a push
-.PHONY: push
-
-# This builds the debug release, and runs it detached. The reason we detach the
-# container is because Rocket has a tendency to ignore ctlr-c when inside a
-# container, which gets annoying really fast.
-run:
-	@ ./build -m dev -a run
-.PHONY: run
-
-# As a workaround, we just have a stop command that stops the container.
-stop:
-	@ docker stop -t 2 fej
-	@ docker stop fej_db
-.PHONY: stop
-
-# This attaches to the running container, essentially giving the same result as
-# just running `cargo run` locally.
-logs:
-	@ docker logs -f fej
-.PHONY: logs
-
-# This just starts up a shell inside the fej container
-sh:
-	@ docker exec -it fej sh
-.PHONY: sh
-
-# Starts a psql session in the database container
-dbsh:
-	@ docker exec -it fej_db psql -U fej -d fej
-
-# Builds the debug version, and runs the tests (but doesn't detach).
-test:
-	@ ./build -m dev -a run -l -- test --no-fail-fast
-.PHONY: test
-
-# Runs the cargo code formatter on your code.
-format:
-	@ cargo fmt
-.PHONY: format
-
-# Lints your code. This also gets run in the pre-commit hook, basically
-# preventing you from committing badly-formatted code.
-lint:
-	@ cargo fmt -- --check
-.PHONY: lint
-
-# This builds the documentation for the project, excluding the documentation.
-docs:
-	@ cargo doc --no-deps
-.PHONY: docs

README.md

@@ -1,25 +1,17 @@
 # Fej
 
-Fej is an API written in Rust. I started this project to learn the language,
-and really just have some fun.
+Fej is a RESTful API that does lots of different things. It started as an
+experiment to learn Rust, but has grown into a full-on passion project.
 
 ## Project Structure
 
-The folder structure follows the structure of the URLs, e.g. the route for
-`/hello/world` is found in the module `src/hello`.
+The `src` folder contains subfolders for the various binaries and the main
+library, called `fej`. The biggest binary is called `server`, which is the
+binary that actually runs the Rocket.rs web server. All the others are utility
+programs, mostly consisting of scrapers for various services.
 
-Each module contains the following base files:
-
-* `mod.rs`: defines the modules' content, and contains the route definitions.
-  The route functions themselves only contain the functionality needed to
-  represent the data, not acquire it.
-* `controller.rs`: this file contains the actual logic of each route. If the
-  logic becomes too complicated to be contained inside a single file,
-  `controller.rs` becomes its own module folder named `controller`.
-* `tests.rs`: this contains tests for the specific module. This can also be a
-  module directory if need be.
-
-Every module has a `routes` function that returns its route macros.
+Version 1.1 also introduces the use of a database, namely
+[PostgreSQL 13](https://www.postgresql.org/).
 
 ## Roadmap
 
@@ -27,19 +19,33 @@ See [roadmap.md](roadmap.md).
 
 ## Development
 
-The entire toolchain runs on Alpine inside Docker. This makes building easier,
-and (hopefully) eliminates any system-specific bugs.
+To make development more consistent (and keep my computer a bit cleaner) I've
+decided to run the entire toolchain using Docker, with Alpine Linux base
+images. This also allows me to make really small final images. Technically,
+Docker is the only dependency you need to contribute to this project
+(not accounting for language servers etc.).
 
-A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as
-all the commands are documented for your understanding ;)
+A [Bash script](fejctl) is provided to speed up development on the various
+binaries. It aids in starting up the containers, choosing which binary to run
+etc. A quick rundown of its most important features:
 
-There's also the `build` script. This script does all the "heavy" lifting. It
-chooses which Dockerfile to build according to the given arguments, and
-generates tags for the images (useful when pushing releases). The Makefile is
-really just a wrapper around this build script, allowing you to write
-`make test` instead of `./build -m dev -a run test`.
+```bash
+# By default, it compiles the 'server' binary (but doesn't run it)
+./fejctl
 
-tl;dr run `make run` to run your build, and `make test` to run the tests.
+# The first argument is the action you wish to perform, the second on which
+# binary (not all commands need a binary as input, so they just ignore it)
+# For example, this will run the binary called server
+./fejctl r server
+
+# This runs the tests (all commands also have their full name if you want)
+./fejctl t
+./fejctl test
+
+# These attach to the two containers
+./fejctl sh   # Opens a new root shell inside the 'fej' container
+./fejctl psql # Open a new psql shell in the fej database
+```
 
 ## Docker images
 

build

@@ -1,108 +0,0 @@
-#!/usr/bin/env bash
-
-image="chewingbever/fej"
-# Should be either dev or rel
-mode="dev"
-action=""
-attach="--detach"
-
-while getopts ":i:m:a:l" c; do
-    case $c in
-        i ) image="$OPTARG" ;;
-        m ) mode="$OPTARG" ;;
-        a ) action="$OPTARG" ;;
-        l ) attach="" ;;
-        ? ) exit 1 ;;
-    esac
-done
-shift $((OPTIND-1))
-
-# Extract current version from Cargo.toml & get current branch
-patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1`
-major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'`
-minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'`
-branch=`git rev-parse --abbrev-ref HEAD`
-
-if [[ "$branch" = "master" ]]; then
-    tags=("$patch_version" "$minor_version" "$major_version" "latest")
-
-elif [[ "$branch" = "develop" ]]; then
-    tags=("$patch_version-dev" "$minor_version-dev" "$major_version-dev" "dev")
-
-else
-    tags=("$branch")
-
-fi
-
-# First, we build the builder
-DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" .
-
-# Run the actual build command
-if [ "$mode" = "rel" ]; then
-    DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel .
-
-elif [[ "$mode" = "dev" ]]; then
-    DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev .
-
-else
-    >&2 echo "Invalid mode."
-    exit 1
-
-fi
-
-if [[ "$action" = push ]]; then
-    [[ "$branch" =~ ^develop|master$ ]] || {
-        >&2 echo "You can only push from develop or master."
-        exit 2
-    }
-
-    [[ "$mode" = "rel" ]] || {
-        >&2 echo "You can only push release builds."
-        exit 3
-    }
-
-    for tag in "${tags[@]}"; do
-        # Create the tag
-        docker tag "$image:$tags" "$image:$tag"
-
-        # Push the tag
-        docker push "$image:$tag"
-
-        # Remove the tag again, if it's not the main tag
-        [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag"
-    done
-
-elif [[ "$action" = run ]]; then
-    # Create the network & start the database container
-    docker network create fej
-    docker volume create fej_db-data
-
-    docker run --rm \
-        --detach \
-        --name fej_db \
-        --network fej \
-        -p 5432:5432 \
-        -e "POSTGRES_DB=fej" \
-        -e "POSTGRES_USER=fej" \
-        -e "POSTGRES_PASSWORD=fej" \
-        -v 'fej_db-data:/var/lib/postgresql/data' \
-        postgres:13-alpine
-
-    if [[ "$mode" = "dev" ]]; then
-        # Create caching volumes if needed (they need to be named)
-        docker volume create fej_build-cache
-        docker volume create fej_registry-cache
-
-        flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry"
-    fi
-
-    docker run $attach $flags \
-        --rm \
-        --interactive \
-        --tty \
-        --publish 8000:8000 \
-        --name fej \
-        --env-file .env.container \
-        --network fej \
-        "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@"
-fi

docker/Dockerfile.dev

@@ -3,4 +3,4 @@ FROM chewingbever/fej-builder:latest
 ENV RUST_BACKTRACE 1
 
 ENTRYPOINT ["cargo"]
-CMD ["run"]
+CMD ["run", "--bin", "server"]

docker/Dockerfile.rel

@@ -7,7 +7,7 @@ FROM chewingbever/fej-builder:latest AS builder
 # NOTE: cargo install auto-appends bin to the path
 RUN --mount=type=cache,target=/usr/src/app/target \
     --mount=type=cache,target=/root/.cargo/registry \
-    cargo install --path . --bin fej --root /usr/local
+    cargo install --path . --root /usr/local
 
 
 # Now, we create the actual image
@@ -17,7 +17,7 @@ FROM alpine:latest
 RUN apk update && apk add --no-cache openssl libgcc curl
 
 # Copy binary over to final image
-COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej
+COPY --from=builder /usr/local/bin/server /usr/local/bin/server
 
 HEALTHCHECK \
     --interval=10s \
@@ -26,4 +26,4 @@ HEALTHCHECK \
     --retries=3 \
     CMD curl -q localhost:8000
 
-CMD ["/usr/local/bin/fej"]
+CMD ["/usr/local/bin/server"]

fejctl

@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+
+image='chewingbever/fej'
+
+# Creates the needed images
+#
+# $1: wether to build the debug or the release image (default debug)
+function create_images() {
+    # First, we build the builder
+    DOCKER_BUILDKIT=1 docker build \
+        -f docker/Dockerfile.builder \
+        -t "$image-builder:latest" .
+
+    if [[ "$1" = "rel" ]]; then
+        DOCKER_BUILDKIT=1 docker build \
+            -t "$image:latest" \
+            -f docker/Dockerfile.rel .
+
+    else
+        # Then, we create the debug image
+        DOCKER_BUILDKIT=1 docker build \
+            -t "$image:dev" \
+            -f docker/Dockerfile.dev .
+    fi
+}
+
+# Execute the debug image (must be built first)
+#
+# $@: the arguments to pass to the image (passed as arguments to cargo)
+function run_image() {
+    # Run the database image
+    docker run --rm \
+        --detach \
+        --name fej_db \
+        --network fej \
+        -p 5432:5432 \
+        -e 'POSTGRES_DB=fej' \
+        -e 'POSTGRES_USER=fej' \
+        -e 'POSTGRES_PASSWORD=fej' \
+        -v 'fej_db-data:/var/lib/postgresql/data' \
+        postgres:13-alpine
+
+    # Run the binary image
+    docker run \
+        --detach \
+        --rm \
+        --interactive \
+        --tty \
+        --publish 8000:8000 \
+        --name fej \
+        --env-file .env.container \
+        --network fej \
+        -v 'fej_build-cache:/usr/src/app/target' \
+        -v 'fej_registry-cache:/root/.cargo/registry' \
+        "$image:dev" "$@"
+}
+
+# Attach to the fej container
+function logs() {
+    docker logs -f fej
+}
+
+# Builds the given binary
+#
+# $1: the binary to build
+function build() {
+    create_images
+    run_image build --bin "$1"
+    logs
+}
+
+# Runs the given binary
+#
+# $1: the binary to run
+function run() {
+    create_images
+    run_image run --bin "$1"
+    logs
+}
+
+# Runs the tests
+function tests() {
+    create_images
+    run_image test --no-fail-fast
+    logs
+}
+
+# Stops both containers
+function stop() {
+    docker stop fej_db
+    docker stop -t 0 fej
+}
+
+function run_release() {
+    echo "Not implemented"
+}
+
+# Tags & pushes the release version to Docker Hub
+function publish() {
+    branch=`git rev-parse --abbrev-ref HEAD`
+
+    if [[ "$branch" != master ]]; then
+        >&2 echo "You can only publish from master."
+        exit 2
+    fi
+
+    patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1`
+    major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'`
+    minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'`
+    tags=("latest" "$patch_version" "$minor_version" "$major_version")
+
+    for tag in "${tags[@]}"; do
+        # Create the tag
+        docker tag "$image:$tags" "$image:$tag"
+
+        # Push the tag
+        docker push "$image:$tag"
+
+        # Remove the tag again, if it's not the main tag
+        [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag"
+    done
+
+}
+
+# Entrypoint to the script
+#
+# $1: action to perform, defaults to 'build'
+# $2: binary to use, defaults to 'server'
+function main() {
+    # Default values
+    cmd="${1:-build}"
+    bin="${2:-server}"
+
+    case $cmd in
+        # Building
+        b | build ) build "$bin" ;;
+        br | build-release ) create_images rel ;;
+
+        # Running
+        r | run ) run "$bin" ;;
+        rr | run-release ) run_release ;;
+        s | stop ) stop ;;
+
+        # Ease of life
+        psql ) docker exec -it fej_db psql -U fej -d fej ;;
+        sh ) docker exec -it fej sh ;;
+
+        # Misc
+        docs ) cargo doc --no-deps ;;
+        format ) cargo fmt ;;
+        l | logs ) logs ;;
+        lint ) cargo fmt -- --check ;;
+        p | push | publish ) publish ;;
+        t | test ) tests ;;
+        * ) >&2 echo "Invalid command."; exit 1 ;;
+    esac
+}
+
+main "$@"

roadmap.md

@@ -69,8 +69,8 @@ setting up a database for this version anyways.
 ## Kissanime
 
 I like watching anime from time to time, and I've always used Kissanime for
-this. However, their sites can be quite slow, and riddled with ads from time to
-time. That's why I'd like to create a high-speed wrapper that extracts all the
-needed info from their sites, removing the need for the user to ever actually
-visit their site. The API can just act as a fast search index, complete with
-indexing of the links to the videos and everything.
+this. However, their sites can be quite slow, and riddled with ads. That's why
+I'd like to create a high-speed wrapper that extracts all the needed info from
+their sites, removing the need for the user to ever actually visit their site.
+The API can just act as a fast search index, complete with indexing of the
+links to the videos and everything.

src/fej/errors.rs

@@ -10,6 +10,8 @@ pub enum FejError {
     FailedRequest,
 }
 
+// I'd love to move this over to the server binary, but right now, error E0117 is making that
+// imopssible
 impl From<FejError> for Status {
     fn from(err: FejError) -> Status {
         match err {

src/fej/lib.rs

@@ -1,11 +1,7 @@
 #![feature(proc_macro_hygiene, decl_macro)]
 
-#[macro_use]
-extern crate rocket;
-
 // Route modules
 pub mod ivago;
 
 // Helper modules
-pub mod catchers;
 pub mod errors;

src/server/main.rs

@@ -1,7 +1,13 @@
+#![feature(proc_macro_hygiene, decl_macro)]
+
 #[macro_use]
 extern crate rocket;
 
-use fej_lib::{catchers, ivago};
+#[cfg(test)]
+mod tests;
+
+mod catchers;
+mod routes;
 
 // Very temporary solution for CORS
 // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs
@@ -33,7 +39,7 @@ impl Fairing for CORS {
 fn rocket() -> rocket::Rocket {
     rocket::ignite()
         .attach(CORS)
-        .mount("/ivago", ivago::routes())
+        .mount("/ivago", routes::ivago())
         .register(catchers![catchers::not_found])
 }
 

src/server/routes/ivago.rs

@@ -1,14 +1,7 @@
-mod controller;
-
-use controller::{get_pickup_times, search_streets};
-use controller::{BasicDate, PickupTime, Street};
+use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street};
 use rocket::http::Status;
 use rocket_contrib::json::Json;
 
-pub fn routes() -> Vec<rocket::Route> {
-    routes![route_search_streets, route_get_pickup_times]
-}
-
 /// This route handles the Ivago search endpoint. It returns a list of streets,
 /// consisting of a street name & a city.
 ///

src/server/routes/mod.rs

@@ -0,0 +1,5 @@
+mod ivago;
+
+pub fn ivago() -> Vec<rocket::Route> {
+    routes![ivago::route_search_streets, ivago::route_get_pickup_times]
+}

src/server/tests.rs

@@ -3,7 +3,7 @@ use rocket::http::Status;
 use rocket::local::Client;
 
 fn rocket() -> rocket::Rocket {
-    rocket::ignite().mount("/", fej_lib::ivago::routes())
+    rocket::ignite().mount("/", super::routes::ivago())
 }
 
 /// Test 404 response