Merge branch 'develop'

.dockerignore

@@ -6,3 +6,15 @@
 # Cargo files
 !Cargo.toml
 !Cargo.lock
+
+# Entrypoint for devop container
+!docker/entrypoint_dev.sh
+!docker/entrypoint.sh
+
+# Config file
+!Rocket.toml
+
+# Database migrations
+!migrations/
+
+!docker/crontab

.env

@@ -0,0 +1,2 @@
+# This file is read by diesel
+DATABASE_URL=postgres://fej:fej@localhost:5432/fej

.hooks/pre-commit

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
 # This hook lints the code, and if we're on develop or master, also forces the tests to pass.
-make lint &> /dev/null 2>&1 || {
-    >&2 echo "Format check failed, use 'make lint' for more information.";
+./fejctl lint &> /dev/null 2>&1 || {
+    >&2 echo "Format check failed, use './fejctl lint' for more information.";
     exit 1;
 }
 

Cargo.lock

@@ -195,9 +195,9 @@ dependencies = [
 
 [[package]]
 name = "const_fn"
-version = "0.4.6"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28"
+checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec"
 
 [[package]]
 name = "cookie"
@@ -233,7 +233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3"
 dependencies = [
  "cookie 0.14.4",
- "idna 0.2.2",
+ "idna 0.2.3",
  "log 0.4.14",
  "publicsuffix",
  "serde",
@@ -321,6 +321,40 @@ dependencies = [
  "syn 0.15.44",
 ]
 
+[[package]]
+name = "diesel"
+version = "1.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "047bfc4d5c3bd2ef6ca6f981941046113524b9a9f9a7cbdfdd7ff40f58e6f542"
+dependencies = [
+ "bitflags",
+ "byteorder",
+ "diesel_derives",
+ "pq-sys",
+ "r2d2",
+]
+
+[[package]]
+name = "diesel_derives"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3"
+dependencies = [
+ "proc-macro2 1.0.26",
+ "quote 1.0.9",
+ "syn 1.0.69",
+]
+
+[[package]]
+name = "diesel_migrations"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c"
+dependencies = [
+ "migrations_internals",
+ "migrations_macros",
+]
+
 [[package]]
 name = "digest"
 version = "0.9.0"
@@ -347,10 +381,12 @@ dependencies = [
 
 [[package]]
 name = "fej"
-version = "1.0.0"
+version = "1.0.2"
 dependencies = [
  "chrono",
  "chrono-tz",
+ "diesel",
+ "diesel_migrations",
  "regex",
  "reqwest",
  "rocket",
@@ -599,9 +635,9 @@ dependencies = [
 
 [[package]]
 name = "httparse"
-version = "1.3.6"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc35c995b9d93ec174cf9a27d425c7892722101e14993cd227fdb51d70cf9589"
+checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437"
 
 [[package]]
 name = "httpdate"
@@ -678,9 +714,9 @@ dependencies = [
 
 [[package]]
 name = "idna"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21"
+checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
 dependencies = [
  "matches",
  "unicode-bidi",
@@ -717,6 +753,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "instant"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -781,6 +826,15 @@ version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41"
 
+[[package]]
+name = "lock_api"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176"
+dependencies = [
+ "scopeguard",
+]
+
 [[package]]
 name = "log"
 version = "0.3.9"
@@ -811,6 +865,27 @@ version = "2.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
 
+[[package]]
+name = "migrations_internals"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860"
+dependencies = [
+ "diesel",
+]
+
+[[package]]
+name = "migrations_macros"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c"
+dependencies = [
+ "migrations_internals",
+ "proc-macro2 1.0.26",
+ "quote 1.0.9",
+ "syn 1.0.69",
+]
+
 [[package]]
 name = "mime"
 version = "0.2.6"
@@ -922,9 +997,9 @@ dependencies = [
 
 [[package]]
 name = "notify"
-version = "4.0.15"
+version = "4.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd"
+checksum = "2599080e87c9bd051ddb11b10074f4da7b1223298df65d4c2ec5bcf309af1533"
 dependencies = [
  "bitflags",
  "filetime",
@@ -1021,6 +1096,31 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "parking_lot"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
+dependencies = [
+ "instant",
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
+dependencies = [
+ "cfg-if 1.0.0",
+ "instant",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "parse-zoneinfo"
 version = "0.3.0"
@@ -1066,18 +1166,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
 [[package]]
 name = "pin-project"
-version = "1.0.6"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6"
+checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.0.6"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5"
+checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f"
 dependencies = [
  "proc-macro2 1.0.26",
  "quote 1.0.9",
@@ -1119,6 +1219,15 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
 
+[[package]]
+name = "pq-sys"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda"
+dependencies = [
+ "vcpkg",
+]
+
 [[package]]
 name = "proc-macro-hack"
 version = "0.5.19"
@@ -1149,7 +1258,7 @@ version = "1.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f"
 dependencies = [
- "idna 0.2.2",
+ "idna 0.2.3",
  "url 2.2.1",
 ]
 
@@ -1171,6 +1280,17 @@ dependencies = [
  "proc-macro2 1.0.26",
 ]
 
+[[package]]
+name = "r2d2"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f"
+dependencies = [
+ "log 0.4.14",
+ "parking_lot",
+ "scheduled-thread-pool",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.3"
@@ -1213,9 +1333,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9"
+checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041"
 dependencies = [
  "bitflags",
 ]
@@ -1326,13 +1446,28 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7954a707f9ca18aa74ca8c1f5d1f900f52a4dceb68e96e3112143f759cfd20e"
 dependencies = [
+ "diesel",
  "log 0.4.14",
  "notify",
+ "r2d2",
  "rocket",
+ "rocket_contrib_codegen",
  "serde",
  "serde_json",
 ]
 
+[[package]]
+name = "rocket_contrib_codegen"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30deb6dec53b91fac3538a2a3935cf13e0f462745f9f33bf27bedffbe7265b5d"
+dependencies = [
+ "devise",
+ "quote 0.6.13",
+ "version_check 0.9.3",
+ "yansi",
+]
+
 [[package]]
 name = "rocket_http"
 version = "0.4.7"
@@ -1390,6 +1525,21 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "scheduled-thread-pool"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7"
+dependencies = [
+ "parking_lot",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
 [[package]]
 name = "security-framework"
 version = "2.2.0"
@@ -1847,16 +1997,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b"
 dependencies = [
  "form_urlencoded",
- "idna 0.2.2",
+ "idna 0.2.3",
  "matches",
  "percent-encoding 2.1.0",
 ]
 
 [[package]]
 name = "vcpkg"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"
+checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d"
 
 [[package]]
 name = "version_check"

Cargo.toml

@@ -1,23 +1,24 @@
 [package]
 name = "fej"
-version = "1.0.0"
+version = "1.0.2"
 authors = ["Jef Roosens <roosensjef@gmail.com>"]
 edition = "2018"
 
 [lib]
-name = "fej_lib"
-src = "src/lib.rs"
+name = "fej"
+path = "src/fej/lib.rs"
 test = true
 bench = true
 doc = true
 doctest = true
 
 [[bin]]
-name = "fej"
-src = "src/main.rs"
-test = false
-bench = false
-doc = false
+name = "server"
+path = "src/server/main.rs"
+test = true
+bench = true
+doc = true
+doctest = true
  
 [dependencies]
 rocket = "0.4.7"
@@ -25,13 +26,11 @@ serde = "1.0.124"
 chrono = "0.4.19"
 chrono-tz = "0.5.3"
 regex = "1.4.5"
-
-[dependencies.reqwest]
-version = "0.11.2"
-default-features = true
-features = ["blocking", "json", "cookies"]
+reqwest = { version = "0.11.2", features = ["blocking", "json", "cookies"] }
+diesel = { version = "1.4.6", features = ["postgres"] }
+diesel_migrations = "1.4.0"
 
 [dependencies.rocket_contrib]
 version = "0.4.7"
 default-features = false
-features = ["json"]
+features = ["json", "diesel_postgres_pool"]

Makefile

@@ -1,48 +0,0 @@
-all: debug
-.PHONY: all
-
-# Builds
-debug:
-	@ ./build -m dev -a run build
-.PHONY: debug
-
-release:
-	@ ./build -m rel
-.PHONY: release
-
-push:
-	@ ./build -m rel -a push
-.PHONY: push
-
-# Run
-run:
-	@ ./build -m dev -a run
-.PHONY: run
-
-stop:
-	@ docker stop -t 2 fej
-.PHONY: stop
-
-logs:
-	@ docker logs -f fej
-.PHONY: logs
-
-
-# Testing
-test:
-	@ ./build -m dev -a run -l -- test --no-fail-fast
-.PHONY: test
-
-format:
-	@ cargo fmt
-.PHONY: format
-
-lint:
-	@ cargo fmt -- --check
-.PHONY: lint
-
-
-# Documentation
-docs:
-	@ cargo doc --no-deps
-.PHONY: docs

README.md

@@ -1,26 +1,52 @@
 # Fej
 
-Fej is an API written in Rust. I started this project to learn the language,
-and really just have some fun.
+Fej is a RESTful API that does lots of different things. It started as an
+experiment to learn Rust, but has grown into a full-on passion project.
 
 ## Project Structure
 
-The folder structure follows the structure of the URLs, e.g. the route for
-`/hello/world` is found in the module `src/hello`.
+The `src` folder contains subfolders for the various binaries and the main
+library, called `fej`. The biggest binary is called `server`, which is the
+binary that actually runs the Rocket.rs web server. All the others are utility
+programs, mostly consisting of scrapers for various services.
 
-Each module contains the following base files:
-
-* `mod.rs`: defines the modules' content, and contains the route definitions.
-  The route functions themselves only contain the functionality needed to
-  represent the data, not acquire it.
-* `controller.rs`: this file contains the actual logic of each route. If the
-  logic becomes too complicated to be contained inside a single file,
-  `controller.rs` becomes its own module folder named `controller`.
-* `tests.rs`: this contains tests for the specific module. This can also be a
-  module directory if need be.
-
-Every module has a `routes` function that returns its route macros.
+Version 1.1 also introduces the use of a database, namely
+[PostgreSQL 13](https://www.postgresql.org/).
 
 ## Roadmap
 
 See [roadmap.md](roadmap.md).
+
+## Development
+
+To make development more consistent (and keep my computer a bit cleaner) I've
+decided to run the entire toolchain using Docker, with Alpine Linux base
+images. This also allows me to make really small final images. Technically,
+Docker is the only dependency you need to contribute to this project
+(not accounting for language servers etc.).
+
+A [Bash script](fejctl) is provided to speed up development on the various
+binaries. It aids in starting up the containers, choosing which binary to run
+etc. A quick rundown of its most important features:
+
+```bash
+# By default, it compiles the 'server' binary (but doesn't run it)
+./fejctl
+
+# The first argument is the action you wish to perform, the second on which
+# binary (not all commands need a binary as input, so they just ignore it)
+# For example, this will run the binary called server
+./fejctl r server
+
+# This runs the tests (all commands also have their full name if you want)
+./fejctl t
+./fejctl test
+
+# These attach to the two containers
+./fejctl sh   # Opens a new root shell inside the 'fej' container
+./fejctl psql # Open a new psql shell in the fej database
+```
+
+## Docker images
+
+The images are available [here](https://hub.docker.com/r/chewingbever/fej).

Rocket.toml

@@ -0,0 +1,14 @@
+[development]
+address = "0.0.0.0"
+port = 8000
+keep_alive = 5
+read_timeout = 5
+write_timeout = 5
+log = "normal"
+limits = { forms = 32768 }
+
+[development.databases]
+postgres_fej = { url = "postgres://fej:fej@fej_db:5432/fej" }
+
+[production.databases]
+postgres_fej = { url = "postgres://fej:fej@db:5432/fej" }

build

@@ -1,91 +0,0 @@
-#!/usr/bin/env bash
-
-image="chewingbever/fej"
-# Should be either dev or rel
-mode="dev"
-action=""
-attach="--detach"
-
-while getopts ":i:m:a:l" c; do
-    case $c in
-        i ) image="$OPTARG" ;;
-        m ) mode="$OPTARG" ;;
-        a ) action="$OPTARG" ;;
-        l ) attach="" ;;
-        ? ) exit 1 ;;
-    esac
-done
-shift $((OPTIND-1))
-
-# Extract current version from Cargo.toml & get current branch
-patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1`
-major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'`
-minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'`
-branch=`git rev-parse --abbrev-ref HEAD`
-
-if [[ "$branch" = "master" ]]; then
-    tags=("$patch_version" "$minor_version" "$major_version" "latest")
-
-elif [[ "$branch" = "develop" ]]; then
-    tags=("$patch_version-dev" "$minor_version-dev" "$major_version-dev" "dev")
-
-else
-    tags=("$branch")
-
-fi
-
-# First, we build the builder
-DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" .
-
-# Run the actual build command
-if [ "$mode" = "rel" ]; then
-    DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel .
-
-elif [[ "$mode" = "dev" ]]; then
-    DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev .
-
-else
-    >&2 echo "Invalid mode."
-    exit 1
-
-fi
-
-if [[ "$action" = push ]]; then
-    [[ "$branch" =~ ^develop|master$ ]] || {
-        >&2 echo "You can only push from develop or master."
-        exit 2
-    }
-
-    [[ "$mode" = "rel" ]] || {
-        >&2 echo "You can only push release builds."
-        exit 3
-    }
-
-    for tag in "${tags[@]}"; do
-        # Create the tag
-        docker tag "$image:$tags" "$image:$tag"
-
-        # Push the tag
-        docker push "$image:$tag"
-
-        # Remove the tag again, if it's not the main tag
-        [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag"
-    done
-
- elif [[ "$action" = run ]]; then
-    if [[ "$mode" = "dev" ]]; then
-        # Create caching volumes if needed (they need to be named)
-        docker volume create fej_build-cache
-        docker volume create fej_registry-cache
-
-        flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry"
-    fi
-
-    docker run $attach $flags \
-        --rm \
-        --interactive \
-        --tty \
-        --publish 8000:8000 \
-        --name fej \
-        "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@"
-fi

diesel.toml

@@ -0,0 +1,5 @@
+# For documentation on how to configure this file,
+# see diesel.rs/guides/configuring-diesel-cli
+
+[print_schema]
+file = "src/schema.rs"

docker/Dockerfile.builder

@@ -1,18 +1,36 @@
+# vim: filetype=dockerfile
 # Our entire toolchain runs in alpine
 FROM alpine:latest AS builder
 
-ENV PATH "$PATH:/root/.cargo/bin"
+ENV PATH "$PATH:/app/.cargo/bin"
 # Needed for proper compiling of openssl-dev
 ENV RUSTFLAGS "-C target-feature=-crt-static"
-# Otherwise, the debug build can't be used from the container
-ENV ROCKET_ADDRESS "0.0.0.0"
 
-WORKDIR /usr/src/app
+# Add the build user
+# Install dependencies
+RUN addgroup -S builder && \
+    adduser -S builder -G builder -h /app && \
+    apk update && \
+    apk add --no-cache \
+        curl \
+        gcc \
+        libgcc \
+        musl-dev \
+        openssl-dev \
+        postgresql-dev
 
-# Install build dependencies, rustup & rust's nightly build & toolchain
-RUN apk update && apk add --no-cache openssl-dev build-base curl && \
-    { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; }
+# Switch to the non-root user
+USER builder
+
+WORKDIR /app
+
+# Install rustup in the new user's home
+# Create mountpoints for volumes with correct permissions
+RUN { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \
+    rustup target add x86_64-unknown-linux-musl --toolchain nightly && \
+    mkdir -p .cargo/registry target
 
 # Copy source code over to builder
-COPY Cargo.toml Cargo.lock ./
-COPY src/ ./src/
+COPY --chown=builder:builder Cargo.toml Cargo.lock ./
+COPY --chown=builder:builder src/ ./src/
+COPY --chown=builder:builder migrations/ ./migrations/

docker/Dockerfile.dev

@@ -1,6 +1,9 @@
+# vim: filetype=dockerfile
 FROM chewingbever/fej-builder:latest
 
 ENV RUST_BACKTRACE 1
 
-ENTRYPOINT ["cargo"]
-CMD ["run"]
+COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh
+COPY --chown=builder:builder ./Rocket.toml /app/Rocket.toml
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.rel

@@ -1,3 +1,4 @@
+# vim: filetype=dockerfile
 FROM chewingbever/fej-builder:latest AS builder
 
 # And then finally, build the project
@@ -5,19 +6,44 @@ FROM chewingbever/fej-builder:latest AS builder
 # https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172
 # TODO add what these flags do & why they work
 # NOTE: cargo install auto-appends bin to the path
-RUN --mount=type=cache,target=/usr/src/app/target \
-    --mount=type=cache,target=/root/.cargo/registry \
-    cargo install --path . --bin fej --root /usr/local
+
+# RUN --mount=type=cache,mode=0777,target=/app/target \
+#     --mount=type=cache,mode=0777,target=/app/.cargo/registry \
+
+# Buildkit cache mounts really don't like it when you're not root,
+# so I guess we're building release without a cache for now
+RUN cargo install \
+    --path . \
+    --root /app/output \
+    --target x86_64-unknown-linux-musl
 
 
 # Now, we create the actual image
 FROM alpine:latest
+COPY ./docker/crontab /var/spool/cron/crontabs/fej
 
 # Install some dynamic libraries needed for everything to work
-RUN apk update && apk add --no-cache openssl libgcc curl
+# Create -non-root user
+# Change permissions for crontab file
+RUN apk update && \
+    apk add --no-cache \
+        curl \
+        libgcc \
+        libpq \
+        openssl && \
+    addgroup -S fej && \
+    adduser -S fej -G fej -h /app
+
+# Switch to non-root user
+USER fej:fej
 
 # Copy binary over to final image
-COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej
+COPY --from=builder --chown=fej:fej /app/output/bin /app/bin
+
+# Embed config file inside container
+# The workdir is changed so that the config file is read properly
+WORKDIR /app
+COPY --chown=fej:fej Rocket.toml /app/Rocket.toml
 
 HEALTHCHECK \
     --interval=10s \
@@ -26,4 +52,4 @@ HEALTHCHECK \
     --retries=3 \
     CMD curl -q localhost:8000
 
-CMD ["/usr/local/bin/fej"]
+ENTRYPOINT ["/app/bin/server"]

docker/crontab

@@ -0,0 +1 @@
+# This'll be filled up later

docker/docker-compose.dev.yml

@@ -0,0 +1,31 @@
+version: '2.4'
+services:
+    app:
+        build:
+            # Make sure the build context is one directory up
+            context: '..'
+            dockerfile: './docker/Dockerfile.dev'
+        image: 'chewingbever/fej:dev'
+        restart: 'no'
+
+        container_name: 'fej_app'
+        volumes:
+            - 'build-cache:/app/target'
+            - 'registry-cache:/app/.cargo/registry'
+        ports:
+            - '8000:8000'
+
+        command: "${CMD}"
+
+    db:
+        container_name: 'fej_db'
+        restart: 'no'
+
+        # the devop environment exposes the database so we can use the Diesel cli
+        ports:
+            - '5432:5432'
+
+volumes:
+    build-cache:
+    registry-cache:
+

docker/docker-compose.override.yml

@@ -0,0 +1,14 @@
+version: '2.4'
+
+services:
+    cron:
+        image: 'chewingbever/fej:latest'
+        restart: 'always'
+
+        entrypoint: 'crond -f'
+        user: 'root'
+        healthcheck:
+            disable: true
+
+        environment:
+            - 'DATABASE_URL=postgres://fej:fej@db:5432/fej'

docker/docker-compose.yml

@@ -0,0 +1,34 @@
+version: '2.4'
+
+services:
+    app:
+        build:
+            context: '..'
+            dockerfile: 'docker/Dockerfile.rel'
+
+        image: 'chewingbever/fej:latest'
+        restart: 'always'
+
+        environment:
+            - 'DATABASE_URL=postgres://fej:fej@db:5432/fej'
+
+    db:
+        image: 'postgres:13-alpine'
+        restart: 'always'
+
+        environment:
+            - 'POSTGRES_DB=fej'
+            - 'POSTGRES_USER=fej'
+            - 'POSTGRES_PASSWORD=fej'
+        healthcheck:
+            test: 'pg_isready -U fej'
+            interval: '30s'
+            timeout: '5s'
+            retries: 3
+            start_period: '0s'
+
+        volumes:
+            - 'db-data:/var/lib/postgresql/data'
+
+volumes:
+    db-data:

docker/entrypoint_dev.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+
+# All this file does is inject the target
+cargo "$@" --target x86_64-unknown-linux-musl

fejctl

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+image='chewingbever/fej'
+
+# Small wrapper around the docker-compose command
+#
+# Flags:
+# -b: build the builder
+# -r: use the release image instead
+function dc() {
+    local OPTIND c build_builder release
+
+    while getopts ":br" c; do
+        case $c in
+            b ) build_builder=1 ;;
+            r ) release=1 ;;
+        esac
+    done
+    shift $((OPTIND-1))
+
+    if [[ "$build_builder" -eq 1 ]]; then
+        # We always rebuild the builder before we run any compose command
+        DOCKER_BUILDKIT=1 docker build \
+            -f docker/Dockerfile.builder \
+            -t "$image-builder:latest" . || {
+                >&2 echo "Failed to build builder.";
+                exit 1;
+            }
+    fi
+
+    if [[ "$release" -eq 1 ]]; then
+        DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \
+            --file docker/docker-compose.yml \
+            --file docker/docker-compose.override.yml \
+            --project-name fej \
+            "$@"
+
+    else
+        DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \
+            --file docker/docker-compose.yml \
+            --file docker/docker-compose.dev.yml \
+            --project-name fej-dev \
+            "$@"
+    fi
+}
+
+# Execute the debug image (must be built first)
+#
+# $@: the arguments to pass to the image (passed as arguments to cargo)
+function dcr() {
+        CMD="$@" dc -b -- up \
+            --build \
+            --detach
+}
+
+# Tags & pushes the release version to Docker Hub
+function publish() {
+    local branch=`git rev-parse --abbrev-ref HEAD`
+
+    if [[ "$branch" != master ]]; then
+        >&2 echo "You can only publish from master."
+        exit 2
+    fi
+
+    # Build the release images
+    dc -br build
+
+    local patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1`
+    local major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'`
+    local minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'`
+    local tags=("latest" "$patch_version" "$minor_version" "$major_version")
+
+    for tag in "${tags[@]}"; do
+        # Create the tag
+        docker tag "$image:$tags" "$image:$tag"
+
+        # Push the tag
+        docker push "$image:$tag"
+
+        # Remove the tag again, if it's not the main tag
+        [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag"
+    done
+
+}
+
+# Entrypoint to the script
+#
+# $1: action to perform, defaults to 'build'
+# $2: binary to use, defaults to 'server'
+function main() {
+    # Default values
+    local cmd="${1:-build}"
+    local bin="${2:-server}"
+
+    case $cmd in
+        # Building
+        b | build ) dcr build --bin "$bin" && dc -- logs -f app ;;
+        br | build-release ) dc -br build ;;
+
+        # Running
+        r | run ) dcr run --bin "$bin" && dc -- logs -f app ;;
+        rr | run-release ) dc -br -- up --build --detach && dc -r -- logs -f app ;;
+        s | stop ) dc down ;;
+        sr | stop-release ) dc -r stop ;;
+
+        # Ease of life
+        psql ) dc -- exec db psql -U fej -d fej ;;
+        sh ) dc -- exec app sh ;;
+
+        # Misc
+        docs ) cargo doc --no-deps ;;
+        format ) cargo fmt ;;
+        l | logs ) dc -- logs -f app ;;
+        lint ) cargo fmt -- --check ;;
+        p | push | publish ) publish ;;
+        t | test ) dcr -- test --no-fail-fast && dc -- logs -f app ;;
+        * ) >&2 echo "Invalid command."; exit 1 ;;
+    esac
+}
+
+main "$@"

migrations/00000000000000_diesel_initial_setup/down.sql

@@ -0,0 +1,6 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
+DROP FUNCTION IF EXISTS diesel_set_updated_at();

migrations/00000000000000_diesel_initial_setup/up.sql

@@ -0,0 +1,36 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+
+
+
+-- Sets up a trigger for the given table to automatically set a column called
+-- `updated_at` whenever the row is modified (unless `updated_at` was included
+-- in the modified columns)
+--
+-- # Example
+--
+-- ```sql
+-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
+--
+-- SELECT diesel_manage_updated_at('users');
+-- ```
+CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
+BEGIN
+    EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
+                    FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
+BEGIN
+    IF (
+        NEW IS DISTINCT FROM OLD AND
+        NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
+    ) THEN
+        NEW.updated_at := current_timestamp;
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;

migrations/2021-04-15-155511_ivago_search/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP SCHEMA ivago CASCADE;

migrations/2021-04-15-155511_ivago_search/up.sql

@@ -0,0 +1,9 @@
+-- Your SQL goes here
+CREATE SCHEMA ivago;
+
+CREATE TABLE ivago.streets (
+    name TEXT NOT NULL,
+    city TEXT NOT NULL,
+
+    PRIMARY KEY (name, city)
+);

roadmap.md

@@ -4,7 +4,7 @@ This file contains a listing of my current ideas/plans for this project. This
 could change at any minute, but it serves as a quick overview of my plans for
 the project.
 
-## 1.0: Ivago
+## Ivago
 
 ### Summary
 
@@ -30,7 +30,7 @@ still collecting the same data. Furthermore, having the data available in
 computer-readable format opens up ways to write a custom frontend for the API
 (which I will be doing sometime in the near future).
 
-## 2.0: The Curse of the Forge
+## The Curse of the Forge
 
 ### Summary
 
@@ -65,3 +65,12 @@ touch their websites or services (besides downloading the mods of course).
 
 As a bonus, I want to cache the Ivago API's calendars, considering I'll be
 setting up a database for this version anyways.
+
+## Kissanime
+
+I like watching anime from time to time, and I've always used Kissanime for
+this. However, their sites can be quite slow, and riddled with ads. That's why
+I'd like to create a high-speed wrapper that extracts all the needed info from
+their sites, removing the need for the user to ever actually visit their site.
+The API can just act as a fast search index, complete with indexing of the
+links to the videos and everything.

src/fej/errors.rs

@@ -10,6 +10,8 @@ pub enum FejError {
     FailedRequest,
 }
 
+// I'd love to move this over to the server binary, but right now, error E0117 is making that
+// imopssible
 impl From<FejError> for Status {
     fn from(err: FejError) -> Status {
         match err {

src/fej/lib.rs

@@ -1,11 +1,7 @@
 #![feature(proc_macro_hygiene, decl_macro)]
 
-#[macro_use]
-extern crate rocket;
-
 // Route modules
 pub mod ivago;
 
 // Helper modules
-pub mod catchers;
 pub mod errors;

src/main.rs

@@ -1,14 +0,0 @@
-#[macro_use]
-extern crate rocket;
-
-use fej_lib::{catchers, ivago};
-
-fn rocket() -> rocket::Rocket {
-    rocket::ignite()
-        .mount("/ivago", ivago::routes())
-        .register(catchers![catchers::not_found])
-}
-
-fn main() {
-    rocket().launch();
-}

src/server/main.rs

@@ -0,0 +1,73 @@
+#![feature(proc_macro_hygiene, decl_macro)]
+
+#[macro_use]
+extern crate rocket;
+#[macro_use]
+extern crate rocket_contrib;
+#[macro_use]
+extern crate diesel_migrations;
+
+#[cfg(test)]
+mod tests;
+
+mod catchers;
+mod routes;
+
+// Very temporary solution for CORS
+// https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs
+use rocket::fairing::AdHoc;
+use rocket::fairing::{Fairing, Info, Kind};
+use rocket::http::Header;
+use rocket::{Request, Response, Rocket};
+use rocket_contrib::databases::diesel;
+
+pub struct CORS;
+
+impl Fairing for CORS {
+    fn info(&self) -> Info {
+        Info {
+            name: "Add CORS headers to responses",
+            kind: Kind::Response,
+        }
+    }
+
+    fn on_response(&self, _: &Request, response: &mut Response) {
+        response.set_header(Header::new("Access-Control-Allow-Origin", "*"));
+        response.set_header(Header::new(
+            "Access-Control-Allow-Methods",
+            "POST, GET, PATCH, OPTIONS",
+        ));
+        response.set_header(Header::new("Access-Control-Allow-Headers", "*"));
+        response.set_header(Header::new("Access-Control-Allow-Credentials", "true"));
+    }
+}
+
+// Macro from diesel_migrations that sets up migrations
+embed_migrations!();
+
+// This defines a connection to the database
+#[database("postgres_fej")]
+struct FejDbConn(diesel::PgConnection);
+
+// I'd like to thank Stackoverflow for helping me with this
+// https://stackoverflow.com/questions/61047355/how-to-run-diesel-migration-with-rocket-in-production
+fn run_db_migrations(rocket: Rocket) -> Result<Rocket, Rocket> {
+    let conn = FejDbConn::get_one(&rocket).expect("database connection");
+    match embedded_migrations::run(&*conn) {
+        Ok(()) => Ok(rocket),
+        Err(e) => Err(rocket),
+    }
+}
+
+fn rocket() -> rocket::Rocket {
+    rocket::ignite()
+        .attach(CORS)
+        .attach(FejDbConn::fairing())
+        .attach(AdHoc::on_attach("Database Migrations", run_db_migrations))
+        .mount("/ivago", routes::ivago())
+        .register(catchers![catchers::not_found])
+}
+
+fn main() {
+    rocket().launch();
+}

src/server/routes/ivago.rs

@@ -1,14 +1,7 @@
-mod controller;
-
-use controller::{get_pickup_times, search_streets};
-use controller::{BasicDate, PickupTime, Street};
+use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street};
 use rocket::http::Status;
 use rocket_contrib::json::Json;
 
-pub fn routes() -> Vec<rocket::Route> {
-    routes![route_search_streets, route_get_pickup_times]
-}
-
 /// This route handles the Ivago search endpoint. It returns a list of streets,
 /// consisting of a street name & a city.
 ///

src/server/routes/mod.rs

@@ -0,0 +1,5 @@
+mod ivago;
+
+pub fn ivago() -> Vec<rocket::Route> {
+    routes![ivago::route_search_streets, ivago::route_get_pickup_times]
+}

src/server/tests.rs

@@ -3,7 +3,7 @@ use rocket::http::Status;
 use rocket::local::Client;
 
 fn rocket() -> rocket::Rocket {
-    rocket::ignite().mount("/", fej_lib::ivago::routes())
+    rocket::ignite().mount("/", super::routes::ivago())
 }
 
 /// Test 404 response