diff --git a/.dockerignore b/.dockerignore index 614c7dd..a96bd42 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,15 @@ # Cargo files !Cargo.toml !Cargo.lock + +# Entrypoint for devop container +!docker/entrypoint_dev.sh +!docker/entrypoint.sh + +# Config file +!Rocket.toml + +# Database migrations +!migrations/ + +!docker/crontab diff --git a/.env b/.env new file mode 100644 index 0000000..6e2ca99 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +# This file is read by diesel +DATABASE_URL=postgres://fej:fej@localhost:5432/fej diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 709fb95..fe55771 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -1,8 +1,8 @@ #!/usr/bin/env bash # This hook lints the code, and if we're on develop or master, also forces the tests to pass. -make lint &> /dev/null 2>&1 || { - >&2 echo "Format check failed, use 'make lint' for more information."; +./fejctl lint &> /dev/null 2>&1 || { + >&2 echo "Format check failed, use './fejctl lint' for more information."; exit 1; } diff --git a/Cargo.lock b/Cargo.lock index 8ab2a9c..acb8e84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28" +checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec" [[package]] name = "cookie" @@ -233,7 +233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3" dependencies = [ "cookie 0.14.4", - "idna 0.2.2", + "idna 0.2.3", "log 0.4.14", "publicsuffix", "serde", @@ -321,6 +321,40 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "diesel" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "047bfc4d5c3bd2ef6ca6f981941046113524b9a9f9a7cbdfdd7ff40f58e6f542" +dependencies = [ + "bitflags", + "byteorder", + "diesel_derives", + "pq-sys", + "r2d2", +] + +[[package]] +name = "diesel_derives" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" +dependencies = [ + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", +] + +[[package]] +name = "diesel_migrations" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" +dependencies = [ + "migrations_internals", + "migrations_macros", +] + [[package]] name = "digest" version = "0.9.0" @@ -347,10 +381,12 @@ dependencies = [ [[package]] name = "fej" -version = "1.0.1" +version = "1.0.2" dependencies = [ "chrono", "chrono-tz", + "diesel", + "diesel_migrations", "regex", "reqwest", "rocket", @@ -599,9 +635,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.6" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc35c995b9d93ec174cf9a27d425c7892722101e14993cd227fdb51d70cf9589" +checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" [[package]] name = "httpdate" @@ -678,9 +714,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", @@ -717,6 +753,15 @@ dependencies = [ "libc", ] +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -781,6 +826,15 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +[[package]] +name = "lock_api" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.3.9" @@ -811,6 +865,27 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "migrations_internals" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" +dependencies = [ + "diesel", +] + +[[package]] +name = "migrations_macros" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" +dependencies = [ + "migrations_internals", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", +] + [[package]] name = "mime" version = "0.2.6" @@ -922,9 +997,9 @@ dependencies = [ [[package]] name = "notify" -version = "4.0.15" +version = "4.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd" +checksum = "2599080e87c9bd051ddb11b10074f4da7b1223298df65d4c2ec5bcf309af1533" dependencies = [ "bitflags", "filetime", @@ -1021,6 +1096,31 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + [[package]] name = "parse-zoneinfo" version = "0.3.0" @@ -1066,18 +1166,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", @@ -1119,6 +1219,15 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "pq-sys" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda" +dependencies = [ + "vcpkg", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -1149,7 +1258,7 @@ version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f" dependencies = [ - "idna 0.2.2", + "idna 0.2.3", "url 2.2.1", ] @@ -1171,6 +1280,17 @@ dependencies = [ "proc-macro2 1.0.26", ] +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log 0.4.14", + "parking_lot", + "scheduled-thread-pool", +] + [[package]] name = "rand" version = "0.8.3" @@ -1213,9 +1333,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041" dependencies = [ "bitflags", ] @@ -1326,13 +1446,28 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7954a707f9ca18aa74ca8c1f5d1f900f52a4dceb68e96e3112143f759cfd20e" dependencies = [ + "diesel", "log 0.4.14", "notify", + "r2d2", "rocket", + "rocket_contrib_codegen", "serde", "serde_json", ] +[[package]] +name = "rocket_contrib_codegen" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30deb6dec53b91fac3538a2a3935cf13e0f462745f9f33bf27bedffbe7265b5d" +dependencies = [ + "devise", + "quote 0.6.13", + "version_check 0.9.3", + "yansi", +] + [[package]] name = "rocket_http" version = "0.4.7" @@ -1390,6 +1525,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "security-framework" version = "2.2.0" @@ -1847,16 +1997,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" dependencies = [ "form_urlencoded", - "idna 0.2.2", + "idna 0.2.3", "matches", "percent-encoding 2.1.0", ] [[package]] name = "vcpkg" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" +checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" [[package]] name = "version_check" diff --git a/Cargo.toml b/Cargo.toml index 83623fa..4d1dd7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,23 +1,24 @@ [package] name = "fej" -version = "1.0.1" +version = "1.0.2" authors = ["Jef Roosens "] edition = "2018" [lib] -name = "fej_lib" -src = "src/lib.rs" +name = "fej" +path = "src/fej/lib.rs" test = true bench = true doc = true doctest = true [[bin]] -name = "fej" -src = "src/main.rs" -test = false -bench = false -doc = false +name = "server" +path = "src/server/main.rs" +test = true +bench = true +doc = true +doctest = true [dependencies] rocket = "0.4.7" @@ -25,13 +26,11 @@ serde = "1.0.124" chrono = "0.4.19" chrono-tz = "0.5.3" regex = "1.4.5" - -[dependencies.reqwest] -version = "0.11.2" -default-features = true -features = ["blocking", "json", "cookies"] +reqwest = { version = "0.11.2", features = ["blocking", "json", "cookies"] } +diesel = { version = "1.4.6", features = ["postgres"] } +diesel_migrations = "1.4.0" [dependencies.rocket_contrib] version = "0.4.7" default-features = false -features = ["json"] +features = ["json", "diesel_postgres_pool"] diff --git a/Makefile b/Makefile deleted file mode 100644 index 6ad1f50..0000000 --- a/Makefile +++ /dev/null @@ -1,66 +0,0 @@ -all: debug -.PHONY: all - -# Builds the debug release inside the Alpine container. For build caching, two -# volumes are used named `fej_build-cache` and `fej_registry-cache`. These -# images are automatically created for you if they don't exist. If you -# encounter any strange build errors, you can try removing this volumes to -# start a completely fresh build. -debug: - @ ./build -m dev -a run build -.PHONY: debug - -# Builds the release version. In contrary to the debug version, this build -# doesn't use volumes for caching, as this would require the build to happen -# during runtime instead of during the building of the image. Instead, it uses -# the new `--mount` feature from Buildkit. This does mean that only very recent -# Docker engines can build the release version (in my case, at the time of -# writing this, 20.10.5). -release: - @ ./build -m rel -.PHONY: release - -# This builds the release version, and pushes all relevant tags to my Docker -# Hub repository, namely chewingbever/fej -push: - @ ./build -m rel -a push -.PHONY: push - -# This builds the debug release, and runs it detached. The reason we detach the -# container is because Rocket has a tendency to ignore ctlr-c when inside a -# container, which gets annoying really fast. -run: - @ ./build -m dev -a run -.PHONY: run - -# As a workaround, we just have a stop command that stops the container. -stop: - @ docker stop -t 2 fej -.PHONY: stop - -# This attaches to the running container, essentially giving the same result as -# just running `cargo run` locally. -logs: - @ docker logs -f fej -.PHONY: logs - -# Builds the debug version, and runs the tests (but doesn't detach). -test: - @ ./build -m dev -a run -l -- test --no-fail-fast -.PHONY: test - -# Runs the cargo code formatter on your code. -format: - @ cargo fmt -.PHONY: format - -# Lints your code. This also gets run in the pre-commit hook, basically -# preventing you from committing badly-formatted code. -lint: - @ cargo fmt -- --check -.PHONY: lint - -# This builds the documentation for the project, excluding the documentation. -docs: - @ cargo doc --no-deps -.PHONY: docs diff --git a/README.md b/README.md index 55d11e3..e2573af 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,17 @@ # Fej -Fej is an API written in Rust. I started this project to learn the language, -and really just have some fun. +Fej is a RESTful API that does lots of different things. It started as an +experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The folder structure follows the structure of the URLs, e.g. the route for -`/hello/world` is found in the module `src/hello`. +The `src` folder contains subfolders for the various binaries and the main +library, called `fej`. The biggest binary is called `server`, which is the +binary that actually runs the Rocket.rs web server. All the others are utility +programs, mostly consisting of scrapers for various services. -Each module contains the following base files: - -* `mod.rs`: defines the modules' content, and contains the route definitions. - The route functions themselves only contain the functionality needed to - represent the data, not acquire it. -* `controller.rs`: this file contains the actual logic of each route. If the - logic becomes too complicated to be contained inside a single file, - `controller.rs` becomes its own module folder named `controller`. -* `tests.rs`: this contains tests for the specific module. This can also be a - module directory if need be. - -Every module has a `routes` function that returns its route macros. +Version 1.1 also introduces the use of a database, namely +[PostgreSQL 13](https://www.postgresql.org/). ## Roadmap @@ -27,19 +19,33 @@ See [roadmap.md](roadmap.md). ## Development -The entire toolchain runs on Alpine inside Docker. This makes building easier, -and (hopefully) eliminates any system-specific bugs. +To make development more consistent (and keep my computer a bit cleaner) I've +decided to run the entire toolchain using Docker, with Alpine Linux base +images. This also allows me to make really small final images. Technically, +Docker is the only dependency you need to contribute to this project +(not accounting for language servers etc.). -A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as -all the commands are documented for your understanding ;) +A [Bash script](fejctl) is provided to speed up development on the various +binaries. It aids in starting up the containers, choosing which binary to run +etc. A quick rundown of its most important features: -There's also the `build` script. This script does all the "heavy" lifting. It -chooses which Dockerfile to build according to the given arguments, and -generates tags for the images (useful when pushing releases). The Makefile is -really just a wrapper around this build script, allowing you to write -`make test` instead of `./build -m dev -a run test`. +```bash +# By default, it compiles the 'server' binary (but doesn't run it) +./fejctl -tl;dr run `make run` to run your build, and `make test` to run the tests. +# The first argument is the action you wish to perform, the second on which +# binary (not all commands need a binary as input, so they just ignore it) +# For example, this will run the binary called server +./fejctl r server + +# This runs the tests (all commands also have their full name if you want) +./fejctl t +./fejctl test + +# These attach to the two containers +./fejctl sh # Opens a new root shell inside the 'fej' container +./fejctl psql # Open a new psql shell in the fej database +``` ## Docker images diff --git a/Rocket.toml b/Rocket.toml new file mode 100644 index 0000000..56291e7 --- /dev/null +++ b/Rocket.toml @@ -0,0 +1,14 @@ +[development] +address = "0.0.0.0" +port = 8000 +keep_alive = 5 +read_timeout = 5 +write_timeout = 5 +log = "normal" +limits = { forms = 32768 } + +[development.databases] +postgres_fej = { url = "postgres://fej:fej@fej_db:5432/fej" } + +[production.databases] +postgres_fej = { url = "postgres://fej:fej@db:5432/fej" } diff --git a/build b/build deleted file mode 100755 index d7d86a3..0000000 --- a/build +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -image="chewingbever/fej" -# Should be either dev or rel -mode="dev" -action="" -attach="--detach" - -while getopts ":i:m:a:l" c; do - case $c in - i ) image="$OPTARG" ;; - m ) mode="$OPTARG" ;; - a ) action="$OPTARG" ;; - l ) attach="" ;; - ? ) exit 1 ;; - esac -done -shift $((OPTIND-1)) - -# Extract current version from Cargo.toml & get current branch -patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` -major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` -minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` -branch=`git rev-parse --abbrev-ref HEAD` - -if [[ "$branch" = "master" ]]; then - tags=("$patch_version" "$minor_version" "$major_version" "latest") - -elif [[ "$branch" = "develop" ]]; then - tags=("$patch_version-dev" "$minor_version-dev" "$major_version-dev" "dev") - -else - tags=("$branch") - -fi - -# First, we build the builder -DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" . - -# Run the actual build command -if [ "$mode" = "rel" ]; then - DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel . - -elif [[ "$mode" = "dev" ]]; then - DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev . - -else - >&2 echo "Invalid mode." - exit 1 - -fi - -if [[ "$action" = push ]]; then - [[ "$branch" =~ ^develop|master$ ]] || { - >&2 echo "You can only push from develop or master." - exit 2 - } - - [[ "$mode" = "rel" ]] || { - >&2 echo "You can only push release builds." - exit 3 - } - - for tag in "${tags[@]}"; do - # Create the tag - docker tag "$image:$tags" "$image:$tag" - - # Push the tag - docker push "$image:$tag" - - # Remove the tag again, if it's not the main tag - [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" - done - - elif [[ "$action" = run ]]; then - if [[ "$mode" = "dev" ]]; then - # Create caching volumes if needed (they need to be named) - docker volume create fej_build-cache - docker volume create fej_registry-cache - - flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry" - fi - - docker run $attach $flags \ - --rm \ - --interactive \ - --tty \ - --publish 8000:8000 \ - --name fej \ - "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" -fi diff --git a/diesel.toml b/diesel.toml new file mode 100644 index 0000000..92267c8 --- /dev/null +++ b/diesel.toml @@ -0,0 +1,5 @@ +# For documentation on how to configure this file, +# see diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema.rs" diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index b968c5b..b4152fa 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -1,18 +1,36 @@ +# vim: filetype=dockerfile # Our entire toolchain runs in alpine FROM alpine:latest AS builder -ENV PATH "$PATH:/root/.cargo/bin" +ENV PATH "$PATH:/app/.cargo/bin" # Needed for proper compiling of openssl-dev ENV RUSTFLAGS "-C target-feature=-crt-static" -# Otherwise, the debug build can't be used from the container -ENV ROCKET_ADDRESS "0.0.0.0" -WORKDIR /usr/src/app +# Add the build user +# Install dependencies +RUN addgroup -S builder && \ + adduser -S builder -G builder -h /app && \ + apk update && \ + apk add --no-cache \ + curl \ + gcc \ + libgcc \ + musl-dev \ + openssl-dev \ + postgresql-dev -# Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev build-base curl && \ - { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } +# Switch to the non-root user +USER builder + +WORKDIR /app + +# Install rustup in the new user's home +# Create mountpoints for volumes with correct permissions +RUN { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ + rustup target add x86_64-unknown-linux-musl --toolchain nightly && \ + mkdir -p .cargo/registry target # Copy source code over to builder -COPY Cargo.toml Cargo.lock ./ -COPY src/ ./src/ +COPY --chown=builder:builder Cargo.toml Cargo.lock ./ +COPY --chown=builder:builder src/ ./src/ +COPY --chown=builder:builder migrations/ ./migrations/ diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 9205fd4..bac8b44 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,6 +1,9 @@ +# vim: filetype=dockerfile FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 -ENTRYPOINT ["cargo"] -CMD ["run"] +COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh +COPY --chown=builder:builder ./Rocket.toml /app/Rocket.toml + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 6b4e438..4f4c54d 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -1,3 +1,4 @@ +# vim: filetype=dockerfile FROM chewingbever/fej-builder:latest AS builder # And then finally, build the project @@ -5,19 +6,44 @@ FROM chewingbever/fej-builder:latest AS builder # https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 # TODO add what these flags do & why they work # NOTE: cargo install auto-appends bin to the path -RUN --mount=type=cache,target=/usr/src/app/target \ - --mount=type=cache,target=/root/.cargo/registry \ - cargo install --path . --bin fej --root /usr/local + +# RUN --mount=type=cache,mode=0777,target=/app/target \ +# --mount=type=cache,mode=0777,target=/app/.cargo/registry \ + +# Buildkit cache mounts really don't like it when you're not root, +# so I guess we're building release without a cache for now +RUN cargo install \ + --path . \ + --root /app/output \ + --target x86_64-unknown-linux-musl # Now, we create the actual image FROM alpine:latest +COPY ./docker/crontab /var/spool/cron/crontabs/fej # Install some dynamic libraries needed for everything to work -RUN apk update && apk add --no-cache openssl libgcc curl +# Create -non-root user +# Change permissions for crontab file +RUN apk update && \ + apk add --no-cache \ + curl \ + libgcc \ + libpq \ + openssl && \ + addgroup -S fej && \ + adduser -S fej -G fej -h /app + +# Switch to non-root user +USER fej:fej # Copy binary over to final image -COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej +COPY --from=builder --chown=fej:fej /app/output/bin /app/bin + +# Embed config file inside container +# The workdir is changed so that the config file is read properly +WORKDIR /app +COPY --chown=fej:fej Rocket.toml /app/Rocket.toml HEALTHCHECK \ --interval=10s \ @@ -26,4 +52,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -CMD ["/usr/local/bin/fej"] +ENTRYPOINT ["/app/bin/server"] diff --git a/docker/crontab b/docker/crontab new file mode 100644 index 0000000..16fbbcd --- /dev/null +++ b/docker/crontab @@ -0,0 +1 @@ +# This'll be filled up later diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 0000000..bf7fb6f --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,31 @@ +version: '2.4' +services: + app: + build: + # Make sure the build context is one directory up + context: '..' + dockerfile: './docker/Dockerfile.dev' + image: 'chewingbever/fej:dev' + restart: 'no' + + container_name: 'fej_app' + volumes: + - 'build-cache:/app/target' + - 'registry-cache:/app/.cargo/registry' + ports: + - '8000:8000' + + command: "${CMD}" + + db: + container_name: 'fej_db' + restart: 'no' + + # the devop environment exposes the database so we can use the Diesel cli + ports: + - '5432:5432' + +volumes: + build-cache: + registry-cache: + diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml new file mode 100644 index 0000000..88c3a22 --- /dev/null +++ b/docker/docker-compose.override.yml @@ -0,0 +1,14 @@ +version: '2.4' + +services: + cron: + image: 'chewingbever/fej:latest' + restart: 'always' + + entrypoint: 'crond -f' + user: 'root' + healthcheck: + disable: true + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..f2a19e0 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,34 @@ +version: '2.4' + +services: + app: + build: + context: '..' + dockerfile: 'docker/Dockerfile.rel' + + image: 'chewingbever/fej:latest' + restart: 'always' + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' + + db: + image: 'postgres:13-alpine' + restart: 'always' + + environment: + - 'POSTGRES_DB=fej' + - 'POSTGRES_USER=fej' + - 'POSTGRES_PASSWORD=fej' + healthcheck: + test: 'pg_isready -U fej' + interval: '30s' + timeout: '5s' + retries: 3 + start_period: '0s' + + volumes: + - 'db-data:/var/lib/postgresql/data' + +volumes: + db-data: diff --git a/docker/entrypoint_dev.sh b/docker/entrypoint_dev.sh new file mode 100755 index 0000000..0838bce --- /dev/null +++ b/docker/entrypoint_dev.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh + +# All this file does is inject the target +cargo "$@" --target x86_64-unknown-linux-musl diff --git a/fejctl b/fejctl new file mode 100755 index 0000000..6701343 --- /dev/null +++ b/fejctl @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +image='chewingbever/fej' + +# Small wrapper around the docker-compose command +# +# Flags: +# -b: build the builder +# -r: use the release image instead +function dc() { + local OPTIND c build_builder release + + while getopts ":br" c; do + case $c in + b ) build_builder=1 ;; + r ) release=1 ;; + esac + done + shift $((OPTIND-1)) + + if [[ "$build_builder" -eq 1 ]]; then + # We always rebuild the builder before we run any compose command + DOCKER_BUILDKIT=1 docker build \ + -f docker/Dockerfile.builder \ + -t "$image-builder:latest" . || { + >&2 echo "Failed to build builder."; + exit 1; + } + fi + + if [[ "$release" -eq 1 ]]; then + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ + --file docker/docker-compose.yml \ + --file docker/docker-compose.override.yml \ + --project-name fej \ + "$@" + + else + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ + --file docker/docker-compose.yml \ + --file docker/docker-compose.dev.yml \ + --project-name fej-dev \ + "$@" + fi +} + +# Execute the debug image (must be built first) +# +# $@: the arguments to pass to the image (passed as arguments to cargo) +function dcr() { + CMD="$@" dc -b -- up \ + --build \ + --detach +} + +# Tags & pushes the release version to Docker Hub +function publish() { + local branch=`git rev-parse --abbrev-ref HEAD` + + if [[ "$branch" != master ]]; then + >&2 echo "You can only publish from master." + exit 2 + fi + + # Build the release images + dc -br build + + local patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` + local major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` + local minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` + local tags=("latest" "$patch_version" "$minor_version" "$major_version") + + for tag in "${tags[@]}"; do + # Create the tag + docker tag "$image:$tags" "$image:$tag" + + # Push the tag + docker push "$image:$tag" + + # Remove the tag again, if it's not the main tag + [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" + done + +} + +# Entrypoint to the script +# +# $1: action to perform, defaults to 'build' +# $2: binary to use, defaults to 'server' +function main() { + # Default values + local cmd="${1:-build}" + local bin="${2:-server}" + + case $cmd in + # Building + b | build ) dcr build --bin "$bin" && dc -- logs -f app ;; + br | build-release ) dc -br build ;; + + # Running + r | run ) dcr run --bin "$bin" && dc -- logs -f app ;; + rr | run-release ) dc -br -- up --build --detach && dc -r -- logs -f app ;; + s | stop ) dc down ;; + sr | stop-release ) dc -r stop ;; + + # Ease of life + psql ) dc -- exec db psql -U fej -d fej ;; + sh ) dc -- exec app sh ;; + + # Misc + docs ) cargo doc --no-deps ;; + format ) cargo fmt ;; + l | logs ) dc -- logs -f app ;; + lint ) cargo fmt -- --check ;; + p | push | publish ) publish ;; + t | test ) dcr -- test --no-fail-fast && dc -- logs -f app ;; + * ) >&2 echo "Invalid command."; exit 1 ;; + esac +} + +main "$@" diff --git a/migrations/.gitkeep b/migrations/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/migrations/00000000000000_diesel_initial_setup/down.sql b/migrations/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 0000000..a9f5260 --- /dev/null +++ b/migrations/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/migrations/00000000000000_diesel_initial_setup/up.sql b/migrations/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 0000000..d68895b --- /dev/null +++ b/migrations/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/migrations/2021-04-15-155511_ivago_search/down.sql b/migrations/2021-04-15-155511_ivago_search/down.sql new file mode 100644 index 0000000..9d9d004 --- /dev/null +++ b/migrations/2021-04-15-155511_ivago_search/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP SCHEMA ivago CASCADE; diff --git a/migrations/2021-04-15-155511_ivago_search/up.sql b/migrations/2021-04-15-155511_ivago_search/up.sql new file mode 100644 index 0000000..f33eff1 --- /dev/null +++ b/migrations/2021-04-15-155511_ivago_search/up.sql @@ -0,0 +1,9 @@ +-- Your SQL goes here +CREATE SCHEMA ivago; + +CREATE TABLE ivago.streets ( + name TEXT NOT NULL, + city TEXT NOT NULL, + + PRIMARY KEY (name, city) +); diff --git a/roadmap.md b/roadmap.md index f03e4b5..bfdd622 100644 --- a/roadmap.md +++ b/roadmap.md @@ -69,8 +69,8 @@ setting up a database for this version anyways. ## Kissanime I like watching anime from time to time, and I've always used Kissanime for -this. However, their sites can be quite slow, and riddled with ads from time to -time. That's why I'd like to create a high-speed wrapper that extracts all the -needed info from their sites, removing the need for the user to ever actually -visit their site. The API can just act as a fast search index, complete with -indexing of the links to the videos and everything. +this. However, their sites can be quite slow, and riddled with ads. That's why +I'd like to create a high-speed wrapper that extracts all the needed info from +their sites, removing the need for the user to ever actually visit their site. +The API can just act as a fast search index, complete with indexing of the +links to the videos and everything. diff --git a/src/errors.rs b/src/fej/errors.rs similarity index 89% rename from src/errors.rs rename to src/fej/errors.rs index afcf04a..730714c 100644 --- a/src/errors.rs +++ b/src/fej/errors.rs @@ -10,6 +10,8 @@ pub enum FejError { FailedRequest, } +// I'd love to move this over to the server binary, but right now, error E0117 is making that +// imopssible impl From for Status { fn from(err: FejError) -> Status { match err { diff --git a/src/ivago/controller/basic_date.rs b/src/fej/ivago/basic_date.rs similarity index 100% rename from src/ivago/controller/basic_date.rs rename to src/fej/ivago/basic_date.rs diff --git a/src/ivago/controller/mod.rs b/src/fej/ivago/mod.rs similarity index 100% rename from src/ivago/controller/mod.rs rename to src/fej/ivago/mod.rs diff --git a/src/ivago/controller/pickup_time.rs b/src/fej/ivago/pickup_time.rs similarity index 100% rename from src/ivago/controller/pickup_time.rs rename to src/fej/ivago/pickup_time.rs diff --git a/src/ivago/controller/street.rs b/src/fej/ivago/street.rs similarity index 100% rename from src/ivago/controller/street.rs rename to src/fej/ivago/street.rs diff --git a/src/lib.rs b/src/fej/lib.rs similarity index 67% rename from src/lib.rs rename to src/fej/lib.rs index b1e2733..761c91b 100644 --- a/src/lib.rs +++ b/src/fej/lib.rs @@ -1,11 +1,7 @@ #![feature(proc_macro_hygiene, decl_macro)] -#[macro_use] -extern crate rocket; - // Route modules pub mod ivago; // Helper modules -pub mod catchers; pub mod errors; diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 8b8edc7..0000000 --- a/src/main.rs +++ /dev/null @@ -1,42 +0,0 @@ -#[macro_use] -extern crate rocket; - -use fej_lib::{catchers, ivago}; - -// Very temporary solution for CORS -// https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs -use rocket::fairing::{Fairing, Info, Kind}; -use rocket::http::Header; -use rocket::{Request, Response}; - -pub struct CORS; - -impl Fairing for CORS { - fn info(&self) -> Info { - Info { - name: "Add CORS headers to responses", - kind: Kind::Response, - } - } - - fn on_response(&self, _: &Request, response: &mut Response) { - response.set_header(Header::new("Access-Control-Allow-Origin", "*")); - response.set_header(Header::new( - "Access-Control-Allow-Methods", - "POST, GET, PATCH, OPTIONS", - )); - response.set_header(Header::new("Access-Control-Allow-Headers", "*")); - response.set_header(Header::new("Access-Control-Allow-Credentials", "true")); - } -} - -fn rocket() -> rocket::Rocket { - rocket::ignite() - .attach(CORS) - .mount("/ivago", ivago::routes()) - .register(catchers![catchers::not_found]) -} - -fn main() { - rocket().launch(); -} diff --git a/src/schema.rs b/src/schema.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/catchers.rs b/src/server/catchers.rs similarity index 100% rename from src/catchers.rs rename to src/server/catchers.rs diff --git a/src/server/main.rs b/src/server/main.rs new file mode 100644 index 0000000..4267776 --- /dev/null +++ b/src/server/main.rs @@ -0,0 +1,73 @@ +#![feature(proc_macro_hygiene, decl_macro)] + +#[macro_use] +extern crate rocket; +#[macro_use] +extern crate rocket_contrib; +#[macro_use] +extern crate diesel_migrations; + +#[cfg(test)] +mod tests; + +mod catchers; +mod routes; + +// Very temporary solution for CORS +// https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs +use rocket::fairing::AdHoc; +use rocket::fairing::{Fairing, Info, Kind}; +use rocket::http::Header; +use rocket::{Request, Response, Rocket}; +use rocket_contrib::databases::diesel; + +pub struct CORS; + +impl Fairing for CORS { + fn info(&self) -> Info { + Info { + name: "Add CORS headers to responses", + kind: Kind::Response, + } + } + + fn on_response(&self, _: &Request, response: &mut Response) { + response.set_header(Header::new("Access-Control-Allow-Origin", "*")); + response.set_header(Header::new( + "Access-Control-Allow-Methods", + "POST, GET, PATCH, OPTIONS", + )); + response.set_header(Header::new("Access-Control-Allow-Headers", "*")); + response.set_header(Header::new("Access-Control-Allow-Credentials", "true")); + } +} + +// Macro from diesel_migrations that sets up migrations +embed_migrations!(); + +// This defines a connection to the database +#[database("postgres_fej")] +struct FejDbConn(diesel::PgConnection); + +// I'd like to thank Stackoverflow for helping me with this +// https://stackoverflow.com/questions/61047355/how-to-run-diesel-migration-with-rocket-in-production +fn run_db_migrations(rocket: Rocket) -> Result { + let conn = FejDbConn::get_one(&rocket).expect("database connection"); + match embedded_migrations::run(&*conn) { + Ok(()) => Ok(rocket), + Err(e) => Err(rocket), + } +} + +fn rocket() -> rocket::Rocket { + rocket::ignite() + .attach(CORS) + .attach(FejDbConn::fairing()) + .attach(AdHoc::on_attach("Database Migrations", run_db_migrations)) + .mount("/ivago", routes::ivago()) + .register(catchers![catchers::not_found]) +} + +fn main() { + rocket().launch(); +} diff --git a/src/ivago/mod.rs b/src/server/routes/ivago.rs similarity index 83% rename from src/ivago/mod.rs rename to src/server/routes/ivago.rs index 627e89b..d8318bd 100644 --- a/src/ivago/mod.rs +++ b/src/server/routes/ivago.rs @@ -1,14 +1,7 @@ -mod controller; - -use controller::{get_pickup_times, search_streets}; -use controller::{BasicDate, PickupTime, Street}; +use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street}; use rocket::http::Status; use rocket_contrib::json::Json; -pub fn routes() -> Vec { - routes![route_search_streets, route_get_pickup_times] -} - /// This route handles the Ivago search endpoint. It returns a list of streets, /// consisting of a street name & a city. /// diff --git a/src/server/routes/mod.rs b/src/server/routes/mod.rs new file mode 100644 index 0000000..449f82f --- /dev/null +++ b/src/server/routes/mod.rs @@ -0,0 +1,5 @@ +mod ivago; + +pub fn ivago() -> Vec { + routes![ivago::route_search_streets, ivago::route_get_pickup_times] +} diff --git a/tests/ivago.rs b/src/server/tests.rs similarity index 93% rename from tests/ivago.rs rename to src/server/tests.rs index d78501a..8871762 100644 --- a/tests/ivago.rs +++ b/src/server/tests.rs @@ -3,7 +3,7 @@ use rocket::http::Status; use rocket::local::Client; fn rocket() -> rocket::Rocket { - rocket::ignite().mount("/", fej_lib::ivago::routes()) + rocket::ignite().mount("/", super::routes::ivago()) } /// Test 404 response