Merge branch 'develop'

master v1.0.2
Jef Roosens 2021-04-17 21:44:49 +02:00
commit 53334fd56f
Signed by: Jef Roosens
GPG Key ID: B580B976584B5F30
38 changed files with 659 additions and 296 deletions

View File

@ -6,3 +6,15 @@
# Cargo files # Cargo files
!Cargo.toml !Cargo.toml
!Cargo.lock !Cargo.lock
# Entrypoint for devop container
!docker/entrypoint_dev.sh
!docker/entrypoint.sh
# Config file
!Rocket.toml
# Database migrations
!migrations/
!docker/crontab

2
.env 100644
View File

@ -0,0 +1,2 @@
# This file is read by diesel
DATABASE_URL=postgres://fej:fej@localhost:5432/fej

View File

@ -1,8 +1,8 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# This hook lints the code, and if we're on develop or master, also forces the tests to pass. # This hook lints the code, and if we're on develop or master, also forces the tests to pass.
make lint &> /dev/null 2>&1 || { ./fejctl lint &> /dev/null 2>&1 || {
>&2 echo "Format check failed, use 'make lint' for more information."; >&2 echo "Format check failed, use './fejctl lint' for more information.";
exit 1; exit 1;
} }

190
Cargo.lock generated
View File

@ -195,9 +195,9 @@ dependencies = [
[[package]] [[package]]
name = "const_fn" name = "const_fn"
version = "0.4.6" version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28" checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec"
[[package]] [[package]]
name = "cookie" name = "cookie"
@ -233,7 +233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3" checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3"
dependencies = [ dependencies = [
"cookie 0.14.4", "cookie 0.14.4",
"idna 0.2.2", "idna 0.2.3",
"log 0.4.14", "log 0.4.14",
"publicsuffix", "publicsuffix",
"serde", "serde",
@ -321,6 +321,40 @@ dependencies = [
"syn 0.15.44", "syn 0.15.44",
] ]
[[package]]
name = "diesel"
version = "1.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "047bfc4d5c3bd2ef6ca6f981941046113524b9a9f9a7cbdfdd7ff40f58e6f542"
dependencies = [
"bitflags",
"byteorder",
"diesel_derives",
"pq-sys",
"r2d2",
]
[[package]]
name = "diesel_derives"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.69",
]
[[package]]
name = "diesel_migrations"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c"
dependencies = [
"migrations_internals",
"migrations_macros",
]
[[package]] [[package]]
name = "digest" name = "digest"
version = "0.9.0" version = "0.9.0"
@ -347,10 +381,12 @@ dependencies = [
[[package]] [[package]]
name = "fej" name = "fej"
version = "1.0.1" version = "1.0.2"
dependencies = [ dependencies = [
"chrono", "chrono",
"chrono-tz", "chrono-tz",
"diesel",
"diesel_migrations",
"regex", "regex",
"reqwest", "reqwest",
"rocket", "rocket",
@ -599,9 +635,9 @@ dependencies = [
[[package]] [[package]]
name = "httparse" name = "httparse"
version = "1.3.6" version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc35c995b9d93ec174cf9a27d425c7892722101e14993cd227fdb51d70cf9589" checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437"
[[package]] [[package]]
name = "httpdate" name = "httpdate"
@ -678,9 +714,9 @@ dependencies = [
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.2.2" version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
dependencies = [ dependencies = [
"matches", "matches",
"unicode-bidi", "unicode-bidi",
@ -717,6 +753,15 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "instant"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
dependencies = [
"cfg-if 1.0.0",
]
[[package]] [[package]]
name = "iovec" name = "iovec"
version = "0.1.4" version = "0.1.4"
@ -781,6 +826,15 @@ version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41"
[[package]]
name = "lock_api"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176"
dependencies = [
"scopeguard",
]
[[package]] [[package]]
name = "log" name = "log"
version = "0.3.9" version = "0.3.9"
@ -811,6 +865,27 @@ version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "migrations_internals"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860"
dependencies = [
"diesel",
]
[[package]]
name = "migrations_macros"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c"
dependencies = [
"migrations_internals",
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.69",
]
[[package]] [[package]]
name = "mime" name = "mime"
version = "0.2.6" version = "0.2.6"
@ -922,9 +997,9 @@ dependencies = [
[[package]] [[package]]
name = "notify" name = "notify"
version = "4.0.15" version = "4.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd" checksum = "2599080e87c9bd051ddb11b10074f4da7b1223298df65d4c2ec5bcf309af1533"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"filetime", "filetime",
@ -1021,6 +1096,31 @@ dependencies = [
"vcpkg", "vcpkg",
] ]
[[package]]
name = "parking_lot"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
dependencies = [
"cfg-if 1.0.0",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi 0.3.9",
]
[[package]] [[package]]
name = "parse-zoneinfo" name = "parse-zoneinfo"
version = "0.3.0" version = "0.3.0"
@ -1066,18 +1166,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]] [[package]]
name = "pin-project" name = "pin-project"
version = "1.0.6" version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6" checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4"
dependencies = [ dependencies = [
"pin-project-internal", "pin-project-internal",
] ]
[[package]] [[package]]
name = "pin-project-internal" name = "pin-project-internal"
version = "1.0.6" version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f"
dependencies = [ dependencies = [
"proc-macro2 1.0.26", "proc-macro2 1.0.26",
"quote 1.0.9", "quote 1.0.9",
@ -1119,6 +1219,15 @@ version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "pq-sys"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda"
dependencies = [
"vcpkg",
]
[[package]] [[package]]
name = "proc-macro-hack" name = "proc-macro-hack"
version = "0.5.19" version = "0.5.19"
@ -1149,7 +1258,7 @@ version = "1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f" checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f"
dependencies = [ dependencies = [
"idna 0.2.2", "idna 0.2.3",
"url 2.2.1", "url 2.2.1",
] ]
@ -1171,6 +1280,17 @@ dependencies = [
"proc-macro2 1.0.26", "proc-macro2 1.0.26",
] ]
[[package]]
name = "r2d2"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f"
dependencies = [
"log 0.4.14",
"parking_lot",
"scheduled-thread-pool",
]
[[package]] [[package]]
name = "rand" name = "rand"
version = "0.8.3" version = "0.8.3"
@ -1213,9 +1333,9 @@ dependencies = [
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.2.5" version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041"
dependencies = [ dependencies = [
"bitflags", "bitflags",
] ]
@ -1326,13 +1446,28 @@ version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7954a707f9ca18aa74ca8c1f5d1f900f52a4dceb68e96e3112143f759cfd20e" checksum = "d7954a707f9ca18aa74ca8c1f5d1f900f52a4dceb68e96e3112143f759cfd20e"
dependencies = [ dependencies = [
"diesel",
"log 0.4.14", "log 0.4.14",
"notify", "notify",
"r2d2",
"rocket", "rocket",
"rocket_contrib_codegen",
"serde", "serde",
"serde_json", "serde_json",
] ]
[[package]]
name = "rocket_contrib_codegen"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30deb6dec53b91fac3538a2a3935cf13e0f462745f9f33bf27bedffbe7265b5d"
dependencies = [
"devise",
"quote 0.6.13",
"version_check 0.9.3",
"yansi",
]
[[package]] [[package]]
name = "rocket_http" name = "rocket_http"
version = "0.4.7" version = "0.4.7"
@ -1390,6 +1525,21 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "scheduled-thread-pool"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7"
dependencies = [
"parking_lot",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]] [[package]]
name = "security-framework" name = "security-framework"
version = "2.2.0" version = "2.2.0"
@ -1847,16 +1997,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b"
dependencies = [ dependencies = [
"form_urlencoded", "form_urlencoded",
"idna 0.2.2", "idna 0.2.3",
"matches", "matches",
"percent-encoding 2.1.0", "percent-encoding 2.1.0",
] ]
[[package]] [[package]]
name = "vcpkg" name = "vcpkg"
version = "0.2.11" version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d"
[[package]] [[package]]
name = "version_check" name = "version_check"

View File

@ -1,23 +1,24 @@
[package] [package]
name = "fej" name = "fej"
version = "1.0.1" version = "1.0.2"
authors = ["Jef Roosens <roosensjef@gmail.com>"] authors = ["Jef Roosens <roosensjef@gmail.com>"]
edition = "2018" edition = "2018"
[lib] [lib]
name = "fej_lib" name = "fej"
src = "src/lib.rs" path = "src/fej/lib.rs"
test = true test = true
bench = true bench = true
doc = true doc = true
doctest = true doctest = true
[[bin]] [[bin]]
name = "fej" name = "server"
src = "src/main.rs" path = "src/server/main.rs"
test = false test = true
bench = false bench = true
doc = false doc = true
doctest = true
[dependencies] [dependencies]
rocket = "0.4.7" rocket = "0.4.7"
@ -25,13 +26,11 @@ serde = "1.0.124"
chrono = "0.4.19" chrono = "0.4.19"
chrono-tz = "0.5.3" chrono-tz = "0.5.3"
regex = "1.4.5" regex = "1.4.5"
reqwest = { version = "0.11.2", features = ["blocking", "json", "cookies"] }
[dependencies.reqwest] diesel = { version = "1.4.6", features = ["postgres"] }
version = "0.11.2" diesel_migrations = "1.4.0"
default-features = true
features = ["blocking", "json", "cookies"]
[dependencies.rocket_contrib] [dependencies.rocket_contrib]
version = "0.4.7" version = "0.4.7"
default-features = false default-features = false
features = ["json"] features = ["json", "diesel_postgres_pool"]

View File

@ -1,66 +0,0 @@
all: debug
.PHONY: all
# Builds the debug release inside the Alpine container. For build caching, two
# volumes are used named `fej_build-cache` and `fej_registry-cache`. These
# images are automatically created for you if they don't exist. If you
# encounter any strange build errors, you can try removing this volumes to
# start a completely fresh build.
debug:
@ ./build -m dev -a run build
.PHONY: debug
# Builds the release version. In contrary to the debug version, this build
# doesn't use volumes for caching, as this would require the build to happen
# during runtime instead of during the building of the image. Instead, it uses
# the new `--mount` feature from Buildkit. This does mean that only very recent
# Docker engines can build the release version (in my case, at the time of
# writing this, 20.10.5).
release:
@ ./build -m rel
.PHONY: release
# This builds the release version, and pushes all relevant tags to my Docker
# Hub repository, namely chewingbever/fej
push:
@ ./build -m rel -a push
.PHONY: push
# This builds the debug release, and runs it detached. The reason we detach the
# container is because Rocket has a tendency to ignore ctlr-c when inside a
# container, which gets annoying really fast.
run:
@ ./build -m dev -a run
.PHONY: run
# As a workaround, we just have a stop command that stops the container.
stop:
@ docker stop -t 2 fej
.PHONY: stop
# This attaches to the running container, essentially giving the same result as
# just running `cargo run` locally.
logs:
@ docker logs -f fej
.PHONY: logs
# Builds the debug version, and runs the tests (but doesn't detach).
test:
@ ./build -m dev -a run -l -- test --no-fail-fast
.PHONY: test
# Runs the cargo code formatter on your code.
format:
@ cargo fmt
.PHONY: format
# Lints your code. This also gets run in the pre-commit hook, basically
# preventing you from committing badly-formatted code.
lint:
@ cargo fmt -- --check
.PHONY: lint
# This builds the documentation for the project, excluding the documentation.
docs:
@ cargo doc --no-deps
.PHONY: docs

View File

@ -1,25 +1,17 @@
# Fej # Fej
Fej is an API written in Rust. I started this project to learn the language, Fej is a RESTful API that does lots of different things. It started as an
and really just have some fun. experiment to learn Rust, but has grown into a full-on passion project.
## Project Structure ## Project Structure
The folder structure follows the structure of the URLs, e.g. the route for The `src` folder contains subfolders for the various binaries and the main
`/hello/world` is found in the module `src/hello`. library, called `fej`. The biggest binary is called `server`, which is the
binary that actually runs the Rocket.rs web server. All the others are utility
programs, mostly consisting of scrapers for various services.
Each module contains the following base files: Version 1.1 also introduces the use of a database, namely
[PostgreSQL 13](https://www.postgresql.org/).
* `mod.rs`: defines the modules' content, and contains the route definitions.
The route functions themselves only contain the functionality needed to
represent the data, not acquire it.
* `controller.rs`: this file contains the actual logic of each route. If the
logic becomes too complicated to be contained inside a single file,
`controller.rs` becomes its own module folder named `controller`.
* `tests.rs`: this contains tests for the specific module. This can also be a
module directory if need be.
Every module has a `routes` function that returns its route macros.
## Roadmap ## Roadmap
@ -27,19 +19,33 @@ See [roadmap.md](roadmap.md).
## Development ## Development
The entire toolchain runs on Alpine inside Docker. This makes building easier, To make development more consistent (and keep my computer a bit cleaner) I've
and (hopefully) eliminates any system-specific bugs. decided to run the entire toolchain using Docker, with Alpine Linux base
images. This also allows me to make really small final images. Technically,
Docker is the only dependency you need to contribute to this project
(not accounting for language servers etc.).
A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as A [Bash script](fejctl) is provided to speed up development on the various
all the commands are documented for your understanding ;) binaries. It aids in starting up the containers, choosing which binary to run
etc. A quick rundown of its most important features:
There's also the `build` script. This script does all the "heavy" lifting. It ```bash
chooses which Dockerfile to build according to the given arguments, and # By default, it compiles the 'server' binary (but doesn't run it)
generates tags for the images (useful when pushing releases). The Makefile is ./fejctl
really just a wrapper around this build script, allowing you to write
`make test` instead of `./build -m dev -a run test`.
tl;dr run `make run` to run your build, and `make test` to run the tests. # The first argument is the action you wish to perform, the second on which
# binary (not all commands need a binary as input, so they just ignore it)
# For example, this will run the binary called server
./fejctl r server
# This runs the tests (all commands also have their full name if you want)
./fejctl t
./fejctl test
# These attach to the two containers
./fejctl sh # Opens a new root shell inside the 'fej' container
./fejctl psql # Open a new psql shell in the fej database
```
## Docker images ## Docker images

14
Rocket.toml 100644
View File

@ -0,0 +1,14 @@
[development]
address = "0.0.0.0"
port = 8000
keep_alive = 5
read_timeout = 5
write_timeout = 5
log = "normal"
limits = { forms = 32768 }
[development.databases]
postgres_fej = { url = "postgres://fej:fej@fej_db:5432/fej" }
[production.databases]
postgres_fej = { url = "postgres://fej:fej@db:5432/fej" }

91
build
View File

@ -1,91 +0,0 @@
#!/usr/bin/env bash
image="chewingbever/fej"
# Should be either dev or rel
mode="dev"
action=""
attach="--detach"
while getopts ":i:m:a:l" c; do
case $c in
i ) image="$OPTARG" ;;
m ) mode="$OPTARG" ;;
a ) action="$OPTARG" ;;
l ) attach="" ;;
? ) exit 1 ;;
esac
done
shift $((OPTIND-1))
# Extract current version from Cargo.toml & get current branch
patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1`
major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'`
minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'`
branch=`git rev-parse --abbrev-ref HEAD`
if [[ "$branch" = "master" ]]; then
tags=("$patch_version" "$minor_version" "$major_version" "latest")
elif [[ "$branch" = "develop" ]]; then
tags=("$patch_version-dev" "$minor_version-dev" "$major_version-dev" "dev")
else
tags=("$branch")
fi
# First, we build the builder
DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" .
# Run the actual build command
if [ "$mode" = "rel" ]; then
DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel .
elif [[ "$mode" = "dev" ]]; then
DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev .
else
>&2 echo "Invalid mode."
exit 1
fi
if [[ "$action" = push ]]; then
[[ "$branch" =~ ^develop|master$ ]] || {
>&2 echo "You can only push from develop or master."
exit 2
}
[[ "$mode" = "rel" ]] || {
>&2 echo "You can only push release builds."
exit 3
}
for tag in "${tags[@]}"; do
# Create the tag
docker tag "$image:$tags" "$image:$tag"
# Push the tag
docker push "$image:$tag"
# Remove the tag again, if it's not the main tag
[[ "$tag" != "$tags" ]] && docker rmi "$image:$tag"
done
elif [[ "$action" = run ]]; then
if [[ "$mode" = "dev" ]]; then
# Create caching volumes if needed (they need to be named)
docker volume create fej_build-cache
docker volume create fej_registry-cache
flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry"
fi
docker run $attach $flags \
--rm \
--interactive \
--tty \
--publish 8000:8000 \
--name fej \
"$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@"
fi

5
diesel.toml 100644
View File

@ -0,0 +1,5 @@
# For documentation on how to configure this file,
# see diesel.rs/guides/configuring-diesel-cli
[print_schema]
file = "src/schema.rs"

View File

@ -1,18 +1,36 @@
# vim: filetype=dockerfile
# Our entire toolchain runs in alpine # Our entire toolchain runs in alpine
FROM alpine:latest AS builder FROM alpine:latest AS builder
ENV PATH "$PATH:/root/.cargo/bin" ENV PATH "$PATH:/app/.cargo/bin"
# Needed for proper compiling of openssl-dev # Needed for proper compiling of openssl-dev
ENV RUSTFLAGS "-C target-feature=-crt-static" ENV RUSTFLAGS "-C target-feature=-crt-static"
# Otherwise, the debug build can't be used from the container
ENV ROCKET_ADDRESS "0.0.0.0"
WORKDIR /usr/src/app # Add the build user
# Install dependencies
RUN addgroup -S builder && \
adduser -S builder -G builder -h /app && \
apk update && \
apk add --no-cache \
curl \
gcc \
libgcc \
musl-dev \
openssl-dev \
postgresql-dev
# Install build dependencies, rustup & rust's nightly build & toolchain # Switch to the non-root user
RUN apk update && apk add --no-cache openssl-dev build-base curl && \ USER builder
{ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; }
WORKDIR /app
# Install rustup in the new user's home
# Create mountpoints for volumes with correct permissions
RUN { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \
rustup target add x86_64-unknown-linux-musl --toolchain nightly && \
mkdir -p .cargo/registry target
# Copy source code over to builder # Copy source code over to builder
COPY Cargo.toml Cargo.lock ./ COPY --chown=builder:builder Cargo.toml Cargo.lock ./
COPY src/ ./src/ COPY --chown=builder:builder src/ ./src/
COPY --chown=builder:builder migrations/ ./migrations/

View File

@ -1,6 +1,9 @@
# vim: filetype=dockerfile
FROM chewingbever/fej-builder:latest FROM chewingbever/fej-builder:latest
ENV RUST_BACKTRACE 1 ENV RUST_BACKTRACE 1
ENTRYPOINT ["cargo"] COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh
CMD ["run"] COPY --chown=builder:builder ./Rocket.toml /app/Rocket.toml
ENTRYPOINT ["/entrypoint.sh"]

View File

@ -1,3 +1,4 @@
# vim: filetype=dockerfile
FROM chewingbever/fej-builder:latest AS builder FROM chewingbever/fej-builder:latest AS builder
# And then finally, build the project # And then finally, build the project
@ -5,19 +6,44 @@ FROM chewingbever/fej-builder:latest AS builder
# https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 # https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172
# TODO add what these flags do & why they work # TODO add what these flags do & why they work
# NOTE: cargo install auto-appends bin to the path # NOTE: cargo install auto-appends bin to the path
RUN --mount=type=cache,target=/usr/src/app/target \
--mount=type=cache,target=/root/.cargo/registry \ # RUN --mount=type=cache,mode=0777,target=/app/target \
cargo install --path . --bin fej --root /usr/local # --mount=type=cache,mode=0777,target=/app/.cargo/registry \
# Buildkit cache mounts really don't like it when you're not root,
# so I guess we're building release without a cache for now
RUN cargo install \
--path . \
--root /app/output \
--target x86_64-unknown-linux-musl
# Now, we create the actual image # Now, we create the actual image
FROM alpine:latest FROM alpine:latest
COPY ./docker/crontab /var/spool/cron/crontabs/fej
# Install some dynamic libraries needed for everything to work # Install some dynamic libraries needed for everything to work
RUN apk update && apk add --no-cache openssl libgcc curl # Create -non-root user
# Change permissions for crontab file
RUN apk update && \
apk add --no-cache \
curl \
libgcc \
libpq \
openssl && \
addgroup -S fej && \
adduser -S fej -G fej -h /app
# Switch to non-root user
USER fej:fej
# Copy binary over to final image # Copy binary over to final image
COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej COPY --from=builder --chown=fej:fej /app/output/bin /app/bin
# Embed config file inside container
# The workdir is changed so that the config file is read properly
WORKDIR /app
COPY --chown=fej:fej Rocket.toml /app/Rocket.toml
HEALTHCHECK \ HEALTHCHECK \
--interval=10s \ --interval=10s \
@ -26,4 +52,4 @@ HEALTHCHECK \
--retries=3 \ --retries=3 \
CMD curl -q localhost:8000 CMD curl -q localhost:8000
CMD ["/usr/local/bin/fej"] ENTRYPOINT ["/app/bin/server"]

1
docker/crontab 100644
View File

@ -0,0 +1 @@
# This'll be filled up later

View File

@ -0,0 +1,31 @@
version: '2.4'
services:
app:
build:
# Make sure the build context is one directory up
context: '..'
dockerfile: './docker/Dockerfile.dev'
image: 'chewingbever/fej:dev'
restart: 'no'
container_name: 'fej_app'
volumes:
- 'build-cache:/app/target'
- 'registry-cache:/app/.cargo/registry'
ports:
- '8000:8000'
command: "${CMD}"
db:
container_name: 'fej_db'
restart: 'no'
# the devop environment exposes the database so we can use the Diesel cli
ports:
- '5432:5432'
volumes:
build-cache:
registry-cache:

View File

@ -0,0 +1,14 @@
version: '2.4'
services:
cron:
image: 'chewingbever/fej:latest'
restart: 'always'
entrypoint: 'crond -f'
user: 'root'
healthcheck:
disable: true
environment:
- 'DATABASE_URL=postgres://fej:fej@db:5432/fej'

View File

@ -0,0 +1,34 @@
version: '2.4'
services:
app:
build:
context: '..'
dockerfile: 'docker/Dockerfile.rel'
image: 'chewingbever/fej:latest'
restart: 'always'
environment:
- 'DATABASE_URL=postgres://fej:fej@db:5432/fej'
db:
image: 'postgres:13-alpine'
restart: 'always'
environment:
- 'POSTGRES_DB=fej'
- 'POSTGRES_USER=fej'
- 'POSTGRES_PASSWORD=fej'
healthcheck:
test: 'pg_isready -U fej'
interval: '30s'
timeout: '5s'
retries: 3
start_period: '0s'
volumes:
- 'db-data:/var/lib/postgresql/data'
volumes:
db-data:

View File

@ -0,0 +1,4 @@
#!/usr/bin/env sh
# All this file does is inject the target
cargo "$@" --target x86_64-unknown-linux-musl

121
fejctl 100755
View File

@ -0,0 +1,121 @@
#!/usr/bin/env bash
image='chewingbever/fej'
# Small wrapper around the docker-compose command
#
# Flags:
# -b: build the builder
# -r: use the release image instead
function dc() {
local OPTIND c build_builder release
while getopts ":br" c; do
case $c in
b ) build_builder=1 ;;
r ) release=1 ;;
esac
done
shift $((OPTIND-1))
if [[ "$build_builder" -eq 1 ]]; then
# We always rebuild the builder before we run any compose command
DOCKER_BUILDKIT=1 docker build \
-f docker/Dockerfile.builder \
-t "$image-builder:latest" . || {
>&2 echo "Failed to build builder.";
exit 1;
}
fi
if [[ "$release" -eq 1 ]]; then
DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \
--file docker/docker-compose.yml \
--file docker/docker-compose.override.yml \
--project-name fej \
"$@"
else
DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \
--file docker/docker-compose.yml \
--file docker/docker-compose.dev.yml \
--project-name fej-dev \
"$@"
fi
}
# Execute the debug image (must be built first)
#
# $@: the arguments to pass to the image (passed as arguments to cargo)
function dcr() {
CMD="$@" dc -b -- up \
--build \
--detach
}
# Tags & pushes the release version to Docker Hub
function publish() {
local branch=`git rev-parse --abbrev-ref HEAD`
if [[ "$branch" != master ]]; then
>&2 echo "You can only publish from master."
exit 2
fi
# Build the release images
dc -br build
local patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1`
local major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'`
local minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'`
local tags=("latest" "$patch_version" "$minor_version" "$major_version")
for tag in "${tags[@]}"; do
# Create the tag
docker tag "$image:$tags" "$image:$tag"
# Push the tag
docker push "$image:$tag"
# Remove the tag again, if it's not the main tag
[[ "$tag" != "$tags" ]] && docker rmi "$image:$tag"
done
}
# Entrypoint to the script
#
# $1: action to perform, defaults to 'build'
# $2: binary to use, defaults to 'server'
function main() {
# Default values
local cmd="${1:-build}"
local bin="${2:-server}"
case $cmd in
# Building
b | build ) dcr build --bin "$bin" && dc -- logs -f app ;;
br | build-release ) dc -br build ;;
# Running
r | run ) dcr run --bin "$bin" && dc -- logs -f app ;;
rr | run-release ) dc -br -- up --build --detach && dc -r -- logs -f app ;;
s | stop ) dc down ;;
sr | stop-release ) dc -r stop ;;
# Ease of life
psql ) dc -- exec db psql -U fej -d fej ;;
sh ) dc -- exec app sh ;;
# Misc
docs ) cargo doc --no-deps ;;
format ) cargo fmt ;;
l | logs ) dc -- logs -f app ;;
lint ) cargo fmt -- --check ;;
p | push | publish ) publish ;;
t | test ) dcr -- test --no-fail-fast && dc -- logs -f app ;;
* ) >&2 echo "Invalid command."; exit 1 ;;
esac
}
main "$@"

View File

View File

@ -0,0 +1,6 @@
-- This file was automatically created by Diesel to setup helper functions
-- and other internal bookkeeping. This file is safe to edit, any future
-- changes will be added to existing projects as new migrations.
DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
DROP FUNCTION IF EXISTS diesel_set_updated_at();

View File

@ -0,0 +1,36 @@
-- This file was automatically created by Diesel to setup helper functions
-- and other internal bookkeeping. This file is safe to edit, any future
-- changes will be added to existing projects as new migrations.
-- Sets up a trigger for the given table to automatically set a column called
-- `updated_at` whenever the row is modified (unless `updated_at` was included
-- in the modified columns)
--
-- # Example
--
-- ```sql
-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
--
-- SELECT diesel_manage_updated_at('users');
-- ```
CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
BEGIN
EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
BEGIN
IF (
NEW IS DISTINCT FROM OLD AND
NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
) THEN
NEW.updated_at := current_timestamp;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;

View File

@ -0,0 +1,2 @@
-- This file should undo anything in `up.sql`
DROP SCHEMA ivago CASCADE;

View File

@ -0,0 +1,9 @@
-- Your SQL goes here
CREATE SCHEMA ivago;
CREATE TABLE ivago.streets (
name TEXT NOT NULL,
city TEXT NOT NULL,
PRIMARY KEY (name, city)
);

View File

@ -69,8 +69,8 @@ setting up a database for this version anyways.
## Kissanime ## Kissanime
I like watching anime from time to time, and I've always used Kissanime for I like watching anime from time to time, and I've always used Kissanime for
this. However, their sites can be quite slow, and riddled with ads from time to this. However, their sites can be quite slow, and riddled with ads. That's why
time. That's why I'd like to create a high-speed wrapper that extracts all the I'd like to create a high-speed wrapper that extracts all the needed info from
needed info from their sites, removing the need for the user to ever actually their sites, removing the need for the user to ever actually visit their site.
visit their site. The API can just act as a fast search index, complete with The API can just act as a fast search index, complete with indexing of the
indexing of the links to the videos and everything. links to the videos and everything.

View File

@ -10,6 +10,8 @@ pub enum FejError {
FailedRequest, FailedRequest,
} }
// I'd love to move this over to the server binary, but right now, error E0117 is making that
// imopssible
impl From<FejError> for Status { impl From<FejError> for Status {
fn from(err: FejError) -> Status { fn from(err: FejError) -> Status {
match err { match err {

View File

@ -1,11 +1,7 @@
#![feature(proc_macro_hygiene, decl_macro)] #![feature(proc_macro_hygiene, decl_macro)]
#[macro_use]
extern crate rocket;
// Route modules // Route modules
pub mod ivago; pub mod ivago;
// Helper modules // Helper modules
pub mod catchers;
pub mod errors; pub mod errors;

View File

@ -1,42 +0,0 @@
#[macro_use]
extern crate rocket;
use fej_lib::{catchers, ivago};
// Very temporary solution for CORS
// https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs
use rocket::fairing::{Fairing, Info, Kind};
use rocket::http::Header;
use rocket::{Request, Response};
pub struct CORS;
impl Fairing for CORS {
fn info(&self) -> Info {
Info {
name: "Add CORS headers to responses",
kind: Kind::Response,
}
}
fn on_response(&self, _: &Request, response: &mut Response) {
response.set_header(Header::new("Access-Control-Allow-Origin", "*"));
response.set_header(Header::new(
"Access-Control-Allow-Methods",
"POST, GET, PATCH, OPTIONS",
));
response.set_header(Header::new("Access-Control-Allow-Headers", "*"));
response.set_header(Header::new("Access-Control-Allow-Credentials", "true"));
}
}
fn rocket() -> rocket::Rocket {
rocket::ignite()
.attach(CORS)
.mount("/ivago", ivago::routes())
.register(catchers![catchers::not_found])
}
fn main() {
rocket().launch();
}

0
src/schema.rs 100644
View File

73
src/server/main.rs 100644
View File

@ -0,0 +1,73 @@
#![feature(proc_macro_hygiene, decl_macro)]
#[macro_use]
extern crate rocket;
#[macro_use]
extern crate rocket_contrib;
#[macro_use]
extern crate diesel_migrations;
#[cfg(test)]
mod tests;
mod catchers;
mod routes;
// Very temporary solution for CORS
// https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs
use rocket::fairing::AdHoc;
use rocket::fairing::{Fairing, Info, Kind};
use rocket::http::Header;
use rocket::{Request, Response, Rocket};
use rocket_contrib::databases::diesel;
pub struct CORS;
impl Fairing for CORS {
fn info(&self) -> Info {
Info {
name: "Add CORS headers to responses",
kind: Kind::Response,
}
}
fn on_response(&self, _: &Request, response: &mut Response) {
response.set_header(Header::new("Access-Control-Allow-Origin", "*"));
response.set_header(Header::new(
"Access-Control-Allow-Methods",
"POST, GET, PATCH, OPTIONS",
));
response.set_header(Header::new("Access-Control-Allow-Headers", "*"));
response.set_header(Header::new("Access-Control-Allow-Credentials", "true"));
}
}
// Macro from diesel_migrations that sets up migrations
embed_migrations!();
// This defines a connection to the database
#[database("postgres_fej")]
struct FejDbConn(diesel::PgConnection);
// I'd like to thank Stackoverflow for helping me with this
// https://stackoverflow.com/questions/61047355/how-to-run-diesel-migration-with-rocket-in-production
fn run_db_migrations(rocket: Rocket) -> Result<Rocket, Rocket> {
let conn = FejDbConn::get_one(&rocket).expect("database connection");
match embedded_migrations::run(&*conn) {
Ok(()) => Ok(rocket),
Err(e) => Err(rocket),
}
}
fn rocket() -> rocket::Rocket {
rocket::ignite()
.attach(CORS)
.attach(FejDbConn::fairing())
.attach(AdHoc::on_attach("Database Migrations", run_db_migrations))
.mount("/ivago", routes::ivago())
.register(catchers![catchers::not_found])
}
fn main() {
rocket().launch();
}

View File

@ -1,14 +1,7 @@
mod controller; use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street};
use controller::{get_pickup_times, search_streets};
use controller::{BasicDate, PickupTime, Street};
use rocket::http::Status; use rocket::http::Status;
use rocket_contrib::json::Json; use rocket_contrib::json::Json;
pub fn routes() -> Vec<rocket::Route> {
routes![route_search_streets, route_get_pickup_times]
}
/// This route handles the Ivago search endpoint. It returns a list of streets, /// This route handles the Ivago search endpoint. It returns a list of streets,
/// consisting of a street name & a city. /// consisting of a street name & a city.
/// ///

View File

@ -0,0 +1,5 @@
mod ivago;
pub fn ivago() -> Vec<rocket::Route> {
routes![ivago::route_search_streets, ivago::route_get_pickup_times]
}

View File

@ -3,7 +3,7 @@ use rocket::http::Status;
use rocket::local::Client; use rocket::local::Client;
fn rocket() -> rocket::Rocket { fn rocket() -> rocket::Rocket {
rocket::ignite().mount("/", fej_lib::ivago::routes()) rocket::ignite().mount("/", super::routes::ivago())
} }
/// Test 404 response /// Test 404 response