From 5b515ecd91cff5bfdc0d2ec33342b0e654e2d2a1 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 17:21:32 +0200 Subject: [PATCH 01/34] [#23] Build script now starts postgres database as well --- Makefile | 6 ++++++ build | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6ad1f50..820f893 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,7 @@ run: # As a workaround, we just have a stop command that stops the container. stop: @ docker stop -t 2 fej + @ docker stop fej_db .PHONY: stop # This attaches to the running container, essentially giving the same result as @@ -44,6 +45,11 @@ logs: @ docker logs -f fej .PHONY: logs +# This just starts up a shell inside the fej container +sh: + @ docker exec -it fej sh +.PHONY: sh + # Builds the debug version, and runs the tests (but doesn't detach). test: @ ./build -m dev -a run -l -- test --no-fail-fast diff --git a/build b/build index d7d86a3..484a8ab 100755 --- a/build +++ b/build @@ -72,7 +72,21 @@ if [[ "$action" = push ]]; then [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" done - elif [[ "$action" = run ]]; then +elif [[ "$action" = run ]]; then + # Create the network & start the database container + docker network create fej + docker volume create fej_db-data + + docker run --rm \ + --detach \ + --name fej_db \ + --network fej \ + -e "POSTGRES_DB=fej" \ + -e "POSTGRES_USER=fej" \ + -e "POSTGRES_PASSWORD=fej" \ + -v 'fej_db-data:/var/lib/postgresql/data' \ + postgres:13-alpine + if [[ "$mode" = "dev" ]]; then # Create caching volumes if needed (they need to be named) docker volume create fej_build-cache @@ -87,5 +101,6 @@ if [[ "$action" = push ]]; then --tty \ --publish 8000:8000 \ --name fej \ + --network fej \ "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" fi From 6954921df622bdf5cd456154c407330be7fca837 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 17:31:45 +0200 Subject: [PATCH 02/34] [#23] Added needed env vars --- .env | 1 + build | 1 + 2 files changed, 2 insertions(+) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..880c44c --- /dev/null +++ b/.env @@ -0,0 +1 @@ +DATABASE_URL=postgres://fej:fej@fej_db:5432/fej diff --git a/build b/build index 484a8ab..962d14c 100755 --- a/build +++ b/build @@ -101,6 +101,7 @@ elif [[ "$action" = run ]]; then --tty \ --publish 8000:8000 \ --name fej \ + --env-file .env \ --network fej \ "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" fi From 727589b10f74c318e472bb3d02c899ff8ddcf6cb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 17:53:35 +0200 Subject: [PATCH 03/34] [#24] Initialized diesel project --- .env | 3 +- .env.container | 2 ++ Cargo.lock | 33 +++++++++++++++++ Cargo.toml | 7 ++-- build | 3 +- diesel.toml | 5 +++ migrations/.gitkeep | 0 .../down.sql | 6 ++++ .../up.sql | 36 +++++++++++++++++++ 9 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 .env.container create mode 100644 diesel.toml create mode 100644 migrations/.gitkeep create mode 100644 migrations/00000000000000_diesel_initial_setup/down.sql create mode 100644 migrations/00000000000000_diesel_initial_setup/up.sql diff --git a/.env b/.env index 880c44c..6e2ca99 100644 --- a/.env +++ b/.env @@ -1 +1,2 @@ -DATABASE_URL=postgres://fej:fej@fej_db:5432/fej +# This file is read by diesel +DATABASE_URL=postgres://fej:fej@localhost:5432/fej diff --git a/.env.container b/.env.container new file mode 100644 index 0000000..8b395fb --- /dev/null +++ b/.env.container @@ -0,0 +1,2 @@ +# This file is read by the container, and therefore by fej +DATABASE_URL=postgres://fej:fej@fej_db:5432/fej diff --git a/Cargo.lock b/Cargo.lock index 8ab2a9c..0006402 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -321,6 +321,29 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "diesel" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "047bfc4d5c3bd2ef6ca6f981941046113524b9a9f9a7cbdfdd7ff40f58e6f542" +dependencies = [ + "bitflags", + "byteorder", + "diesel_derives", + "pq-sys", +] + +[[package]] +name = "diesel_derives" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" +dependencies = [ + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", +] + [[package]] name = "digest" version = "0.9.0" @@ -351,6 +374,7 @@ version = "1.0.1" dependencies = [ "chrono", "chrono-tz", + "diesel", "regex", "reqwest", "rocket", @@ -1119,6 +1143,15 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "pq-sys" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda" +dependencies = [ + "vcpkg", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" diff --git a/Cargo.toml b/Cargo.toml index 83623fa..f359a51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,11 +25,8 @@ serde = "1.0.124" chrono = "0.4.19" chrono-tz = "0.5.3" regex = "1.4.5" - -[dependencies.reqwest] -version = "0.11.2" -default-features = true -features = ["blocking", "json", "cookies"] +reqwest = { version = "0.11.2", features = ["blocking", "json", "cookies"] } +diesel = { version = "1.4.6", features = ["postgres"] } [dependencies.rocket_contrib] version = "0.4.7" diff --git a/build b/build index 962d14c..271a3d9 100755 --- a/build +++ b/build @@ -81,6 +81,7 @@ elif [[ "$action" = run ]]; then --detach \ --name fej_db \ --network fej \ + -p 5432:5432 \ -e "POSTGRES_DB=fej" \ -e "POSTGRES_USER=fej" \ -e "POSTGRES_PASSWORD=fej" \ @@ -101,7 +102,7 @@ elif [[ "$action" = run ]]; then --tty \ --publish 8000:8000 \ --name fej \ - --env-file .env \ + --env-file .env.container \ --network fej \ "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" fi diff --git a/diesel.toml b/diesel.toml new file mode 100644 index 0000000..92267c8 --- /dev/null +++ b/diesel.toml @@ -0,0 +1,5 @@ +# For documentation on how to configure this file, +# see diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema.rs" diff --git a/migrations/.gitkeep b/migrations/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/migrations/00000000000000_diesel_initial_setup/down.sql b/migrations/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 0000000..a9f5260 --- /dev/null +++ b/migrations/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/migrations/00000000000000_diesel_initial_setup/up.sql b/migrations/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 0000000..d68895b --- /dev/null +++ b/migrations/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; From 81ae1a2555370da8738226946ef333d2541689bf Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 18:05:02 +0200 Subject: [PATCH 04/34] [#24] Added first ivago migration --- Makefile | 4 ++++ migrations/2021-04-15-155511_ivago_search/down.sql | 2 ++ migrations/2021-04-15-155511_ivago_search/up.sql | 9 +++++++++ src/schema.rs | 0 4 files changed, 15 insertions(+) create mode 100644 migrations/2021-04-15-155511_ivago_search/down.sql create mode 100644 migrations/2021-04-15-155511_ivago_search/up.sql create mode 100644 src/schema.rs diff --git a/Makefile b/Makefile index 820f893..5d8c22e 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,10 @@ sh: @ docker exec -it fej sh .PHONY: sh +# Starts a psql session in the database container +dbsh: + @ docker exec -it fej_db psql -U fej -d fej + # Builds the debug version, and runs the tests (but doesn't detach). test: @ ./build -m dev -a run -l -- test --no-fail-fast diff --git a/migrations/2021-04-15-155511_ivago_search/down.sql b/migrations/2021-04-15-155511_ivago_search/down.sql new file mode 100644 index 0000000..9d9d004 --- /dev/null +++ b/migrations/2021-04-15-155511_ivago_search/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP SCHEMA ivago CASCADE; diff --git a/migrations/2021-04-15-155511_ivago_search/up.sql b/migrations/2021-04-15-155511_ivago_search/up.sql new file mode 100644 index 0000000..f33eff1 --- /dev/null +++ b/migrations/2021-04-15-155511_ivago_search/up.sql @@ -0,0 +1,9 @@ +-- Your SQL goes here +CREATE SCHEMA ivago; + +CREATE TABLE ivago.streets ( + name TEXT NOT NULL, + city TEXT NOT NULL, + + PRIMARY KEY (name, city) +); diff --git a/src/schema.rs b/src/schema.rs new file mode 100644 index 0000000..e69de29 From cb78af62ba0060465d07f42493d379dce2310a1b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 19:26:37 +0200 Subject: [PATCH 05/34] [#26] Moved main binary to separate location --- Cargo.toml | 6 +++--- docker/Dockerfile.dev | 2 +- docker/Dockerfile.rel | 6 +++--- src/{ => bin/server}/main.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) rename src/{ => bin/server}/main.rs (97%) diff --git a/Cargo.toml b/Cargo.toml index f359a51..46b01a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Jef Roosens "] edition = "2018" [lib] -name = "fej_lib" +name = "fej" src = "src/lib.rs" test = true bench = true @@ -13,8 +13,8 @@ doc = true doctest = true [[bin]] -name = "fej" -src = "src/main.rs" +name = "server" +src = "src/bin/server/main.rs" test = false bench = false doc = false diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 9205fd4..d2029af 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -3,4 +3,4 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 ENTRYPOINT ["cargo"] -CMD ["run"] +CMD ["run", "--bin", "server"] diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 6b4e438..b64916a 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -7,7 +7,7 @@ FROM chewingbever/fej-builder:latest AS builder # NOTE: cargo install auto-appends bin to the path RUN --mount=type=cache,target=/usr/src/app/target \ --mount=type=cache,target=/root/.cargo/registry \ - cargo install --path . --bin fej --root /usr/local + cargo install --path . --root /usr/local # Now, we create the actual image @@ -17,7 +17,7 @@ FROM alpine:latest RUN apk update && apk add --no-cache openssl libgcc curl # Copy binary over to final image -COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej +COPY --from=builder /usr/local/bin/server /usr/local/bin/server HEALTHCHECK \ --interval=10s \ @@ -26,4 +26,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -CMD ["/usr/local/bin/fej"] +CMD ["/usr/local/bin/server"] diff --git a/src/main.rs b/src/bin/server/main.rs similarity index 97% rename from src/main.rs rename to src/bin/server/main.rs index 8b8edc7..a27e00f 100644 --- a/src/main.rs +++ b/src/bin/server/main.rs @@ -1,7 +1,7 @@ #[macro_use] extern crate rocket; -use fej_lib::{catchers, ivago}; +use fej::{catchers, ivago}; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs From 21e142e9a4feaaa01fe292772b262018597c2c8e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 19:36:42 +0200 Subject: [PATCH 06/34] [#26] Added image cleaning recipe --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5d8c22e..f9c6456 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -all: debug +all: debug logs .PHONY: all # Builds the debug release inside the Alpine container. For build caching, two @@ -74,3 +74,7 @@ lint: docs: @ cargo doc --no-deps .PHONY: docs + +# This recipe removes all chewingbever/fej images from your system +clean-images: + @ docker images | grep '^chewingbever/fej' | sed 's/ \+/ /g' | cut -f3 -d' ' | xargs docker rmi From 22e9dcceaf9ce028649d70038b8e702aea571a40 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 22:46:23 +0200 Subject: [PATCH 07/34] [#26] Started writing fejctl --- fejctl | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 fejctl diff --git a/fejctl b/fejctl new file mode 100755 index 0000000..8f3eb74 --- /dev/null +++ b/fejctl @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# This script is a helpful utility for developing fej inside Docker. + +# Default values +image="chewingbever/fej" +mode="dev" # Should be either 'rel' or 'dev' +action="" +attach="--detach" +cmd="b" +bin="server" + +# Calculated variables +patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` +major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` +minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` +branch=`git rev-parse --abbrev-ref HEAD` + + + +function main() { + # Parse the flags + while getopts ":i:m:a:l" c; do + case $c in + i ) image="$OPTARG" ;; + m ) mode="$OPTARG" ;; + a ) action="$OPTARG" ;; + l ) attach="" ;; + ? ) exit 1 ;; + esac + done + shift $((OPTIND-1)) +} + +main "$@" From 0ba31bd8ba944b4bf503366654a70c16e673fff3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 23:59:31 +0200 Subject: [PATCH 08/34] [#26] Started fejctl --- fejctl | 160 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 136 insertions(+), 24 deletions(-) diff --git a/fejctl b/fejctl index 8f3eb74..17a43a0 100755 --- a/fejctl +++ b/fejctl @@ -1,35 +1,147 @@ #!/usr/bin/env bash -# This script is a helpful utility for developing fej inside Docker. +image='chewingbever/fej' -# Default values -image="chewingbever/fej" -mode="dev" # Should be either 'rel' or 'dev' -action="" -attach="--detach" -cmd="b" -bin="server" +# Creates the needed images +# +# $1: wether to build the debug or the release image (default debug) +function create_images() { + # First, we build the builder + DOCKER_BUILDKIT=1 docker build \ + -f docker/Dockerfile.builder \ + -t "$image-builder:latest" . -# Calculated variables -patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` -major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` -minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` -branch=`git rev-parse --abbrev-ref HEAD` + if [[ "$1" = "rel" ]]; then + DOCKER_BUILDKIT=1 docker build \ + -t "$image:latest" \ + -f docker/Dockerfile.rel . + else + # Then, we create the debug image + DOCKER_BUILDKIT=1 docker build \ + -t "$image:dev" \ + -f docker/Dockerfile.dev . + fi +} +# Execute the debug image (must be built first) +# +# $@: the arguments to pass to the image (passed as arguments to cargo) +function run_image() { + # Run the database image + docker run --rm \ + --detach \ + --name fej_db \ + --network fej \ + -p 5432:5432 \ + -e 'POSTGRES_DB=fej' \ + -e 'POSTGRES_USER=fej' \ + -e 'POSTGRES_PASSWORD=fej' \ + -v 'fej_db-data:/var/lib/postgresql/data' \ + postgres:13-alpine -function main() { - # Parse the flags - while getopts ":i:m:a:l" c; do - case $c in - i ) image="$OPTARG" ;; - m ) mode="$OPTARG" ;; - a ) action="$OPTARG" ;; - l ) attach="" ;; - ? ) exit 1 ;; - esac + # Run the binary image + docker run \ + --detach \ + --rm \ + --interactive \ + --tty \ + --publish 8000:8000 \ + --name fej \ + --env-file .env.container \ + --network fej \ + -v 'fej_build-cache:/usr/src/app/target' \ + -v 'fej_registry-cache:/root/.cargo/registry' \ + "$image:dev" "$@" +} + +# Attach to the fej container +function logs() { + docker logs -f fej +} + +# Builds the given binary +# +# $1: the binary to build +function build() { + create_images + run_image build --bin "$1" + logs +} + +# Runs the given binary +# +# $1: the binary to run +function run() { + create_images + run_image run --bin "$1" + logs +} + +# Runs the tests +function tests() { + create_images + run_image test --no-fail-fast + logs +} + +# Stops both containers +function stop() { + docker stop fej_db + docker stop -t 0 fej +} + +function run_release() { + echo "Not implemented" +} + +# Tags & pushes the release version to Docker Hub +function publish() { + branch=`git rev-parse --abbrev-ref HEAD` + + if [[ "$branch" != master ]]; then + >&2 echo "You can only publish from master." + exit 2 + fi + + patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` + major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` + minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` + tags=("latest" "$patch_version" "$minor_version" "$major_version") + + for tag in "${tags[@]}"; do + # Create the tag + docker tag "$image:$tags" "$image:$tag" + + # Push the tag + docker push "$image:$tag" + + # Remove the tag again, if it's not the main tag + [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" done - shift $((OPTIND-1)) + +} + +# Entrypoint to the script +# +# $1: action to perform, defaults to 'build' +# $2: binary to use, defaults to 'server' +function main() { + # Default values + cmd="${1:-build}" + bin="${2:-server}" + + case $cmd in + b | build ) build "$bin" ;; + br | build-release ) create_images rel ;; + r | run ) run "$bin" ;; + rr | run-release ) run_release ;; + p | push | publish ) publish ;; + t | test ) tests ;; + s | stop ) stop ;; + p | push | publish ) publish ;; + * ) >&2 echo "Invalid command."; exit 1 ;; + esac } main "$@" From 0828dd36d6b26010bb3f8b783eccf9f4fdd736b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:06:48 +0200 Subject: [PATCH 09/34] [#26] fejctl now fully replace Makefile & build --- .hooks/pre-commit | 2 +- Makefile | 80 ---------------------------------- build | 108 ---------------------------------------------- fejctl | 16 ++++++- 4 files changed, 15 insertions(+), 191 deletions(-) delete mode 100644 Makefile delete mode 100755 build diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 709fb95..6947bc7 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This hook lints the code, and if we're on develop or master, also forces the tests to pass. -make lint &> /dev/null 2>&1 || { +./fejctl lint &> /dev/null 2>&1 || { >&2 echo "Format check failed, use 'make lint' for more information."; exit 1; } diff --git a/Makefile b/Makefile deleted file mode 100644 index f9c6456..0000000 --- a/Makefile +++ /dev/null @@ -1,80 +0,0 @@ -all: debug logs -.PHONY: all - -# Builds the debug release inside the Alpine container. For build caching, two -# volumes are used named `fej_build-cache` and `fej_registry-cache`. These -# images are automatically created for you if they don't exist. If you -# encounter any strange build errors, you can try removing this volumes to -# start a completely fresh build. -debug: - @ ./build -m dev -a run build -.PHONY: debug - -# Builds the release version. In contrary to the debug version, this build -# doesn't use volumes for caching, as this would require the build to happen -# during runtime instead of during the building of the image. Instead, it uses -# the new `--mount` feature from Buildkit. This does mean that only very recent -# Docker engines can build the release version (in my case, at the time of -# writing this, 20.10.5). -release: - @ ./build -m rel -.PHONY: release - -# This builds the release version, and pushes all relevant tags to my Docker -# Hub repository, namely chewingbever/fej -push: - @ ./build -m rel -a push -.PHONY: push - -# This builds the debug release, and runs it detached. The reason we detach the -# container is because Rocket has a tendency to ignore ctlr-c when inside a -# container, which gets annoying really fast. -run: - @ ./build -m dev -a run -.PHONY: run - -# As a workaround, we just have a stop command that stops the container. -stop: - @ docker stop -t 2 fej - @ docker stop fej_db -.PHONY: stop - -# This attaches to the running container, essentially giving the same result as -# just running `cargo run` locally. -logs: - @ docker logs -f fej -.PHONY: logs - -# This just starts up a shell inside the fej container -sh: - @ docker exec -it fej sh -.PHONY: sh - -# Starts a psql session in the database container -dbsh: - @ docker exec -it fej_db psql -U fej -d fej - -# Builds the debug version, and runs the tests (but doesn't detach). -test: - @ ./build -m dev -a run -l -- test --no-fail-fast -.PHONY: test - -# Runs the cargo code formatter on your code. -format: - @ cargo fmt -.PHONY: format - -# Lints your code. This also gets run in the pre-commit hook, basically -# preventing you from committing badly-formatted code. -lint: - @ cargo fmt -- --check -.PHONY: lint - -# This builds the documentation for the project, excluding the documentation. -docs: - @ cargo doc --no-deps -.PHONY: docs - -# This recipe removes all chewingbever/fej images from your system -clean-images: - @ docker images | grep '^chewingbever/fej' | sed 's/ \+/ /g' | cut -f3 -d' ' | xargs docker rmi diff --git a/build b/build deleted file mode 100755 index 271a3d9..0000000 --- a/build +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash - -image="chewingbever/fej" -# Should be either dev or rel -mode="dev" -action="" -attach="--detach" - -while getopts ":i:m:a:l" c; do - case $c in - i ) image="$OPTARG" ;; - m ) mode="$OPTARG" ;; - a ) action="$OPTARG" ;; - l ) attach="" ;; - ? ) exit 1 ;; - esac -done -shift $((OPTIND-1)) - -# Extract current version from Cargo.toml & get current branch -patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` -major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` -minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` -branch=`git rev-parse --abbrev-ref HEAD` - -if [[ "$branch" = "master" ]]; then - tags=("$patch_version" "$minor_version" "$major_version" "latest") - -elif [[ "$branch" = "develop" ]]; then - tags=("$patch_version-dev" "$minor_version-dev" "$major_version-dev" "dev") - -else - tags=("$branch") - -fi - -# First, we build the builder -DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" . - -# Run the actual build command -if [ "$mode" = "rel" ]; then - DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel . - -elif [[ "$mode" = "dev" ]]; then - DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev . - -else - >&2 echo "Invalid mode." - exit 1 - -fi - -if [[ "$action" = push ]]; then - [[ "$branch" =~ ^develop|master$ ]] || { - >&2 echo "You can only push from develop or master." - exit 2 - } - - [[ "$mode" = "rel" ]] || { - >&2 echo "You can only push release builds." - exit 3 - } - - for tag in "${tags[@]}"; do - # Create the tag - docker tag "$image:$tags" "$image:$tag" - - # Push the tag - docker push "$image:$tag" - - # Remove the tag again, if it's not the main tag - [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" - done - -elif [[ "$action" = run ]]; then - # Create the network & start the database container - docker network create fej - docker volume create fej_db-data - - docker run --rm \ - --detach \ - --name fej_db \ - --network fej \ - -p 5432:5432 \ - -e "POSTGRES_DB=fej" \ - -e "POSTGRES_USER=fej" \ - -e "POSTGRES_PASSWORD=fej" \ - -v 'fej_db-data:/var/lib/postgresql/data' \ - postgres:13-alpine - - if [[ "$mode" = "dev" ]]; then - # Create caching volumes if needed (they need to be named) - docker volume create fej_build-cache - docker volume create fej_registry-cache - - flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry" - fi - - docker run $attach $flags \ - --rm \ - --interactive \ - --tty \ - --publish 8000:8000 \ - --name fej \ - --env-file .env.container \ - --network fej \ - "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" -fi diff --git a/fejctl b/fejctl index 17a43a0..c977f79 100755 --- a/fejctl +++ b/fejctl @@ -132,14 +132,26 @@ function main() { bin="${2:-server}" case $cmd in + # Building b | build ) build "$bin" ;; br | build-release ) create_images rel ;; + + # Running r | run ) run "$bin" ;; rr | run-release ) run_release ;; + s | stop ) stop ;; + + # Ease of life + psql ) docker exec -it fej_db psql -U fej -d fej ;; + sh ) docker exec -it fej sh ;; + + # Misc + docs ) cargo doc --no-deps ;; + format ) cargo fmt ;; + l | logs ) logs ;; + lint ) cargo fmt -- --check ;; p | push | publish ) publish ;; t | test ) tests ;; - s | stop ) stop ;; - p | push | publish ) publish ;; * ) >&2 echo "Invalid command."; exit 1 ;; esac } From d19fe5c42e29af96e3be02662ca6e507b90f19b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:32:03 +0200 Subject: [PATCH 10/34] [#26] Moved all routing to server binary --- .hooks/pre-commit | 2 +- src/{ => bin/server}/catchers.rs | 0 src/bin/server/main.rs | 8 +- src/bin/server/routes/ivago.rs | 38 ++++++++ src/bin/server/routes/mod.rs | 5 + src/ivago/{controller => }/basic_date.rs | 0 src/ivago/controller/mod.rs | 95 ------------------ src/ivago/mod.rs | 112 ++++++++++++++++------ src/ivago/{controller => }/pickup_time.rs | 0 src/ivago/{controller => }/street.rs | 0 src/lib.rs | 4 - 11 files changed, 130 insertions(+), 134 deletions(-) rename src/{ => bin/server}/catchers.rs (100%) create mode 100644 src/bin/server/routes/ivago.rs create mode 100644 src/bin/server/routes/mod.rs rename src/ivago/{controller => }/basic_date.rs (100%) delete mode 100644 src/ivago/controller/mod.rs rename src/ivago/{controller => }/pickup_time.rs (100%) rename src/ivago/{controller => }/street.rs (100%) diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 6947bc7..fe55771 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -2,7 +2,7 @@ # This hook lints the code, and if we're on develop or master, also forces the tests to pass. ./fejctl lint &> /dev/null 2>&1 || { - >&2 echo "Format check failed, use 'make lint' for more information."; + >&2 echo "Format check failed, use './fejctl lint' for more information."; exit 1; } diff --git a/src/catchers.rs b/src/bin/server/catchers.rs similarity index 100% rename from src/catchers.rs rename to src/bin/server/catchers.rs diff --git a/src/bin/server/main.rs b/src/bin/server/main.rs index a27e00f..3fde2c7 100644 --- a/src/bin/server/main.rs +++ b/src/bin/server/main.rs @@ -1,7 +1,9 @@ +#![feature(proc_macro_hygiene, decl_macro)] + #[macro_use] extern crate rocket; - -use fej::{catchers, ivago}; +mod catchers; +mod routes; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs @@ -33,7 +35,7 @@ impl Fairing for CORS { fn rocket() -> rocket::Rocket { rocket::ignite() .attach(CORS) - .mount("/ivago", ivago::routes()) + .mount("/ivago", routes::ivago()) .register(catchers![catchers::not_found]) } diff --git a/src/bin/server/routes/ivago.rs b/src/bin/server/routes/ivago.rs new file mode 100644 index 0000000..d8318bd --- /dev/null +++ b/src/bin/server/routes/ivago.rs @@ -0,0 +1,38 @@ +use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street}; +use rocket::http::Status; +use rocket_contrib::json::Json; + +/// This route handles the Ivago search endpoint. It returns a list of streets, +/// consisting of a street name & a city. +/// +/// # Arguments +/// +/// * `search_term` - Search term to use to look for streets +#[get("/search?")] +pub fn route_search_streets(q: String) -> Result>, Status> { + Ok(Json(search_streets(q.as_str())?)) +} + +/// Handles returning of pickup times for a specific address. It returns a list +/// of pickup times, containing a date and a description of the trash type. +/// +/// # Arguments +/// +/// * `street` - Street to look up +/// * `number` - House number in the given street +/// * `start_date` - Earliest date that can be returned +/// * `end_date` - Latest date that can be returned +#[get("/?&&&")] +pub fn route_get_pickup_times( + street: Street, + number: u32, + start_date: BasicDate, + end_date: BasicDate, +) -> Result>, Status> { + Ok(Json(get_pickup_times( + &street, + &number, + &start_date.0, + &end_date.0, + )?)) +} diff --git a/src/bin/server/routes/mod.rs b/src/bin/server/routes/mod.rs new file mode 100644 index 0000000..449f82f --- /dev/null +++ b/src/bin/server/routes/mod.rs @@ -0,0 +1,5 @@ +mod ivago; + +pub fn ivago() -> Vec { + routes![ivago::route_search_streets, ivago::route_get_pickup_times] +} diff --git a/src/ivago/controller/basic_date.rs b/src/ivago/basic_date.rs similarity index 100% rename from src/ivago/controller/basic_date.rs rename to src/ivago/basic_date.rs diff --git a/src/ivago/controller/mod.rs b/src/ivago/controller/mod.rs deleted file mode 100644 index 7d3a9f9..0000000 --- a/src/ivago/controller/mod.rs +++ /dev/null @@ -1,95 +0,0 @@ -use crate::errors::FejError; -use chrono::DateTime; -use chrono_tz::Tz; -use reqwest::blocking as reqwest; -use std::collections::HashMap; -use std::convert::{From, TryFrom}; - -mod basic_date; -mod pickup_time; -mod street; - -pub use basic_date::BasicDate; -pub use pickup_time::PickupTime; -pub use street::Street; - -/// Endpoint for the search feature -const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; -/// Endpoint for populating the initial cookies -const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; -/// Endpoint for the actual calendar output -const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; - -/// Searches the Ivago API for streets in the given city. -/// -/// # Arguments -/// -/// * `search_term` - Search term to use to look for streets -pub fn search_streets(search_term: &str) -> Result, FejError> { - let client = reqwest::Client::new(); - let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; - let data: Vec> = response.json()?; - - // This is pretty cool, filter_map first does get() on all the maps, and - // then filters out any None values - // Then, we do the same thing for streets - Ok(data - .iter() - .filter_map(|m| m.get("value")) - .filter_map(|v| Street::try_from(v.as_str()).ok()) - .collect()) -} - -/// Returns the pickup times for the various trash types. -/// -/// # Arguments -/// -/// * `street` - Street to look up -/// * `number` - House number in given street -/// * `start_date` - Earliest date for the results -/// * `end_date` - Latest date for the results -pub fn get_pickup_times( - street: &Street, - number: &u32, - start_date: &DateTime, - end_date: &DateTime, -) -> Result, FejError> { - let client = reqwest::Client::builder().cookie_store(true).build()?; - - // This populates the cookies with the necessary values - client - .post(BASE_URL) - .form(&[ - ("garbage_type", ""), - ("ivago_street", &String::from(street)), - ("number", &number.to_string()), - ("form_id", "garbage_address_form"), - ]) - .send()?; - - let response = client - .get(CAL_URL) - .query(&[ - ("_format", "json"), - ("type", ""), - ("start", &start_date.timestamp().to_string()), - ("end", &end_date.timestamp().to_string()), - ]) - .send()?; - let data: Vec> = response.json()?; - - let mut output: Vec = Vec::new(); - - for map in data - .iter() - .filter(|m| m.contains_key("date") && m.contains_key("label")) - { - // Because we filtered the maps in the loop, we can safely use unwrap - // here - if let Ok(date) = BasicDate::try_from(map.get("date").unwrap().as_str()) { - output.push(PickupTime::new(date, map.get("label").unwrap().to_string())) - } - } - - Ok(output) -} diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index 627e89b..7d3a9f9 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -1,45 +1,95 @@ -mod controller; +use crate::errors::FejError; +use chrono::DateTime; +use chrono_tz::Tz; +use reqwest::blocking as reqwest; +use std::collections::HashMap; +use std::convert::{From, TryFrom}; -use controller::{get_pickup_times, search_streets}; -use controller::{BasicDate, PickupTime, Street}; -use rocket::http::Status; -use rocket_contrib::json::Json; +mod basic_date; +mod pickup_time; +mod street; -pub fn routes() -> Vec { - routes![route_search_streets, route_get_pickup_times] -} +pub use basic_date::BasicDate; +pub use pickup_time::PickupTime; +pub use street::Street; -/// This route handles the Ivago search endpoint. It returns a list of streets, -/// consisting of a street name & a city. +/// Endpoint for the search feature +const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; +/// Endpoint for populating the initial cookies +const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; +/// Endpoint for the actual calendar output +const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; + +/// Searches the Ivago API for streets in the given city. /// /// # Arguments /// /// * `search_term` - Search term to use to look for streets -#[get("/search?")] -pub fn route_search_streets(q: String) -> Result>, Status> { - Ok(Json(search_streets(q.as_str())?)) +pub fn search_streets(search_term: &str) -> Result, FejError> { + let client = reqwest::Client::new(); + let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; + let data: Vec> = response.json()?; + + // This is pretty cool, filter_map first does get() on all the maps, and + // then filters out any None values + // Then, we do the same thing for streets + Ok(data + .iter() + .filter_map(|m| m.get("value")) + .filter_map(|v| Street::try_from(v.as_str()).ok()) + .collect()) } -/// Handles returning of pickup times for a specific address. It returns a list -/// of pickup times, containing a date and a description of the trash type. +/// Returns the pickup times for the various trash types. /// /// # Arguments /// /// * `street` - Street to look up -/// * `number` - House number in the given street -/// * `start_date` - Earliest date that can be returned -/// * `end_date` - Latest date that can be returned -#[get("/?&&&")] -pub fn route_get_pickup_times( - street: Street, - number: u32, - start_date: BasicDate, - end_date: BasicDate, -) -> Result>, Status> { - Ok(Json(get_pickup_times( - &street, - &number, - &start_date.0, - &end_date.0, - )?)) +/// * `number` - House number in given street +/// * `start_date` - Earliest date for the results +/// * `end_date` - Latest date for the results +pub fn get_pickup_times( + street: &Street, + number: &u32, + start_date: &DateTime, + end_date: &DateTime, +) -> Result, FejError> { + let client = reqwest::Client::builder().cookie_store(true).build()?; + + // This populates the cookies with the necessary values + client + .post(BASE_URL) + .form(&[ + ("garbage_type", ""), + ("ivago_street", &String::from(street)), + ("number", &number.to_string()), + ("form_id", "garbage_address_form"), + ]) + .send()?; + + let response = client + .get(CAL_URL) + .query(&[ + ("_format", "json"), + ("type", ""), + ("start", &start_date.timestamp().to_string()), + ("end", &end_date.timestamp().to_string()), + ]) + .send()?; + let data: Vec> = response.json()?; + + let mut output: Vec = Vec::new(); + + for map in data + .iter() + .filter(|m| m.contains_key("date") && m.contains_key("label")) + { + // Because we filtered the maps in the loop, we can safely use unwrap + // here + if let Ok(date) = BasicDate::try_from(map.get("date").unwrap().as_str()) { + output.push(PickupTime::new(date, map.get("label").unwrap().to_string())) + } + } + + Ok(output) } diff --git a/src/ivago/controller/pickup_time.rs b/src/ivago/pickup_time.rs similarity index 100% rename from src/ivago/controller/pickup_time.rs rename to src/ivago/pickup_time.rs diff --git a/src/ivago/controller/street.rs b/src/ivago/street.rs similarity index 100% rename from src/ivago/controller/street.rs rename to src/ivago/street.rs diff --git a/src/lib.rs b/src/lib.rs index b1e2733..761c91b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,7 @@ #![feature(proc_macro_hygiene, decl_macro)] -#[macro_use] -extern crate rocket; - // Route modules pub mod ivago; // Helper modules -pub mod catchers; pub mod errors; From 4b51ee20caad1f1b155900f457c4728cbfe11f01 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:36:03 +0200 Subject: [PATCH 11/34] [#26] Threw in the towel for now --- src/errors.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/errors.rs b/src/errors.rs index afcf04a..730714c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -10,6 +10,8 @@ pub enum FejError { FailedRequest, } +// I'd love to move this over to the server binary, but right now, error E0117 is making that +// imopssible impl From for Status { fn from(err: FejError) -> Status { match err { From 45c4a4e257efad0edba3d2f94149f33da591c1f8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:18:50 +0200 Subject: [PATCH 12/34] [#26] Moved lib & bin to own folders; Moved server tests; wrote some readmes --- Cargo.toml | 11 ++--- README.md | 60 ++++++++++++++++----------- src/{ => fej}/errors.rs | 0 src/{ => fej}/ivago/basic_date.rs | 0 src/{ => fej}/ivago/mod.rs | 0 src/{ => fej}/ivago/pickup_time.rs | 0 src/{ => fej}/ivago/street.rs | 0 src/{ => fej}/lib.rs | 0 src/{bin => }/server/catchers.rs | 0 src/{bin => }/server/main.rs | 4 ++ src/{bin => }/server/routes/ivago.rs | 0 src/{bin => }/server/routes/mod.rs | 0 tests/ivago.rs => src/server/tests.rs | 2 +- 13 files changed, 46 insertions(+), 31 deletions(-) rename src/{ => fej}/errors.rs (100%) rename src/{ => fej}/ivago/basic_date.rs (100%) rename src/{ => fej}/ivago/mod.rs (100%) rename src/{ => fej}/ivago/pickup_time.rs (100%) rename src/{ => fej}/ivago/street.rs (100%) rename src/{ => fej}/lib.rs (100%) rename src/{bin => }/server/catchers.rs (100%) rename src/{bin => }/server/main.rs (97%) rename src/{bin => }/server/routes/ivago.rs (100%) rename src/{bin => }/server/routes/mod.rs (100%) rename tests/ivago.rs => src/server/tests.rs (93%) diff --git a/Cargo.toml b/Cargo.toml index 46b01a1..4081b85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [lib] name = "fej" -src = "src/lib.rs" +path = "src/fej/lib.rs" test = true bench = true doc = true @@ -14,10 +14,11 @@ doctest = true [[bin]] name = "server" -src = "src/bin/server/main.rs" -test = false -bench = false -doc = false +path = "src/server/main.rs" +test = true +bench = true +doc = true +doctest = true [dependencies] rocket = "0.4.7" diff --git a/README.md b/README.md index 55d11e3..15e4e03 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,21 @@ # Fej -Fej is an API written in Rust. I started this project to learn the language, -and really just have some fun. +Fej is a RESTful API that does lots of different things. It started as an +experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The folder structure follows the structure of the URLs, e.g. the route for -`/hello/world` is found in the module `src/hello`. +The bulk of the project consists of the main `fej` library. The `src` folder +contains the `lib.rs` file for this library, and all other binaries import from +this main library. -Each module contains the following base files: +All binaries can be found in [`/src/bin`](src/bin), with the biggest one being +`server`. This is what actually runs as the Rocket.rs server. The other +binaries (with more possibly coming) are utility tools that will most likely be +run as cron jobs inside the containers, e.g. scrapers. -* `mod.rs`: defines the modules' content, and contains the route definitions. - The route functions themselves only contain the functionality needed to - represent the data, not acquire it. -* `controller.rs`: this file contains the actual logic of each route. If the - logic becomes too complicated to be contained inside a single file, - `controller.rs` becomes its own module folder named `controller`. -* `tests.rs`: this contains tests for the specific module. This can also be a - module directory if need be. - -Every module has a `routes` function that returns its route macros. +Version 1.1 also introduces the use of a database, namely +[PostgreSQL 13](https://www.postgresql.org/). ## Roadmap @@ -27,19 +23,33 @@ See [roadmap.md](roadmap.md). ## Development -The entire toolchain runs on Alpine inside Docker. This makes building easier, -and (hopefully) eliminates any system-specific bugs. +To make development more consistent (and keep my computer a bit cleaner) I've +decided to run the entire toolchain using Docker, with Alpine Linux base +images. This also allows me to make really small final images. Technically, +Docker is the only dependency you need to contribute to this project +(not accounting for language servers etc.). -A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as -all the commands are documented for your understanding ;) +A [Bash script](fejctl) is provided to speed up development on the various +binaries. It aids in starting up the containers, choosing which binary to run +etc. A quick rundown of its most important features: -There's also the `build` script. This script does all the "heavy" lifting. It -chooses which Dockerfile to build according to the given arguments, and -generates tags for the images (useful when pushing releases). The Makefile is -really just a wrapper around this build script, allowing you to write -`make test` instead of `./build -m dev -a run test`. +```bash +# By default, it compiles the 'server' binary (but doesn't run it) +./fejctl -tl;dr run `make run` to run your build, and `make test` to run the tests. +# The first argument is the action you wish to perform, the second on which +# binary (not all commands need a binary as input, so they just ignore it) +# For example, this will run the binary called server +./fejctl r server + +# This runs the tests (all commands also have their full name if you want) +./fejctl t +./fejctl test + +# These attach to the two containers +./fejctl sh # Opens a new root shell inside the 'fej' container +./fejctl psql # Open a new psql shell in the fej database +``` ## Docker images diff --git a/src/errors.rs b/src/fej/errors.rs similarity index 100% rename from src/errors.rs rename to src/fej/errors.rs diff --git a/src/ivago/basic_date.rs b/src/fej/ivago/basic_date.rs similarity index 100% rename from src/ivago/basic_date.rs rename to src/fej/ivago/basic_date.rs diff --git a/src/ivago/mod.rs b/src/fej/ivago/mod.rs similarity index 100% rename from src/ivago/mod.rs rename to src/fej/ivago/mod.rs diff --git a/src/ivago/pickup_time.rs b/src/fej/ivago/pickup_time.rs similarity index 100% rename from src/ivago/pickup_time.rs rename to src/fej/ivago/pickup_time.rs diff --git a/src/ivago/street.rs b/src/fej/ivago/street.rs similarity index 100% rename from src/ivago/street.rs rename to src/fej/ivago/street.rs diff --git a/src/lib.rs b/src/fej/lib.rs similarity index 100% rename from src/lib.rs rename to src/fej/lib.rs diff --git a/src/bin/server/catchers.rs b/src/server/catchers.rs similarity index 100% rename from src/bin/server/catchers.rs rename to src/server/catchers.rs diff --git a/src/bin/server/main.rs b/src/server/main.rs similarity index 97% rename from src/bin/server/main.rs rename to src/server/main.rs index 3fde2c7..58b0300 100644 --- a/src/bin/server/main.rs +++ b/src/server/main.rs @@ -2,6 +2,10 @@ #[macro_use] extern crate rocket; + +#[cfg(test)] +mod tests; + mod catchers; mod routes; diff --git a/src/bin/server/routes/ivago.rs b/src/server/routes/ivago.rs similarity index 100% rename from src/bin/server/routes/ivago.rs rename to src/server/routes/ivago.rs diff --git a/src/bin/server/routes/mod.rs b/src/server/routes/mod.rs similarity index 100% rename from src/bin/server/routes/mod.rs rename to src/server/routes/mod.rs diff --git a/tests/ivago.rs b/src/server/tests.rs similarity index 93% rename from tests/ivago.rs rename to src/server/tests.rs index d78501a..8871762 100644 --- a/tests/ivago.rs +++ b/src/server/tests.rs @@ -3,7 +3,7 @@ use rocket::http::Status; use rocket::local::Client; fn rocket() -> rocket::Rocket { - rocket::ignite().mount("/", fej_lib::ivago::routes()) + rocket::ignite().mount("/", super::routes::ivago()) } /// Test 404 response From c2fa764e8002352728030e512f6f99b15b0014f6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:22:30 +0200 Subject: [PATCH 13/34] [#26] Updated README again --- README.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 15e4e03..e2573af 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,10 @@ experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The bulk of the project consists of the main `fej` library. The `src` folder -contains the `lib.rs` file for this library, and all other binaries import from -this main library. - -All binaries can be found in [`/src/bin`](src/bin), with the biggest one being -`server`. This is what actually runs as the Rocket.rs server. The other -binaries (with more possibly coming) are utility tools that will most likely be -run as cron jobs inside the containers, e.g. scrapers. +The `src` folder contains subfolders for the various binaries and the main +library, called `fej`. The biggest binary is called `server`, which is the +binary that actually runs the Rocket.rs web server. All the others are utility +programs, mostly consisting of scrapers for various services. Version 1.1 also introduces the use of a database, namely [PostgreSQL 13](https://www.postgresql.org/). From 0e4eb6812164b52d77c4532ef53fbf79fe15048b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:30:25 +0200 Subject: [PATCH 14/34] [#26] Updated roadmap.md --- roadmap.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roadmap.md b/roadmap.md index f03e4b5..bfdd622 100644 --- a/roadmap.md +++ b/roadmap.md @@ -69,8 +69,8 @@ setting up a database for this version anyways. ## Kissanime I like watching anime from time to time, and I've always used Kissanime for -this. However, their sites can be quite slow, and riddled with ads from time to -time. That's why I'd like to create a high-speed wrapper that extracts all the -needed info from their sites, removing the need for the user to ever actually -visit their site. The API can just act as a fast search index, complete with -indexing of the links to the videos and everything. +this. However, their sites can be quite slow, and riddled with ads. That's why +I'd like to create a high-speed wrapper that extracts all the needed info from +their sites, removing the need for the user to ever actually visit their site. +The API can just act as a fast search index, complete with indexing of the +links to the videos and everything. From 14321db3165797e6b170eace5977d7826fe6c6fd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 20:17:00 +0200 Subject: [PATCH 15/34] [#24] Added diesel_postgres_pool to deps --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4081b85..886d903 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,4 +32,4 @@ diesel = { version = "1.4.6", features = ["postgres"] } [dependencies.rocket_contrib] version = "0.4.7" default-features = false -features = ["json"] +features = ["json", "diesel_postgres_pool"] From d5a513482a41b48d0452449241b6d2244b4de3db Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 20:39:09 +0200 Subject: [PATCH 16/34] [#24] First try at compiling diesel in container (DOESN'T WORK) --- Cargo.lock | 85 +++++++++++++++++++++++++++++++++++++++ Rocket.toml | 11 +++++ docker/Dockerfile.builder | 4 +- fejctl | 1 + 4 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 Rocket.toml diff --git a/Cargo.lock b/Cargo.lock index 0006402..55062dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -331,6 +331,7 @@ dependencies = [ "byteorder", "diesel_derives", "pq-sys", + "r2d2", ] [[package]] @@ -741,6 +742,15 @@ dependencies = [ "libc", ] +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -805,6 +815,15 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +[[package]] +name = "lock_api" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.3.9" @@ -1045,6 +1064,31 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + [[package]] name = "parse-zoneinfo" version = "0.3.0" @@ -1204,6 +1248,17 @@ dependencies = [ "proc-macro2 1.0.26", ] +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log 0.4.14", + "parking_lot", + "scheduled-thread-pool", +] + [[package]] name = "rand" version = "0.8.3" @@ -1359,13 +1414,28 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7954a707f9ca18aa74ca8c1f5d1f900f52a4dceb68e96e3112143f759cfd20e" dependencies = [ + "diesel", "log 0.4.14", "notify", + "r2d2", "rocket", + "rocket_contrib_codegen", "serde", "serde_json", ] +[[package]] +name = "rocket_contrib_codegen" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30deb6dec53b91fac3538a2a3935cf13e0f462745f9f33bf27bedffbe7265b5d" +dependencies = [ + "devise", + "quote 0.6.13", + "version_check 0.9.3", + "yansi", +] + [[package]] name = "rocket_http" version = "0.4.7" @@ -1423,6 +1493,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "security-framework" version = "2.2.0" diff --git a/Rocket.toml b/Rocket.toml new file mode 100644 index 0000000..2eba66c --- /dev/null +++ b/Rocket.toml @@ -0,0 +1,11 @@ +[development] +address = "localhost" +port = 7999 +keep_alive = 5 +read_timeout = 5 +write_timeout = 5 +log = "normal" +limits = { forms = 32768 } + +[global.databases] +postgres_fej = { url = "postgres://fej:fej@fej_db:5432/fej" } diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index b968c5b..4d04e9b 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -4,13 +4,11 @@ FROM alpine:latest AS builder ENV PATH "$PATH:/root/.cargo/bin" # Needed for proper compiling of openssl-dev ENV RUSTFLAGS "-C target-feature=-crt-static" -# Otherwise, the debug build can't be used from the container -ENV ROCKET_ADDRESS "0.0.0.0" WORKDIR /usr/src/app # Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev build-base curl && \ +RUN apk update && apk add --no-cache openssl-dev build-base curl libpq && \ { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } # Copy source code over to builder diff --git a/fejctl b/fejctl index c977f79..acaaeee 100755 --- a/fejctl +++ b/fejctl @@ -52,6 +52,7 @@ function run_image() { --network fej \ -v 'fej_build-cache:/usr/src/app/target' \ -v 'fej_registry-cache:/root/.cargo/registry' \ + -v "$PWD/Rocket.toml:/usr/src/app/Rocket.toml:ro" \ "$image:dev" "$@" } From fd7a3e23317d00f655cca87b1d1ba1214969af19 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 23:20:22 +0200 Subject: [PATCH 17/34] [#24] Second attempt (failed) --- .dockerignore | 3 +++ docker/Dockerfile.builder | 5 +++-- docker/Dockerfile.dev | 4 +++- docker/entrypoint_dev.sh | 4 ++++ fejctl | 4 ++++ 5 files changed, 17 insertions(+), 3 deletions(-) create mode 100755 docker/entrypoint_dev.sh diff --git a/.dockerignore b/.dockerignore index 614c7dd..aa9bcf8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,6 @@ # Cargo files !Cargo.toml !Cargo.lock + +# Entrypoint for devop container +!docker/entrypoint_dev.sh diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index 4d04e9b..126a302 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -8,8 +8,9 @@ ENV RUSTFLAGS "-C target-feature=-crt-static" WORKDIR /usr/src/app # Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev build-base curl libpq && \ - { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } +RUN apk update && apk add --no-cache openssl-dev curl libpq libgcc musl-dev gcc && \ + { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ + rustup target add x86_64-unknown-linux-musl --toolchain nightly # Copy source code over to builder COPY Cargo.toml Cargo.lock ./ diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index d2029af..7281bc3 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -2,5 +2,7 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 -ENTRYPOINT ["cargo"] +COPY ./docker/entrypoint_dev.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] CMD ["run", "--bin", "server"] diff --git a/docker/entrypoint_dev.sh b/docker/entrypoint_dev.sh new file mode 100755 index 0000000..0838bce --- /dev/null +++ b/docker/entrypoint_dev.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh + +# All this file does is inject the target +cargo "$@" --target x86_64-unknown-linux-musl diff --git a/fejctl b/fejctl index acaaeee..ee4cf8e 100755 --- a/fejctl +++ b/fejctl @@ -28,6 +28,10 @@ function create_images() { # # $@: the arguments to pass to the image (passed as arguments to cargo) function run_image() { + docker volume create fej_build-cache + docker volume create fej_registry-cache + docker volume create fej_db-data + # Run the database image docker run --rm \ --detach \ From f381c5b9108a09c3e612b4d2ee64ace4d1e5eb0f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 10:12:37 +0200 Subject: [PATCH 18/34] [#24] Fixed libpq not wanting to link --- Rocket.toml | 4 ++-- docker/Dockerfile.builder | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Rocket.toml b/Rocket.toml index 2eba66c..20ca98c 100644 --- a/Rocket.toml +++ b/Rocket.toml @@ -1,6 +1,6 @@ [development] -address = "localhost" -port = 7999 +address = "0.0.0.0" +port = 8000 keep_alive = 5 read_timeout = 5 write_timeout = 5 diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index 126a302..dc28716 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -8,7 +8,7 @@ ENV RUSTFLAGS "-C target-feature=-crt-static" WORKDIR /usr/src/app # Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev curl libpq libgcc musl-dev gcc && \ +RUN apk update && apk add --no-cache openssl-dev curl postgresql-dev libgcc musl-dev gcc && \ { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ rustup target add x86_64-unknown-linux-musl --toolchain nightly From 5746d295564aabff2410344f278184e4bb85a28a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 10:32:16 +0200 Subject: [PATCH 19/34] [#24] Added musl target to release builder --- docker/Dockerfile.rel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index b64916a..e293ccc 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -7,7 +7,10 @@ FROM chewingbever/fej-builder:latest AS builder # NOTE: cargo install auto-appends bin to the path RUN --mount=type=cache,target=/usr/src/app/target \ --mount=type=cache,target=/root/.cargo/registry \ - cargo install --path . --root /usr/local + cargo install \ + --path . \ + --root /usr/local \ + --target x86_64-unknown-linux-musl # Now, we create the actual image From 3a490fca353fa6d955a3d44e125bd031aeaca70c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 10:52:55 +0200 Subject: [PATCH 20/34] [#24] Added db connection to server --- src/server/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/server/main.rs b/src/server/main.rs index 58b0300..9e01983 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -2,6 +2,8 @@ #[macro_use] extern crate rocket; +#[macro_use] +extern crate rocket_contrib; #[cfg(test)] mod tests; @@ -14,6 +16,7 @@ mod routes; use rocket::fairing::{Fairing, Info, Kind}; use rocket::http::Header; use rocket::{Request, Response}; +use rocket_contrib::databases::diesel; pub struct CORS; @@ -36,9 +39,14 @@ impl Fairing for CORS { } } +// This defines a connection to the database +#[database("postgres_fej")] +struct FejDbConn(diesel::PgConnection); + fn rocket() -> rocket::Rocket { rocket::ignite() .attach(CORS) + .attach(FejDbConn::fairing()) .mount("/ivago", routes::ivago()) .register(catchers![catchers::not_found]) } From 0b2b986205dbf3c4e3aa84f80ae3db61b745b9fb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 13:46:38 +0200 Subject: [PATCH 21/34] [#31] Debug build is now rootless --- docker/Dockerfile.builder | 34 ++++++++++++++++++++++++++-------- docker/Dockerfile.dev | 3 ++- docker/Dockerfile.rel | 1 + fejctl | 29 ++++++++++++++++++++--------- 4 files changed, 49 insertions(+), 18 deletions(-) diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index dc28716..4790f7c 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -1,17 +1,35 @@ +# vim: filetype=dockerfile # Our entire toolchain runs in alpine FROM alpine:latest AS builder -ENV PATH "$PATH:/root/.cargo/bin" +ENV PATH "$PATH:/app/.cargo/bin" # Needed for proper compiling of openssl-dev ENV RUSTFLAGS "-C target-feature=-crt-static" -WORKDIR /usr/src/app +# Add the build user +# Install dependencies +RUN addgroup -S builder && \ + adduser -S builder -G builder -h /app && \ + apk update && \ + apk add --no-cache \ + curl \ + gcc \ + libgcc \ + musl-dev \ + openssl-dev \ + postgresql-dev -# Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev curl postgresql-dev libgcc musl-dev gcc && \ - { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ - rustup target add x86_64-unknown-linux-musl --toolchain nightly +# Switch to the non-root user +USER builder + +WORKDIR /app + +# Install rustup in the new user's home +# Create mountpoints for volumes with correct permissions +RUN { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ + rustup target add x86_64-unknown-linux-musl --toolchain nightly && \ + mkdir -p .cargo/registry target # Copy source code over to builder -COPY Cargo.toml Cargo.lock ./ -COPY src/ ./src/ +COPY --chown=builder:builder Cargo.toml Cargo.lock ./ +COPY --chown=builder:builder src/ ./src/ diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 7281bc3..28004ac 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,8 +1,9 @@ +# vim: filetype=dockerfile FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 -COPY ./docker/entrypoint_dev.sh /entrypoint.sh +COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] CMD ["run", "--bin", "server"] diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index e293ccc..0fa71ad 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -1,3 +1,4 @@ +# vim: filetype=dockerfile FROM chewingbever/fej-builder:latest AS builder # And then finally, build the project diff --git a/fejctl b/fejctl index ee4cf8e..688395c 100755 --- a/fejctl +++ b/fejctl @@ -9,18 +9,27 @@ function create_images() { # First, we build the builder DOCKER_BUILDKIT=1 docker build \ -f docker/Dockerfile.builder \ - -t "$image-builder:latest" . + -t "$image-builder:latest" . || { + >&2 echo "Failed to build builder."; + exit 1; + } if [[ "$1" = "rel" ]]; then DOCKER_BUILDKIT=1 docker build \ -t "$image:latest" \ - -f docker/Dockerfile.rel . + -f docker/Dockerfile.rel . || { + >&2 echo "Failed to build release image."; + exit 1; + } else # Then, we create the debug image DOCKER_BUILDKIT=1 docker build \ -t "$image:dev" \ - -f docker/Dockerfile.dev . + -f docker/Dockerfile.dev . || { + >&2 echo "Failed to build debug image."; + exit 1; + } fi } @@ -28,9 +37,9 @@ function create_images() { # # $@: the arguments to pass to the image (passed as arguments to cargo) function run_image() { - docker volume create fej_build-cache - docker volume create fej_registry-cache - docker volume create fej_db-data + docker volume create fej_build-cache > /dev/null + docker volume create fej_registry-cache > /dev/null + docker volume create fej_db-data > /dev/null # Run the database image docker run --rm \ @@ -54,9 +63,9 @@ function run_image() { --name fej \ --env-file .env.container \ --network fej \ - -v 'fej_build-cache:/usr/src/app/target' \ - -v 'fej_registry-cache:/root/.cargo/registry' \ - -v "$PWD/Rocket.toml:/usr/src/app/Rocket.toml:ro" \ + -v 'fej_build-cache:/app/target' \ + -v 'fej_registry-cache:/app/.cargo/registry' \ + -v "$PWD/Rocket.toml:/app/Rocket.toml:ro" \ "$image:dev" "$@" } @@ -109,6 +118,8 @@ function publish() { exit 2 fi + create_images rel + patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` From adbe6d1fb7e566af58fb433039132ec723613b4b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 14:09:31 +0200 Subject: [PATCH 22/34] [#31] Release is now rootless (rip caching) --- .dockerignore | 3 +++ docker/Dockerfile.rel | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index aa9bcf8..41f29f5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,6 @@ # Entrypoint for devop container !docker/entrypoint_dev.sh + +# Config file +!Rocket.toml diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 0fa71ad..2d3672c 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -6,22 +6,42 @@ FROM chewingbever/fej-builder:latest AS builder # https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 # TODO add what these flags do & why they work # NOTE: cargo install auto-appends bin to the path -RUN --mount=type=cache,target=/usr/src/app/target \ - --mount=type=cache,target=/root/.cargo/registry \ - cargo install \ - --path . \ - --root /usr/local \ - --target x86_64-unknown-linux-musl + +# RUN --mount=type=cache,mode=0777,target=/app/target \ +# --mount=type=cache,mode=0777,target=/app/.cargo/registry \ + +# Buildkit cache mounts really don't like it when you're not root, +# so I guess we're building release without a cache for now +RUN cargo install \ + --path . \ + --root /app/output \ + --target x86_64-unknown-linux-musl # Now, we create the actual image FROM alpine:latest # Install some dynamic libraries needed for everything to work -RUN apk update && apk add --no-cache openssl libgcc curl +# Create -non-root user +RUN apk update && \ + apk add --no-cache \ + curl \ + libgcc \ + libpq \ + openssl && \ + addgroup -S fej && \ + adduser -S fej -G fej -h /app + +# Switch to non-root user +USER fej:fej # Copy binary over to final image -COPY --from=builder /usr/local/bin/server /usr/local/bin/server +COPY --from=builder --chown=fej:fej /app/output/bin /app/bin + +# Embed config file inside container +# The workdir is changed so that the config file is read properly +WORKDIR /app +COPY --chown=fej:fej Rocket.toml /app/Rocket.toml HEALTHCHECK \ --interval=10s \ @@ -30,4 +50,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -CMD ["/usr/local/bin/server"] +CMD ["/app/bin/server"] From 0bd09db295b96ec2dcec23fc92caf25353697f5a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 14:19:53 +0200 Subject: [PATCH 23/34] [#31] Config file is now copied into images --- docker/Dockerfile.dev | 1 + fejctl | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 28004ac..ea63e1e 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -4,6 +4,7 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh +COPY --chown=builder:builder ./Rocket.toml /app/Rocket.toml ENTRYPOINT ["/entrypoint.sh"] CMD ["run", "--bin", "server"] diff --git a/fejctl b/fejctl index 688395c..879419e 100755 --- a/fejctl +++ b/fejctl @@ -65,7 +65,6 @@ function run_image() { --network fej \ -v 'fej_build-cache:/app/target' \ -v 'fej_registry-cache:/app/.cargo/registry' \ - -v "$PWD/Rocket.toml:/app/Rocket.toml:ro" \ "$image:dev" "$@" } From 65c3d616de73b65f503a5cbc3e9808477e6f26dd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 14:43:24 +0200 Subject: [PATCH 24/34] Migrations should now run on startup --- .dockerignore | 3 +++ Cargo.toml | 1 + docker/Dockerfile.builder | 1 + src/server/main.rs | 19 ++++++++++++++++++- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 41f29f5..b7c1108 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,3 +12,6 @@ # Config file !Rocket.toml + +# Database migrations +!migrations/ diff --git a/Cargo.toml b/Cargo.toml index 886d903..9677304 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ chrono-tz = "0.5.3" regex = "1.4.5" reqwest = { version = "0.11.2", features = ["blocking", "json", "cookies"] } diesel = { version = "1.4.6", features = ["postgres"] } +diesel_migrations = "1.4.0" [dependencies.rocket_contrib] version = "0.4.7" diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index 4790f7c..b4152fa 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -33,3 +33,4 @@ RUN { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -- # Copy source code over to builder COPY --chown=builder:builder Cargo.toml Cargo.lock ./ COPY --chown=builder:builder src/ ./src/ +COPY --chown=builder:builder migrations/ ./migrations/ diff --git a/src/server/main.rs b/src/server/main.rs index 9e01983..4267776 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -4,6 +4,8 @@ extern crate rocket; #[macro_use] extern crate rocket_contrib; +#[macro_use] +extern crate diesel_migrations; #[cfg(test)] mod tests; @@ -13,9 +15,10 @@ mod routes; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs +use rocket::fairing::AdHoc; use rocket::fairing::{Fairing, Info, Kind}; use rocket::http::Header; -use rocket::{Request, Response}; +use rocket::{Request, Response, Rocket}; use rocket_contrib::databases::diesel; pub struct CORS; @@ -39,14 +42,28 @@ impl Fairing for CORS { } } +// Macro from diesel_migrations that sets up migrations +embed_migrations!(); + // This defines a connection to the database #[database("postgres_fej")] struct FejDbConn(diesel::PgConnection); +// I'd like to thank Stackoverflow for helping me with this +// https://stackoverflow.com/questions/61047355/how-to-run-diesel-migration-with-rocket-in-production +fn run_db_migrations(rocket: Rocket) -> Result { + let conn = FejDbConn::get_one(&rocket).expect("database connection"); + match embedded_migrations::run(&*conn) { + Ok(()) => Ok(rocket), + Err(e) => Err(rocket), + } +} + fn rocket() -> rocket::Rocket { rocket::ignite() .attach(CORS) .attach(FejDbConn::fairing()) + .attach(AdHoc::on_attach("Database Migrations", run_db_migrations)) .mount("/ivago", routes::ivago()) .register(catchers![catchers::not_found]) } From 00bf8501dd5f3e0e966ab9923f19e8be14bab1eb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 15:10:42 +0200 Subject: [PATCH 25/34] [closes #33] Added compose file; ./fejctl rr works now --- Rocket.toml | 5 ++++- docker/docker-compose.yml | 35 +++++++++++++++++++++++++++++++++++ fejctl | 14 +++++++++++++- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 docker/docker-compose.yml diff --git a/Rocket.toml b/Rocket.toml index 20ca98c..56291e7 100644 --- a/Rocket.toml +++ b/Rocket.toml @@ -7,5 +7,8 @@ write_timeout = 5 log = "normal" limits = { forms = 32768 } -[global.databases] +[development.databases] postgres_fej = { url = "postgres://fej:fej@fej_db:5432/fej" } + +[production.databases] +postgres_fej = { url = "postgres://fej:fej@db:5432/fej" } diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..c6d6066 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,35 @@ +version: '2.4' + +services: + app: + image: 'chewingbever/fej:latest' + restart: 'always' + + depends_on: + db: + condition: 'service_healthy' + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' + + db: + image: 'postgres:13-alpine' + restart: 'always' + + environment: + - 'POSTGRES_DB=fej' + - 'POSTGRES_USER=fej' + - 'POSTGRES_PASSWORD=fej' + healthcheck: + test: 'pg_isready -U fej' + interval: '30s' + timeout: '5s' + retries: 3 + start_period: '0s' + + volumes: + - 'db-data:/var/lib/postgresql/data' + + +volumes: + db-data: diff --git a/fejctl b/fejctl index 879419e..b60be50 100755 --- a/fejctl +++ b/fejctl @@ -105,7 +105,18 @@ function stop() { } function run_release() { - echo "Not implemented" + docker-compose \ + --file docker/docker-compose.yml \ + --project-name fej-release \ + up \ + --detach +} + +function stop_release() { + docker-compose \ + --file docker/docker-compose.yml \ + --project-name fej-release \ + down } # Tags & pushes the release version to Docker Hub @@ -155,6 +166,7 @@ function main() { r | run ) run "$bin" ;; rr | run-release ) run_release ;; s | stop ) stop ;; + sr | stop-release ) stop_release ;; # Ease of life psql ) docker exec -it fej_db psql -U fej -d fej ;; From 43e26191ccd816224ce259982dcc35a1332c40e8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 16:04:04 +0200 Subject: [PATCH 26/34] Moved development stuff to docker-compose --- Cargo.lock | 32 +++++++ docker/Dockerfile.dev | 1 - docker/docker-compose.dev.yml | 31 +++++++ docker/docker-compose.yml | 8 +- fejctl | 158 ++++++++++------------------------ 5 files changed, 112 insertions(+), 118 deletions(-) create mode 100644 docker/docker-compose.dev.yml diff --git a/Cargo.lock b/Cargo.lock index 55062dc..9c229ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,6 +345,16 @@ dependencies = [ "syn 1.0.69", ] +[[package]] +name = "diesel_migrations" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" +dependencies = [ + "migrations_internals", + "migrations_macros", +] + [[package]] name = "digest" version = "0.9.0" @@ -376,6 +386,7 @@ dependencies = [ "chrono", "chrono-tz", "diesel", + "diesel_migrations", "regex", "reqwest", "rocket", @@ -854,6 +865,27 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "migrations_internals" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" +dependencies = [ + "diesel", +] + +[[package]] +name = "migrations_macros" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" +dependencies = [ + "migrations_internals", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", +] + [[package]] name = "mime" version = "0.2.6" diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index ea63e1e..bac8b44 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -7,4 +7,3 @@ COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh COPY --chown=builder:builder ./Rocket.toml /app/Rocket.toml ENTRYPOINT ["/entrypoint.sh"] -CMD ["run", "--bin", "server"] diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 0000000..bf7fb6f --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,31 @@ +version: '2.4' +services: + app: + build: + # Make sure the build context is one directory up + context: '..' + dockerfile: './docker/Dockerfile.dev' + image: 'chewingbever/fej:dev' + restart: 'no' + + container_name: 'fej_app' + volumes: + - 'build-cache:/app/target' + - 'registry-cache:/app/.cargo/registry' + ports: + - '8000:8000' + + command: "${CMD}" + + db: + container_name: 'fej_db' + restart: 'no' + + # the devop environment exposes the database so we can use the Diesel cli + ports: + - '5432:5432' + +volumes: + build-cache: + registry-cache: + diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c6d6066..14e90d2 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,13 +2,13 @@ version: '2.4' services: app: + build: + context: '..' + dockerfile: 'docker/Dockerfile.rel' + image: 'chewingbever/fej:latest' restart: 'always' - depends_on: - db: - condition: 'service_healthy' - environment: - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' diff --git a/fejctl b/fejctl index b60be50..d55e090 100755 --- a/fejctl +++ b/fejctl @@ -2,121 +2,52 @@ image='chewingbever/fej' -# Creates the needed images +# Small wrapper around the docker-compose command # -# $1: wether to build the debug or the release image (default debug) -function create_images() { - # First, we build the builder - DOCKER_BUILDKIT=1 docker build \ - -f docker/Dockerfile.builder \ - -t "$image-builder:latest" . || { - >&2 echo "Failed to build builder."; - exit 1; - } +# Flags: +# -b: build the builder +# -r: use the release image instead +function dc() { + while getopts ":br" c; do + case $c in + b ) build_builder=1 ;; + r ) release=1 ;; + esac + done + shift $((OPTIND-1)) - if [[ "$1" = "rel" ]]; then + if [[ "$build_builder" -eq 1 ]]; then + # We always rebuild the builder before we run any compose command DOCKER_BUILDKIT=1 docker build \ - -t "$image:latest" \ - -f docker/Dockerfile.rel . || { - >&2 echo "Failed to build release image."; + -f docker/Dockerfile.builder \ + -t "$image-builder:latest" . || { + >&2 echo "Failed to build builder."; exit 1; } + fi + + if [[ "$release" -eq 1 ]]; then + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ + --file docker/docker-compose.yml \ + --project-name fej \ + "$@" else - # Then, we create the debug image - DOCKER_BUILDKIT=1 docker build \ - -t "$image:dev" \ - -f docker/Dockerfile.dev . || { - >&2 echo "Failed to build debug image."; - exit 1; - } + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ + --file docker/docker-compose.yml \ + --file docker/docker-compose.dev.yml \ + --project-name fej-dev \ + "$@" fi } # Execute the debug image (must be built first) # # $@: the arguments to pass to the image (passed as arguments to cargo) -function run_image() { - docker volume create fej_build-cache > /dev/null - docker volume create fej_registry-cache > /dev/null - docker volume create fej_db-data > /dev/null - - # Run the database image - docker run --rm \ - --detach \ - --name fej_db \ - --network fej \ - -p 5432:5432 \ - -e 'POSTGRES_DB=fej' \ - -e 'POSTGRES_USER=fej' \ - -e 'POSTGRES_PASSWORD=fej' \ - -v 'fej_db-data:/var/lib/postgresql/data' \ - postgres:13-alpine - - # Run the binary image - docker run \ - --detach \ - --rm \ - --interactive \ - --tty \ - --publish 8000:8000 \ - --name fej \ - --env-file .env.container \ - --network fej \ - -v 'fej_build-cache:/app/target' \ - -v 'fej_registry-cache:/app/.cargo/registry' \ - "$image:dev" "$@" -} - -# Attach to the fej container -function logs() { - docker logs -f fej -} - -# Builds the given binary -# -# $1: the binary to build -function build() { - create_images - run_image build --bin "$1" - logs -} - -# Runs the given binary -# -# $1: the binary to run -function run() { - create_images - run_image run --bin "$1" - logs -} - -# Runs the tests -function tests() { - create_images - run_image test --no-fail-fast - logs -} - -# Stops both containers -function stop() { - docker stop fej_db - docker stop -t 0 fej -} - -function run_release() { - docker-compose \ - --file docker/docker-compose.yml \ - --project-name fej-release \ - up \ - --detach -} - -function stop_release() { - docker-compose \ - --file docker/docker-compose.yml \ - --project-name fej-release \ - down +function dcr() { + CMD="$@" dc -b -- up \ + --build \ + --detach } # Tags & pushes the release version to Docker Hub @@ -128,7 +59,8 @@ function publish() { exit 2 fi - create_images rel + # Build the release images + dc -br build patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` @@ -159,26 +91,26 @@ function main() { case $cmd in # Building - b | build ) build "$bin" ;; - br | build-release ) create_images rel ;; + b | build ) dcr build --bin "$bin" && dc -- logs -f app ;; + br | build-release ) dc -br build ;; # Running - r | run ) run "$bin" ;; - rr | run-release ) run_release ;; - s | stop ) stop ;; - sr | stop-release ) stop_release ;; + r | run ) dcr run --bin "$bin" && dc -- logs -f app ;; + rr | run-release ) dc -br -- run --build --detach ;; + s | stop ) dc down ;; + sr | stop-release ) dc -r stop ;; # Ease of life - psql ) docker exec -it fej_db psql -U fej -d fej ;; - sh ) docker exec -it fej sh ;; + psql ) dc -- exec db psql -U fej -d fej ;; + sh ) dc -- exec app sh ;; # Misc docs ) cargo doc --no-deps ;; format ) cargo fmt ;; - l | logs ) logs ;; + l | logs ) dc -- logs -f app ;; lint ) cargo fmt -- --check ;; p | push | publish ) publish ;; - t | test ) tests ;; + t | test ) dc test --no-fail-fast ;; * ) >&2 echo "Invalid command."; exit 1 ;; esac } From 95f45cab76f6909a738dc128ed5f962b1cad16c9 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 17:46:16 +0200 Subject: [PATCH 27/34] Fixed strange dc behavior --- fejctl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fejctl b/fejctl index d55e090..7ee69df 100755 --- a/fejctl +++ b/fejctl @@ -8,7 +8,10 @@ image='chewingbever/fej' # -b: build the builder # -r: use the release image instead function dc() { + local OPTIND c build_builder release + while getopts ":br" c; do + echo "$c" case $c in b ) build_builder=1 ;; r ) release=1 ;; @@ -52,7 +55,7 @@ function dcr() { # Tags & pushes the release version to Docker Hub function publish() { - branch=`git rev-parse --abbrev-ref HEAD` + local branch=`git rev-parse --abbrev-ref HEAD` if [[ "$branch" != master ]]; then >&2 echo "You can only publish from master." @@ -62,10 +65,10 @@ function publish() { # Build the release images dc -br build - patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` - major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` - minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` - tags=("latest" "$patch_version" "$minor_version" "$major_version") + local patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` + local major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` + local minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` + local tags=("latest" "$patch_version" "$minor_version" "$major_version") for tag in "${tags[@]}"; do # Create the tag @@ -86,8 +89,8 @@ function publish() { # $2: binary to use, defaults to 'server' function main() { # Default values - cmd="${1:-build}" - bin="${2:-server}" + local cmd="${1:-build}" + local bin="${2:-server}" case $cmd in # Building From a072941ec3a3d7692dac98c115c6320803ca947e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 17:48:02 +0200 Subject: [PATCH 28/34] Removed unneeded file --- .env.container | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .env.container diff --git a/.env.container b/.env.container deleted file mode 100644 index 8b395fb..0000000 --- a/.env.container +++ /dev/null @@ -1,2 +0,0 @@ -# This file is read by the container, and therefore by fej -DATABASE_URL=postgres://fej:fej@fej_db:5432/fej From 6beca85154112bb17ad2bdb3869183fcbf6da3e1 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 18:06:31 +0200 Subject: [PATCH 29/34] Added cron as rel dependency --- docker/Dockerfile.rel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 2d3672c..90cefde 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -26,6 +26,7 @@ FROM alpine:latest RUN apk update && \ apk add --no-cache \ curl \ + cron \ libgcc \ libpq \ openssl && \ @@ -50,4 +51,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -CMD ["/app/bin/server"] +ENTRYPOINT ["/app/bin/server"] From e62cd0bb9308af8c2c5246b10cde20b544c5639f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 18:20:59 +0200 Subject: [PATCH 30/34] Crond now runs in container (#29) --- .dockerignore | 1 + docker/Dockerfile.rel | 4 ++-- docker/entrypoint.sh | 5 +++++ fejctl | 3 +-- 4 files changed, 9 insertions(+), 4 deletions(-) create mode 100755 docker/entrypoint.sh diff --git a/.dockerignore b/.dockerignore index b7c1108..29fa987 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ # Entrypoint for devop container !docker/entrypoint_dev.sh +!docker/entrypoint.sh # Config file !Rocket.toml diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 90cefde..acb3cb4 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -26,7 +26,6 @@ FROM alpine:latest RUN apk update && \ apk add --no-cache \ curl \ - cron \ libgcc \ libpq \ openssl && \ @@ -43,6 +42,7 @@ COPY --from=builder --chown=fej:fej /app/output/bin /app/bin # The workdir is changed so that the config file is read properly WORKDIR /app COPY --chown=fej:fej Rocket.toml /app/Rocket.toml +COPY ./docker/entrypoint.sh /entrypoint.sh HEALTHCHECK \ --interval=10s \ @@ -51,4 +51,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -ENTRYPOINT ["/app/bin/server"] +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..5c345c7 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +crond -b + +exec "/app/bin/server" diff --git a/fejctl b/fejctl index 7ee69df..a8fc12d 100755 --- a/fejctl +++ b/fejctl @@ -11,7 +11,6 @@ function dc() { local OPTIND c build_builder release while getopts ":br" c; do - echo "$c" case $c in b ) build_builder=1 ;; r ) release=1 ;; @@ -99,7 +98,7 @@ function main() { # Running r | run ) dcr run --bin "$bin" && dc -- logs -f app ;; - rr | run-release ) dc -br -- run --build --detach ;; + rr | run-release ) dc -br -- up --build --detach && dc -r -- logs -f app ;; s | stop ) dc down ;; sr | stop-release ) dc -r stop ;; From fee37420cda3675b0ac5ec6f0fd9cddebc1a9787 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 20:52:29 +0200 Subject: [PATCH 31/34] Added cronjob container (closes #29) --- .dockerignore | 2 ++ docker/Dockerfile.rel | 5 +++-- docker/crontab | 1 + docker/docker-compose.yml | 10 ++++++++++ docker/entrypoint.sh | 5 ----- 5 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 docker/crontab delete mode 100755 docker/entrypoint.sh diff --git a/.dockerignore b/.dockerignore index 29fa987..a96bd42 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,5 @@ # Database migrations !migrations/ + +!docker/crontab diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index acb3cb4..4f4c54d 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -20,9 +20,11 @@ RUN cargo install \ # Now, we create the actual image FROM alpine:latest +COPY ./docker/crontab /var/spool/cron/crontabs/fej # Install some dynamic libraries needed for everything to work # Create -non-root user +# Change permissions for crontab file RUN apk update && \ apk add --no-cache \ curl \ @@ -42,7 +44,6 @@ COPY --from=builder --chown=fej:fej /app/output/bin /app/bin # The workdir is changed so that the config file is read properly WORKDIR /app COPY --chown=fej:fej Rocket.toml /app/Rocket.toml -COPY ./docker/entrypoint.sh /entrypoint.sh HEALTHCHECK \ --interval=10s \ @@ -51,4 +52,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -ENTRYPOINT ["/entrypoint.sh"] +ENTRYPOINT ["/app/bin/server"] diff --git a/docker/crontab b/docker/crontab new file mode 100644 index 0000000..16fbbcd --- /dev/null +++ b/docker/crontab @@ -0,0 +1 @@ +# This'll be filled up later diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 14e90d2..4e1f9fb 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,6 +12,16 @@ services: environment: - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' + cron: + image: 'chewingbever/fej:latest' + restart: 'always' + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' + + entrypoint: 'crond -f' + user: 'root' + db: image: 'postgres:13-alpine' restart: 'always' diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index 5c345c7..0000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env sh - -crond -b - -exec "/app/bin/server" From 2c8f1ac601a55974f7fd49c9abdcec037751128c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 20:59:54 +0200 Subject: [PATCH 32/34] Removed unneeded healthcheck --- docker/docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4e1f9fb..55c5990 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -16,11 +16,13 @@ services: image: 'chewingbever/fej:latest' restart: 'always' - environment: - - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' - entrypoint: 'crond -f' user: 'root' + healthcheck: + disable: true + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' db: image: 'postgres:13-alpine' From a0e55e4830f00ef4b890afe92b16b9674e0dab9c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 21:39:29 +0200 Subject: [PATCH 33/34] Removed cron container from debug build --- docker/docker-compose.override.yml | 14 ++++++++++++++ docker/docker-compose.yml | 13 ------------- fejctl | 3 ++- 3 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 docker/docker-compose.override.yml diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml new file mode 100644 index 0000000..88c3a22 --- /dev/null +++ b/docker/docker-compose.override.yml @@ -0,0 +1,14 @@ +version: '2.4' + +services: + cron: + image: 'chewingbever/fej:latest' + restart: 'always' + + entrypoint: 'crond -f' + user: 'root' + healthcheck: + disable: true + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 55c5990..f2a19e0 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,18 +12,6 @@ services: environment: - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' - cron: - image: 'chewingbever/fej:latest' - restart: 'always' - - entrypoint: 'crond -f' - user: 'root' - healthcheck: - disable: true - - environment: - - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' - db: image: 'postgres:13-alpine' restart: 'always' @@ -42,6 +30,5 @@ services: volumes: - 'db-data:/var/lib/postgresql/data' - volumes: db-data: diff --git a/fejctl b/fejctl index a8fc12d..6701343 100755 --- a/fejctl +++ b/fejctl @@ -31,6 +31,7 @@ function dc() { if [[ "$release" -eq 1 ]]; then DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ --file docker/docker-compose.yml \ + --file docker/docker-compose.override.yml \ --project-name fej \ "$@" @@ -112,7 +113,7 @@ function main() { l | logs ) dc -- logs -f app ;; lint ) cargo fmt -- --check ;; p | push | publish ) publish ;; - t | test ) dc test --no-fail-fast ;; + t | test ) dcr -- test --no-fail-fast && dc -- logs -f app ;; * ) >&2 echo "Invalid command."; exit 1 ;; esac } From 95564e81119a4bc937476dee60521306d5348cfa Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 21:44:20 +0200 Subject: [PATCH 34/34] Updated deps; bumped version --- Cargo.lock | 40 ++++++++++++++++++++-------------------- Cargo.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c229ee..acb8e84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28" +checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec" [[package]] name = "cookie" @@ -233,7 +233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3" dependencies = [ "cookie 0.14.4", - "idna 0.2.2", + "idna 0.2.3", "log 0.4.14", "publicsuffix", "serde", @@ -381,7 +381,7 @@ dependencies = [ [[package]] name = "fej" -version = "1.0.1" +version = "1.0.2" dependencies = [ "chrono", "chrono-tz", @@ -635,9 +635,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.6" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc35c995b9d93ec174cf9a27d425c7892722101e14993cd227fdb51d70cf9589" +checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" [[package]] name = "httpdate" @@ -714,9 +714,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", @@ -997,9 +997,9 @@ dependencies = [ [[package]] name = "notify" -version = "4.0.15" +version = "4.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd" +checksum = "2599080e87c9bd051ddb11b10074f4da7b1223298df65d4c2ec5bcf309af1533" dependencies = [ "bitflags", "filetime", @@ -1166,18 +1166,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", @@ -1258,7 +1258,7 @@ version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f" dependencies = [ - "idna 0.2.2", + "idna 0.2.3", "url 2.2.1", ] @@ -1333,9 +1333,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041" dependencies = [ "bitflags", ] @@ -1997,16 +1997,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" dependencies = [ "form_urlencoded", - "idna 0.2.2", + "idna 0.2.3", "matches", "percent-encoding 2.1.0", ] [[package]] name = "vcpkg" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" +checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" [[package]] name = "version_check" diff --git a/Cargo.toml b/Cargo.toml index 9677304..4d1dd7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fej" -version = "1.0.1" +version = "1.0.2" authors = ["Jef Roosens "] edition = "2018"