From cb78af62ba0060465d07f42493d379dce2310a1b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 19:26:37 +0200 Subject: [PATCH 01/10] [#26] Moved main binary to separate location --- Cargo.toml | 6 +++--- docker/Dockerfile.dev | 2 +- docker/Dockerfile.rel | 6 +++--- src/{ => bin/server}/main.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) rename src/{ => bin/server}/main.rs (97%) diff --git a/Cargo.toml b/Cargo.toml index f359a51..46b01a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Jef Roosens "] edition = "2018" [lib] -name = "fej_lib" +name = "fej" src = "src/lib.rs" test = true bench = true @@ -13,8 +13,8 @@ doc = true doctest = true [[bin]] -name = "fej" -src = "src/main.rs" +name = "server" +src = "src/bin/server/main.rs" test = false bench = false doc = false diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 9205fd4..d2029af 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -3,4 +3,4 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 ENTRYPOINT ["cargo"] -CMD ["run"] +CMD ["run", "--bin", "server"] diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 6b4e438..b64916a 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -7,7 +7,7 @@ FROM chewingbever/fej-builder:latest AS builder # NOTE: cargo install auto-appends bin to the path RUN --mount=type=cache,target=/usr/src/app/target \ --mount=type=cache,target=/root/.cargo/registry \ - cargo install --path . --bin fej --root /usr/local + cargo install --path . --root /usr/local # Now, we create the actual image @@ -17,7 +17,7 @@ FROM alpine:latest RUN apk update && apk add --no-cache openssl libgcc curl # Copy binary over to final image -COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej +COPY --from=builder /usr/local/bin/server /usr/local/bin/server HEALTHCHECK \ --interval=10s \ @@ -26,4 +26,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -CMD ["/usr/local/bin/fej"] +CMD ["/usr/local/bin/server"] diff --git a/src/main.rs b/src/bin/server/main.rs similarity index 97% rename from src/main.rs rename to src/bin/server/main.rs index 8b8edc7..a27e00f 100644 --- a/src/main.rs +++ b/src/bin/server/main.rs @@ -1,7 +1,7 @@ #[macro_use] extern crate rocket; -use fej_lib::{catchers, ivago}; +use fej::{catchers, ivago}; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs From 21e142e9a4feaaa01fe292772b262018597c2c8e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 19:36:42 +0200 Subject: [PATCH 02/10] [#26] Added image cleaning recipe --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5d8c22e..f9c6456 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -all: debug +all: debug logs .PHONY: all # Builds the debug release inside the Alpine container. For build caching, two @@ -74,3 +74,7 @@ lint: docs: @ cargo doc --no-deps .PHONY: docs + +# This recipe removes all chewingbever/fej images from your system +clean-images: + @ docker images | grep '^chewingbever/fej' | sed 's/ \+/ /g' | cut -f3 -d' ' | xargs docker rmi From 22e9dcceaf9ce028649d70038b8e702aea571a40 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 22:46:23 +0200 Subject: [PATCH 03/10] [#26] Started writing fejctl --- fejctl | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 fejctl diff --git a/fejctl b/fejctl new file mode 100755 index 0000000..8f3eb74 --- /dev/null +++ b/fejctl @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# This script is a helpful utility for developing fej inside Docker. + +# Default values +image="chewingbever/fej" +mode="dev" # Should be either 'rel' or 'dev' +action="" +attach="--detach" +cmd="b" +bin="server" + +# Calculated variables +patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` +major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` +minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` +branch=`git rev-parse --abbrev-ref HEAD` + + + +function main() { + # Parse the flags + while getopts ":i:m:a:l" c; do + case $c in + i ) image="$OPTARG" ;; + m ) mode="$OPTARG" ;; + a ) action="$OPTARG" ;; + l ) attach="" ;; + ? ) exit 1 ;; + esac + done + shift $((OPTIND-1)) +} + +main "$@" From 0ba31bd8ba944b4bf503366654a70c16e673fff3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 23:59:31 +0200 Subject: [PATCH 04/10] [#26] Started fejctl --- fejctl | 160 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 136 insertions(+), 24 deletions(-) diff --git a/fejctl b/fejctl index 8f3eb74..17a43a0 100755 --- a/fejctl +++ b/fejctl @@ -1,35 +1,147 @@ #!/usr/bin/env bash -# This script is a helpful utility for developing fej inside Docker. +image='chewingbever/fej' -# Default values -image="chewingbever/fej" -mode="dev" # Should be either 'rel' or 'dev' -action="" -attach="--detach" -cmd="b" -bin="server" +# Creates the needed images +# +# $1: wether to build the debug or the release image (default debug) +function create_images() { + # First, we build the builder + DOCKER_BUILDKIT=1 docker build \ + -f docker/Dockerfile.builder \ + -t "$image-builder:latest" . -# Calculated variables -patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` -major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` -minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` -branch=`git rev-parse --abbrev-ref HEAD` + if [[ "$1" = "rel" ]]; then + DOCKER_BUILDKIT=1 docker build \ + -t "$image:latest" \ + -f docker/Dockerfile.rel . + else + # Then, we create the debug image + DOCKER_BUILDKIT=1 docker build \ + -t "$image:dev" \ + -f docker/Dockerfile.dev . + fi +} +# Execute the debug image (must be built first) +# +# $@: the arguments to pass to the image (passed as arguments to cargo) +function run_image() { + # Run the database image + docker run --rm \ + --detach \ + --name fej_db \ + --network fej \ + -p 5432:5432 \ + -e 'POSTGRES_DB=fej' \ + -e 'POSTGRES_USER=fej' \ + -e 'POSTGRES_PASSWORD=fej' \ + -v 'fej_db-data:/var/lib/postgresql/data' \ + postgres:13-alpine -function main() { - # Parse the flags - while getopts ":i:m:a:l" c; do - case $c in - i ) image="$OPTARG" ;; - m ) mode="$OPTARG" ;; - a ) action="$OPTARG" ;; - l ) attach="" ;; - ? ) exit 1 ;; - esac + # Run the binary image + docker run \ + --detach \ + --rm \ + --interactive \ + --tty \ + --publish 8000:8000 \ + --name fej \ + --env-file .env.container \ + --network fej \ + -v 'fej_build-cache:/usr/src/app/target' \ + -v 'fej_registry-cache:/root/.cargo/registry' \ + "$image:dev" "$@" +} + +# Attach to the fej container +function logs() { + docker logs -f fej +} + +# Builds the given binary +# +# $1: the binary to build +function build() { + create_images + run_image build --bin "$1" + logs +} + +# Runs the given binary +# +# $1: the binary to run +function run() { + create_images + run_image run --bin "$1" + logs +} + +# Runs the tests +function tests() { + create_images + run_image test --no-fail-fast + logs +} + +# Stops both containers +function stop() { + docker stop fej_db + docker stop -t 0 fej +} + +function run_release() { + echo "Not implemented" +} + +# Tags & pushes the release version to Docker Hub +function publish() { + branch=`git rev-parse --abbrev-ref HEAD` + + if [[ "$branch" != master ]]; then + >&2 echo "You can only publish from master." + exit 2 + fi + + patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` + major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` + minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` + tags=("latest" "$patch_version" "$minor_version" "$major_version") + + for tag in "${tags[@]}"; do + # Create the tag + docker tag "$image:$tags" "$image:$tag" + + # Push the tag + docker push "$image:$tag" + + # Remove the tag again, if it's not the main tag + [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" done - shift $((OPTIND-1)) + +} + +# Entrypoint to the script +# +# $1: action to perform, defaults to 'build' +# $2: binary to use, defaults to 'server' +function main() { + # Default values + cmd="${1:-build}" + bin="${2:-server}" + + case $cmd in + b | build ) build "$bin" ;; + br | build-release ) create_images rel ;; + r | run ) run "$bin" ;; + rr | run-release ) run_release ;; + p | push | publish ) publish ;; + t | test ) tests ;; + s | stop ) stop ;; + p | push | publish ) publish ;; + * ) >&2 echo "Invalid command."; exit 1 ;; + esac } main "$@" From 0828dd36d6b26010bb3f8b783eccf9f4fdd736b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:06:48 +0200 Subject: [PATCH 05/10] [#26] fejctl now fully replace Makefile & build --- .hooks/pre-commit | 2 +- Makefile | 80 ---------------------------------- build | 108 ---------------------------------------------- fejctl | 16 ++++++- 4 files changed, 15 insertions(+), 191 deletions(-) delete mode 100644 Makefile delete mode 100755 build diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 709fb95..6947bc7 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This hook lints the code, and if we're on develop or master, also forces the tests to pass. -make lint &> /dev/null 2>&1 || { +./fejctl lint &> /dev/null 2>&1 || { >&2 echo "Format check failed, use 'make lint' for more information."; exit 1; } diff --git a/Makefile b/Makefile deleted file mode 100644 index f9c6456..0000000 --- a/Makefile +++ /dev/null @@ -1,80 +0,0 @@ -all: debug logs -.PHONY: all - -# Builds the debug release inside the Alpine container. For build caching, two -# volumes are used named `fej_build-cache` and `fej_registry-cache`. These -# images are automatically created for you if they don't exist. If you -# encounter any strange build errors, you can try removing this volumes to -# start a completely fresh build. -debug: - @ ./build -m dev -a run build -.PHONY: debug - -# Builds the release version. In contrary to the debug version, this build -# doesn't use volumes for caching, as this would require the build to happen -# during runtime instead of during the building of the image. Instead, it uses -# the new `--mount` feature from Buildkit. This does mean that only very recent -# Docker engines can build the release version (in my case, at the time of -# writing this, 20.10.5). -release: - @ ./build -m rel -.PHONY: release - -# This builds the release version, and pushes all relevant tags to my Docker -# Hub repository, namely chewingbever/fej -push: - @ ./build -m rel -a push -.PHONY: push - -# This builds the debug release, and runs it detached. The reason we detach the -# container is because Rocket has a tendency to ignore ctlr-c when inside a -# container, which gets annoying really fast. -run: - @ ./build -m dev -a run -.PHONY: run - -# As a workaround, we just have a stop command that stops the container. -stop: - @ docker stop -t 2 fej - @ docker stop fej_db -.PHONY: stop - -# This attaches to the running container, essentially giving the same result as -# just running `cargo run` locally. -logs: - @ docker logs -f fej -.PHONY: logs - -# This just starts up a shell inside the fej container -sh: - @ docker exec -it fej sh -.PHONY: sh - -# Starts a psql session in the database container -dbsh: - @ docker exec -it fej_db psql -U fej -d fej - -# Builds the debug version, and runs the tests (but doesn't detach). -test: - @ ./build -m dev -a run -l -- test --no-fail-fast -.PHONY: test - -# Runs the cargo code formatter on your code. -format: - @ cargo fmt -.PHONY: format - -# Lints your code. This also gets run in the pre-commit hook, basically -# preventing you from committing badly-formatted code. -lint: - @ cargo fmt -- --check -.PHONY: lint - -# This builds the documentation for the project, excluding the documentation. -docs: - @ cargo doc --no-deps -.PHONY: docs - -# This recipe removes all chewingbever/fej images from your system -clean-images: - @ docker images | grep '^chewingbever/fej' | sed 's/ \+/ /g' | cut -f3 -d' ' | xargs docker rmi diff --git a/build b/build deleted file mode 100755 index 271a3d9..0000000 --- a/build +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash - -image="chewingbever/fej" -# Should be either dev or rel -mode="dev" -action="" -attach="--detach" - -while getopts ":i:m:a:l" c; do - case $c in - i ) image="$OPTARG" ;; - m ) mode="$OPTARG" ;; - a ) action="$OPTARG" ;; - l ) attach="" ;; - ? ) exit 1 ;; - esac -done -shift $((OPTIND-1)) - -# Extract current version from Cargo.toml & get current branch -patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` -major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` -minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` -branch=`git rev-parse --abbrev-ref HEAD` - -if [[ "$branch" = "master" ]]; then - tags=("$patch_version" "$minor_version" "$major_version" "latest") - -elif [[ "$branch" = "develop" ]]; then - tags=("$patch_version-dev" "$minor_version-dev" "$major_version-dev" "dev") - -else - tags=("$branch") - -fi - -# First, we build the builder -DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" . - -# Run the actual build command -if [ "$mode" = "rel" ]; then - DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel . - -elif [[ "$mode" = "dev" ]]; then - DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev . - -else - >&2 echo "Invalid mode." - exit 1 - -fi - -if [[ "$action" = push ]]; then - [[ "$branch" =~ ^develop|master$ ]] || { - >&2 echo "You can only push from develop or master." - exit 2 - } - - [[ "$mode" = "rel" ]] || { - >&2 echo "You can only push release builds." - exit 3 - } - - for tag in "${tags[@]}"; do - # Create the tag - docker tag "$image:$tags" "$image:$tag" - - # Push the tag - docker push "$image:$tag" - - # Remove the tag again, if it's not the main tag - [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" - done - -elif [[ "$action" = run ]]; then - # Create the network & start the database container - docker network create fej - docker volume create fej_db-data - - docker run --rm \ - --detach \ - --name fej_db \ - --network fej \ - -p 5432:5432 \ - -e "POSTGRES_DB=fej" \ - -e "POSTGRES_USER=fej" \ - -e "POSTGRES_PASSWORD=fej" \ - -v 'fej_db-data:/var/lib/postgresql/data' \ - postgres:13-alpine - - if [[ "$mode" = "dev" ]]; then - # Create caching volumes if needed (they need to be named) - docker volume create fej_build-cache - docker volume create fej_registry-cache - - flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry" - fi - - docker run $attach $flags \ - --rm \ - --interactive \ - --tty \ - --publish 8000:8000 \ - --name fej \ - --env-file .env.container \ - --network fej \ - "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" -fi diff --git a/fejctl b/fejctl index 17a43a0..c977f79 100755 --- a/fejctl +++ b/fejctl @@ -132,14 +132,26 @@ function main() { bin="${2:-server}" case $cmd in + # Building b | build ) build "$bin" ;; br | build-release ) create_images rel ;; + + # Running r | run ) run "$bin" ;; rr | run-release ) run_release ;; + s | stop ) stop ;; + + # Ease of life + psql ) docker exec -it fej_db psql -U fej -d fej ;; + sh ) docker exec -it fej sh ;; + + # Misc + docs ) cargo doc --no-deps ;; + format ) cargo fmt ;; + l | logs ) logs ;; + lint ) cargo fmt -- --check ;; p | push | publish ) publish ;; t | test ) tests ;; - s | stop ) stop ;; - p | push | publish ) publish ;; * ) >&2 echo "Invalid command."; exit 1 ;; esac } From d19fe5c42e29af96e3be02662ca6e507b90f19b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:32:03 +0200 Subject: [PATCH 06/10] [#26] Moved all routing to server binary --- .hooks/pre-commit | 2 +- src/{ => bin/server}/catchers.rs | 0 src/bin/server/main.rs | 8 +- src/bin/server/routes/ivago.rs | 38 ++++++++ src/bin/server/routes/mod.rs | 5 + src/ivago/{controller => }/basic_date.rs | 0 src/ivago/controller/mod.rs | 95 ------------------ src/ivago/mod.rs | 112 ++++++++++++++++------ src/ivago/{controller => }/pickup_time.rs | 0 src/ivago/{controller => }/street.rs | 0 src/lib.rs | 4 - 11 files changed, 130 insertions(+), 134 deletions(-) rename src/{ => bin/server}/catchers.rs (100%) create mode 100644 src/bin/server/routes/ivago.rs create mode 100644 src/bin/server/routes/mod.rs rename src/ivago/{controller => }/basic_date.rs (100%) delete mode 100644 src/ivago/controller/mod.rs rename src/ivago/{controller => }/pickup_time.rs (100%) rename src/ivago/{controller => }/street.rs (100%) diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 6947bc7..fe55771 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -2,7 +2,7 @@ # This hook lints the code, and if we're on develop or master, also forces the tests to pass. ./fejctl lint &> /dev/null 2>&1 || { - >&2 echo "Format check failed, use 'make lint' for more information."; + >&2 echo "Format check failed, use './fejctl lint' for more information."; exit 1; } diff --git a/src/catchers.rs b/src/bin/server/catchers.rs similarity index 100% rename from src/catchers.rs rename to src/bin/server/catchers.rs diff --git a/src/bin/server/main.rs b/src/bin/server/main.rs index a27e00f..3fde2c7 100644 --- a/src/bin/server/main.rs +++ b/src/bin/server/main.rs @@ -1,7 +1,9 @@ +#![feature(proc_macro_hygiene, decl_macro)] + #[macro_use] extern crate rocket; - -use fej::{catchers, ivago}; +mod catchers; +mod routes; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs @@ -33,7 +35,7 @@ impl Fairing for CORS { fn rocket() -> rocket::Rocket { rocket::ignite() .attach(CORS) - .mount("/ivago", ivago::routes()) + .mount("/ivago", routes::ivago()) .register(catchers![catchers::not_found]) } diff --git a/src/bin/server/routes/ivago.rs b/src/bin/server/routes/ivago.rs new file mode 100644 index 0000000..d8318bd --- /dev/null +++ b/src/bin/server/routes/ivago.rs @@ -0,0 +1,38 @@ +use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street}; +use rocket::http::Status; +use rocket_contrib::json::Json; + +/// This route handles the Ivago search endpoint. It returns a list of streets, +/// consisting of a street name & a city. +/// +/// # Arguments +/// +/// * `search_term` - Search term to use to look for streets +#[get("/search?")] +pub fn route_search_streets(q: String) -> Result>, Status> { + Ok(Json(search_streets(q.as_str())?)) +} + +/// Handles returning of pickup times for a specific address. It returns a list +/// of pickup times, containing a date and a description of the trash type. +/// +/// # Arguments +/// +/// * `street` - Street to look up +/// * `number` - House number in the given street +/// * `start_date` - Earliest date that can be returned +/// * `end_date` - Latest date that can be returned +#[get("/?&&&")] +pub fn route_get_pickup_times( + street: Street, + number: u32, + start_date: BasicDate, + end_date: BasicDate, +) -> Result>, Status> { + Ok(Json(get_pickup_times( + &street, + &number, + &start_date.0, + &end_date.0, + )?)) +} diff --git a/src/bin/server/routes/mod.rs b/src/bin/server/routes/mod.rs new file mode 100644 index 0000000..449f82f --- /dev/null +++ b/src/bin/server/routes/mod.rs @@ -0,0 +1,5 @@ +mod ivago; + +pub fn ivago() -> Vec { + routes![ivago::route_search_streets, ivago::route_get_pickup_times] +} diff --git a/src/ivago/controller/basic_date.rs b/src/ivago/basic_date.rs similarity index 100% rename from src/ivago/controller/basic_date.rs rename to src/ivago/basic_date.rs diff --git a/src/ivago/controller/mod.rs b/src/ivago/controller/mod.rs deleted file mode 100644 index 7d3a9f9..0000000 --- a/src/ivago/controller/mod.rs +++ /dev/null @@ -1,95 +0,0 @@ -use crate::errors::FejError; -use chrono::DateTime; -use chrono_tz::Tz; -use reqwest::blocking as reqwest; -use std::collections::HashMap; -use std::convert::{From, TryFrom}; - -mod basic_date; -mod pickup_time; -mod street; - -pub use basic_date::BasicDate; -pub use pickup_time::PickupTime; -pub use street::Street; - -/// Endpoint for the search feature -const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; -/// Endpoint for populating the initial cookies -const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; -/// Endpoint for the actual calendar output -const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; - -/// Searches the Ivago API for streets in the given city. -/// -/// # Arguments -/// -/// * `search_term` - Search term to use to look for streets -pub fn search_streets(search_term: &str) -> Result, FejError> { - let client = reqwest::Client::new(); - let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; - let data: Vec> = response.json()?; - - // This is pretty cool, filter_map first does get() on all the maps, and - // then filters out any None values - // Then, we do the same thing for streets - Ok(data - .iter() - .filter_map(|m| m.get("value")) - .filter_map(|v| Street::try_from(v.as_str()).ok()) - .collect()) -} - -/// Returns the pickup times for the various trash types. -/// -/// # Arguments -/// -/// * `street` - Street to look up -/// * `number` - House number in given street -/// * `start_date` - Earliest date for the results -/// * `end_date` - Latest date for the results -pub fn get_pickup_times( - street: &Street, - number: &u32, - start_date: &DateTime, - end_date: &DateTime, -) -> Result, FejError> { - let client = reqwest::Client::builder().cookie_store(true).build()?; - - // This populates the cookies with the necessary values - client - .post(BASE_URL) - .form(&[ - ("garbage_type", ""), - ("ivago_street", &String::from(street)), - ("number", &number.to_string()), - ("form_id", "garbage_address_form"), - ]) - .send()?; - - let response = client - .get(CAL_URL) - .query(&[ - ("_format", "json"), - ("type", ""), - ("start", &start_date.timestamp().to_string()), - ("end", &end_date.timestamp().to_string()), - ]) - .send()?; - let data: Vec> = response.json()?; - - let mut output: Vec = Vec::new(); - - for map in data - .iter() - .filter(|m| m.contains_key("date") && m.contains_key("label")) - { - // Because we filtered the maps in the loop, we can safely use unwrap - // here - if let Ok(date) = BasicDate::try_from(map.get("date").unwrap().as_str()) { - output.push(PickupTime::new(date, map.get("label").unwrap().to_string())) - } - } - - Ok(output) -} diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index 627e89b..7d3a9f9 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -1,45 +1,95 @@ -mod controller; +use crate::errors::FejError; +use chrono::DateTime; +use chrono_tz::Tz; +use reqwest::blocking as reqwest; +use std::collections::HashMap; +use std::convert::{From, TryFrom}; -use controller::{get_pickup_times, search_streets}; -use controller::{BasicDate, PickupTime, Street}; -use rocket::http::Status; -use rocket_contrib::json::Json; +mod basic_date; +mod pickup_time; +mod street; -pub fn routes() -> Vec { - routes![route_search_streets, route_get_pickup_times] -} +pub use basic_date::BasicDate; +pub use pickup_time::PickupTime; +pub use street::Street; -/// This route handles the Ivago search endpoint. It returns a list of streets, -/// consisting of a street name & a city. +/// Endpoint for the search feature +const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; +/// Endpoint for populating the initial cookies +const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; +/// Endpoint for the actual calendar output +const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; + +/// Searches the Ivago API for streets in the given city. /// /// # Arguments /// /// * `search_term` - Search term to use to look for streets -#[get("/search?")] -pub fn route_search_streets(q: String) -> Result>, Status> { - Ok(Json(search_streets(q.as_str())?)) +pub fn search_streets(search_term: &str) -> Result, FejError> { + let client = reqwest::Client::new(); + let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; + let data: Vec> = response.json()?; + + // This is pretty cool, filter_map first does get() on all the maps, and + // then filters out any None values + // Then, we do the same thing for streets + Ok(data + .iter() + .filter_map(|m| m.get("value")) + .filter_map(|v| Street::try_from(v.as_str()).ok()) + .collect()) } -/// Handles returning of pickup times for a specific address. It returns a list -/// of pickup times, containing a date and a description of the trash type. +/// Returns the pickup times for the various trash types. /// /// # Arguments /// /// * `street` - Street to look up -/// * `number` - House number in the given street -/// * `start_date` - Earliest date that can be returned -/// * `end_date` - Latest date that can be returned -#[get("/?&&&")] -pub fn route_get_pickup_times( - street: Street, - number: u32, - start_date: BasicDate, - end_date: BasicDate, -) -> Result>, Status> { - Ok(Json(get_pickup_times( - &street, - &number, - &start_date.0, - &end_date.0, - )?)) +/// * `number` - House number in given street +/// * `start_date` - Earliest date for the results +/// * `end_date` - Latest date for the results +pub fn get_pickup_times( + street: &Street, + number: &u32, + start_date: &DateTime, + end_date: &DateTime, +) -> Result, FejError> { + let client = reqwest::Client::builder().cookie_store(true).build()?; + + // This populates the cookies with the necessary values + client + .post(BASE_URL) + .form(&[ + ("garbage_type", ""), + ("ivago_street", &String::from(street)), + ("number", &number.to_string()), + ("form_id", "garbage_address_form"), + ]) + .send()?; + + let response = client + .get(CAL_URL) + .query(&[ + ("_format", "json"), + ("type", ""), + ("start", &start_date.timestamp().to_string()), + ("end", &end_date.timestamp().to_string()), + ]) + .send()?; + let data: Vec> = response.json()?; + + let mut output: Vec = Vec::new(); + + for map in data + .iter() + .filter(|m| m.contains_key("date") && m.contains_key("label")) + { + // Because we filtered the maps in the loop, we can safely use unwrap + // here + if let Ok(date) = BasicDate::try_from(map.get("date").unwrap().as_str()) { + output.push(PickupTime::new(date, map.get("label").unwrap().to_string())) + } + } + + Ok(output) } diff --git a/src/ivago/controller/pickup_time.rs b/src/ivago/pickup_time.rs similarity index 100% rename from src/ivago/controller/pickup_time.rs rename to src/ivago/pickup_time.rs diff --git a/src/ivago/controller/street.rs b/src/ivago/street.rs similarity index 100% rename from src/ivago/controller/street.rs rename to src/ivago/street.rs diff --git a/src/lib.rs b/src/lib.rs index b1e2733..761c91b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,7 @@ #![feature(proc_macro_hygiene, decl_macro)] -#[macro_use] -extern crate rocket; - // Route modules pub mod ivago; // Helper modules -pub mod catchers; pub mod errors; From 4b51ee20caad1f1b155900f457c4728cbfe11f01 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:36:03 +0200 Subject: [PATCH 07/10] [#26] Threw in the towel for now --- src/errors.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/errors.rs b/src/errors.rs index afcf04a..730714c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -10,6 +10,8 @@ pub enum FejError { FailedRequest, } +// I'd love to move this over to the server binary, but right now, error E0117 is making that +// imopssible impl From for Status { fn from(err: FejError) -> Status { match err { From 45c4a4e257efad0edba3d2f94149f33da591c1f8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:18:50 +0200 Subject: [PATCH 08/10] [#26] Moved lib & bin to own folders; Moved server tests; wrote some readmes --- Cargo.toml | 11 ++--- README.md | 60 ++++++++++++++++----------- src/{ => fej}/errors.rs | 0 src/{ => fej}/ivago/basic_date.rs | 0 src/{ => fej}/ivago/mod.rs | 0 src/{ => fej}/ivago/pickup_time.rs | 0 src/{ => fej}/ivago/street.rs | 0 src/{ => fej}/lib.rs | 0 src/{bin => }/server/catchers.rs | 0 src/{bin => }/server/main.rs | 4 ++ src/{bin => }/server/routes/ivago.rs | 0 src/{bin => }/server/routes/mod.rs | 0 tests/ivago.rs => src/server/tests.rs | 2 +- 13 files changed, 46 insertions(+), 31 deletions(-) rename src/{ => fej}/errors.rs (100%) rename src/{ => fej}/ivago/basic_date.rs (100%) rename src/{ => fej}/ivago/mod.rs (100%) rename src/{ => fej}/ivago/pickup_time.rs (100%) rename src/{ => fej}/ivago/street.rs (100%) rename src/{ => fej}/lib.rs (100%) rename src/{bin => }/server/catchers.rs (100%) rename src/{bin => }/server/main.rs (97%) rename src/{bin => }/server/routes/ivago.rs (100%) rename src/{bin => }/server/routes/mod.rs (100%) rename tests/ivago.rs => src/server/tests.rs (93%) diff --git a/Cargo.toml b/Cargo.toml index 46b01a1..4081b85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [lib] name = "fej" -src = "src/lib.rs" +path = "src/fej/lib.rs" test = true bench = true doc = true @@ -14,10 +14,11 @@ doctest = true [[bin]] name = "server" -src = "src/bin/server/main.rs" -test = false -bench = false -doc = false +path = "src/server/main.rs" +test = true +bench = true +doc = true +doctest = true [dependencies] rocket = "0.4.7" diff --git a/README.md b/README.md index 55d11e3..15e4e03 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,21 @@ # Fej -Fej is an API written in Rust. I started this project to learn the language, -and really just have some fun. +Fej is a RESTful API that does lots of different things. It started as an +experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The folder structure follows the structure of the URLs, e.g. the route for -`/hello/world` is found in the module `src/hello`. +The bulk of the project consists of the main `fej` library. The `src` folder +contains the `lib.rs` file for this library, and all other binaries import from +this main library. -Each module contains the following base files: +All binaries can be found in [`/src/bin`](src/bin), with the biggest one being +`server`. This is what actually runs as the Rocket.rs server. The other +binaries (with more possibly coming) are utility tools that will most likely be +run as cron jobs inside the containers, e.g. scrapers. -* `mod.rs`: defines the modules' content, and contains the route definitions. - The route functions themselves only contain the functionality needed to - represent the data, not acquire it. -* `controller.rs`: this file contains the actual logic of each route. If the - logic becomes too complicated to be contained inside a single file, - `controller.rs` becomes its own module folder named `controller`. -* `tests.rs`: this contains tests for the specific module. This can also be a - module directory if need be. - -Every module has a `routes` function that returns its route macros. +Version 1.1 also introduces the use of a database, namely +[PostgreSQL 13](https://www.postgresql.org/). ## Roadmap @@ -27,19 +23,33 @@ See [roadmap.md](roadmap.md). ## Development -The entire toolchain runs on Alpine inside Docker. This makes building easier, -and (hopefully) eliminates any system-specific bugs. +To make development more consistent (and keep my computer a bit cleaner) I've +decided to run the entire toolchain using Docker, with Alpine Linux base +images. This also allows me to make really small final images. Technically, +Docker is the only dependency you need to contribute to this project +(not accounting for language servers etc.). -A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as -all the commands are documented for your understanding ;) +A [Bash script](fejctl) is provided to speed up development on the various +binaries. It aids in starting up the containers, choosing which binary to run +etc. A quick rundown of its most important features: -There's also the `build` script. This script does all the "heavy" lifting. It -chooses which Dockerfile to build according to the given arguments, and -generates tags for the images (useful when pushing releases). The Makefile is -really just a wrapper around this build script, allowing you to write -`make test` instead of `./build -m dev -a run test`. +```bash +# By default, it compiles the 'server' binary (but doesn't run it) +./fejctl -tl;dr run `make run` to run your build, and `make test` to run the tests. +# The first argument is the action you wish to perform, the second on which +# binary (not all commands need a binary as input, so they just ignore it) +# For example, this will run the binary called server +./fejctl r server + +# This runs the tests (all commands also have their full name if you want) +./fejctl t +./fejctl test + +# These attach to the two containers +./fejctl sh # Opens a new root shell inside the 'fej' container +./fejctl psql # Open a new psql shell in the fej database +``` ## Docker images diff --git a/src/errors.rs b/src/fej/errors.rs similarity index 100% rename from src/errors.rs rename to src/fej/errors.rs diff --git a/src/ivago/basic_date.rs b/src/fej/ivago/basic_date.rs similarity index 100% rename from src/ivago/basic_date.rs rename to src/fej/ivago/basic_date.rs diff --git a/src/ivago/mod.rs b/src/fej/ivago/mod.rs similarity index 100% rename from src/ivago/mod.rs rename to src/fej/ivago/mod.rs diff --git a/src/ivago/pickup_time.rs b/src/fej/ivago/pickup_time.rs similarity index 100% rename from src/ivago/pickup_time.rs rename to src/fej/ivago/pickup_time.rs diff --git a/src/ivago/street.rs b/src/fej/ivago/street.rs similarity index 100% rename from src/ivago/street.rs rename to src/fej/ivago/street.rs diff --git a/src/lib.rs b/src/fej/lib.rs similarity index 100% rename from src/lib.rs rename to src/fej/lib.rs diff --git a/src/bin/server/catchers.rs b/src/server/catchers.rs similarity index 100% rename from src/bin/server/catchers.rs rename to src/server/catchers.rs diff --git a/src/bin/server/main.rs b/src/server/main.rs similarity index 97% rename from src/bin/server/main.rs rename to src/server/main.rs index 3fde2c7..58b0300 100644 --- a/src/bin/server/main.rs +++ b/src/server/main.rs @@ -2,6 +2,10 @@ #[macro_use] extern crate rocket; + +#[cfg(test)] +mod tests; + mod catchers; mod routes; diff --git a/src/bin/server/routes/ivago.rs b/src/server/routes/ivago.rs similarity index 100% rename from src/bin/server/routes/ivago.rs rename to src/server/routes/ivago.rs diff --git a/src/bin/server/routes/mod.rs b/src/server/routes/mod.rs similarity index 100% rename from src/bin/server/routes/mod.rs rename to src/server/routes/mod.rs diff --git a/tests/ivago.rs b/src/server/tests.rs similarity index 93% rename from tests/ivago.rs rename to src/server/tests.rs index d78501a..8871762 100644 --- a/tests/ivago.rs +++ b/src/server/tests.rs @@ -3,7 +3,7 @@ use rocket::http::Status; use rocket::local::Client; fn rocket() -> rocket::Rocket { - rocket::ignite().mount("/", fej_lib::ivago::routes()) + rocket::ignite().mount("/", super::routes::ivago()) } /// Test 404 response From c2fa764e8002352728030e512f6f99b15b0014f6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:22:30 +0200 Subject: [PATCH 09/10] [#26] Updated README again --- README.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 15e4e03..e2573af 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,10 @@ experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The bulk of the project consists of the main `fej` library. The `src` folder -contains the `lib.rs` file for this library, and all other binaries import from -this main library. - -All binaries can be found in [`/src/bin`](src/bin), with the biggest one being -`server`. This is what actually runs as the Rocket.rs server. The other -binaries (with more possibly coming) are utility tools that will most likely be -run as cron jobs inside the containers, e.g. scrapers. +The `src` folder contains subfolders for the various binaries and the main +library, called `fej`. The biggest binary is called `server`, which is the +binary that actually runs the Rocket.rs web server. All the others are utility +programs, mostly consisting of scrapers for various services. Version 1.1 also introduces the use of a database, namely [PostgreSQL 13](https://www.postgresql.org/). From 0e4eb6812164b52d77c4532ef53fbf79fe15048b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:30:25 +0200 Subject: [PATCH 10/10] [#26] Updated roadmap.md --- roadmap.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roadmap.md b/roadmap.md index f03e4b5..bfdd622 100644 --- a/roadmap.md +++ b/roadmap.md @@ -69,8 +69,8 @@ setting up a database for this version anyways. ## Kissanime I like watching anime from time to time, and I've always used Kissanime for -this. However, their sites can be quite slow, and riddled with ads from time to -time. That's why I'd like to create a high-speed wrapper that extracts all the -needed info from their sites, removing the need for the user to ever actually -visit their site. The API can just act as a fast search index, complete with -indexing of the links to the videos and everything. +this. However, their sites can be quite slow, and riddled with ads. That's why +I'd like to create a high-speed wrapper that extracts all the needed info from +their sites, removing the need for the user to ever actually visit their site. +The API can just act as a fast search index, complete with indexing of the +links to the videos and everything.