From d19fe5c42e29af96e3be02662ca6e507b90f19b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:32:03 +0200 Subject: [PATCH 1/5] [#26] Moved all routing to server binary --- .hooks/pre-commit | 2 +- src/{ => bin/server}/catchers.rs | 0 src/bin/server/main.rs | 8 +- src/bin/server/routes/ivago.rs | 38 ++++++++ src/bin/server/routes/mod.rs | 5 + src/ivago/{controller => }/basic_date.rs | 0 src/ivago/controller/mod.rs | 95 ------------------ src/ivago/mod.rs | 112 ++++++++++++++++------ src/ivago/{controller => }/pickup_time.rs | 0 src/ivago/{controller => }/street.rs | 0 src/lib.rs | 4 - 11 files changed, 130 insertions(+), 134 deletions(-) rename src/{ => bin/server}/catchers.rs (100%) create mode 100644 src/bin/server/routes/ivago.rs create mode 100644 src/bin/server/routes/mod.rs rename src/ivago/{controller => }/basic_date.rs (100%) delete mode 100644 src/ivago/controller/mod.rs rename src/ivago/{controller => }/pickup_time.rs (100%) rename src/ivago/{controller => }/street.rs (100%) diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 6947bc7..fe55771 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -2,7 +2,7 @@ # This hook lints the code, and if we're on develop or master, also forces the tests to pass. ./fejctl lint &> /dev/null 2>&1 || { - >&2 echo "Format check failed, use 'make lint' for more information."; + >&2 echo "Format check failed, use './fejctl lint' for more information."; exit 1; } diff --git a/src/catchers.rs b/src/bin/server/catchers.rs similarity index 100% rename from src/catchers.rs rename to src/bin/server/catchers.rs diff --git a/src/bin/server/main.rs b/src/bin/server/main.rs index a27e00f..3fde2c7 100644 --- a/src/bin/server/main.rs +++ b/src/bin/server/main.rs @@ -1,7 +1,9 @@ +#![feature(proc_macro_hygiene, decl_macro)] + #[macro_use] extern crate rocket; - -use fej::{catchers, ivago}; +mod catchers; +mod routes; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs @@ -33,7 +35,7 @@ impl Fairing for CORS { fn rocket() -> rocket::Rocket { rocket::ignite() .attach(CORS) - .mount("/ivago", ivago::routes()) + .mount("/ivago", routes::ivago()) .register(catchers![catchers::not_found]) } diff --git a/src/bin/server/routes/ivago.rs b/src/bin/server/routes/ivago.rs new file mode 100644 index 0000000..d8318bd --- /dev/null +++ b/src/bin/server/routes/ivago.rs @@ -0,0 +1,38 @@ +use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street}; +use rocket::http::Status; +use rocket_contrib::json::Json; + +/// This route handles the Ivago search endpoint. It returns a list of streets, +/// consisting of a street name & a city. +/// +/// # Arguments +/// +/// * `search_term` - Search term to use to look for streets +#[get("/search?")] +pub fn route_search_streets(q: String) -> Result>, Status> { + Ok(Json(search_streets(q.as_str())?)) +} + +/// Handles returning of pickup times for a specific address. It returns a list +/// of pickup times, containing a date and a description of the trash type. +/// +/// # Arguments +/// +/// * `street` - Street to look up +/// * `number` - House number in the given street +/// * `start_date` - Earliest date that can be returned +/// * `end_date` - Latest date that can be returned +#[get("/?&&&")] +pub fn route_get_pickup_times( + street: Street, + number: u32, + start_date: BasicDate, + end_date: BasicDate, +) -> Result>, Status> { + Ok(Json(get_pickup_times( + &street, + &number, + &start_date.0, + &end_date.0, + )?)) +} diff --git a/src/bin/server/routes/mod.rs b/src/bin/server/routes/mod.rs new file mode 100644 index 0000000..449f82f --- /dev/null +++ b/src/bin/server/routes/mod.rs @@ -0,0 +1,5 @@ +mod ivago; + +pub fn ivago() -> Vec { + routes![ivago::route_search_streets, ivago::route_get_pickup_times] +} diff --git a/src/ivago/controller/basic_date.rs b/src/ivago/basic_date.rs similarity index 100% rename from src/ivago/controller/basic_date.rs rename to src/ivago/basic_date.rs diff --git a/src/ivago/controller/mod.rs b/src/ivago/controller/mod.rs deleted file mode 100644 index 7d3a9f9..0000000 --- a/src/ivago/controller/mod.rs +++ /dev/null @@ -1,95 +0,0 @@ -use crate::errors::FejError; -use chrono::DateTime; -use chrono_tz::Tz; -use reqwest::blocking as reqwest; -use std::collections::HashMap; -use std::convert::{From, TryFrom}; - -mod basic_date; -mod pickup_time; -mod street; - -pub use basic_date::BasicDate; -pub use pickup_time::PickupTime; -pub use street::Street; - -/// Endpoint for the search feature -const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; -/// Endpoint for populating the initial cookies -const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; -/// Endpoint for the actual calendar output -const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; - -/// Searches the Ivago API for streets in the given city. -/// -/// # Arguments -/// -/// * `search_term` - Search term to use to look for streets -pub fn search_streets(search_term: &str) -> Result, FejError> { - let client = reqwest::Client::new(); - let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; - let data: Vec> = response.json()?; - - // This is pretty cool, filter_map first does get() on all the maps, and - // then filters out any None values - // Then, we do the same thing for streets - Ok(data - .iter() - .filter_map(|m| m.get("value")) - .filter_map(|v| Street::try_from(v.as_str()).ok()) - .collect()) -} - -/// Returns the pickup times for the various trash types. -/// -/// # Arguments -/// -/// * `street` - Street to look up -/// * `number` - House number in given street -/// * `start_date` - Earliest date for the results -/// * `end_date` - Latest date for the results -pub fn get_pickup_times( - street: &Street, - number: &u32, - start_date: &DateTime, - end_date: &DateTime, -) -> Result, FejError> { - let client = reqwest::Client::builder().cookie_store(true).build()?; - - // This populates the cookies with the necessary values - client - .post(BASE_URL) - .form(&[ - ("garbage_type", ""), - ("ivago_street", &String::from(street)), - ("number", &number.to_string()), - ("form_id", "garbage_address_form"), - ]) - .send()?; - - let response = client - .get(CAL_URL) - .query(&[ - ("_format", "json"), - ("type", ""), - ("start", &start_date.timestamp().to_string()), - ("end", &end_date.timestamp().to_string()), - ]) - .send()?; - let data: Vec> = response.json()?; - - let mut output: Vec = Vec::new(); - - for map in data - .iter() - .filter(|m| m.contains_key("date") && m.contains_key("label")) - { - // Because we filtered the maps in the loop, we can safely use unwrap - // here - if let Ok(date) = BasicDate::try_from(map.get("date").unwrap().as_str()) { - output.push(PickupTime::new(date, map.get("label").unwrap().to_string())) - } - } - - Ok(output) -} diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index 627e89b..7d3a9f9 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -1,45 +1,95 @@ -mod controller; +use crate::errors::FejError; +use chrono::DateTime; +use chrono_tz::Tz; +use reqwest::blocking as reqwest; +use std::collections::HashMap; +use std::convert::{From, TryFrom}; -use controller::{get_pickup_times, search_streets}; -use controller::{BasicDate, PickupTime, Street}; -use rocket::http::Status; -use rocket_contrib::json::Json; +mod basic_date; +mod pickup_time; +mod street; -pub fn routes() -> Vec { - routes![route_search_streets, route_get_pickup_times] -} +pub use basic_date::BasicDate; +pub use pickup_time::PickupTime; +pub use street::Street; -/// This route handles the Ivago search endpoint. It returns a list of streets, -/// consisting of a street name & a city. +/// Endpoint for the search feature +const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; +/// Endpoint for populating the initial cookies +const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; +/// Endpoint for the actual calendar output +const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; + +/// Searches the Ivago API for streets in the given city. /// /// # Arguments /// /// * `search_term` - Search term to use to look for streets -#[get("/search?")] -pub fn route_search_streets(q: String) -> Result>, Status> { - Ok(Json(search_streets(q.as_str())?)) +pub fn search_streets(search_term: &str) -> Result, FejError> { + let client = reqwest::Client::new(); + let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; + let data: Vec> = response.json()?; + + // This is pretty cool, filter_map first does get() on all the maps, and + // then filters out any None values + // Then, we do the same thing for streets + Ok(data + .iter() + .filter_map(|m| m.get("value")) + .filter_map(|v| Street::try_from(v.as_str()).ok()) + .collect()) } -/// Handles returning of pickup times for a specific address. It returns a list -/// of pickup times, containing a date and a description of the trash type. +/// Returns the pickup times for the various trash types. /// /// # Arguments /// /// * `street` - Street to look up -/// * `number` - House number in the given street -/// * `start_date` - Earliest date that can be returned -/// * `end_date` - Latest date that can be returned -#[get("/?&&&")] -pub fn route_get_pickup_times( - street: Street, - number: u32, - start_date: BasicDate, - end_date: BasicDate, -) -> Result>, Status> { - Ok(Json(get_pickup_times( - &street, - &number, - &start_date.0, - &end_date.0, - )?)) +/// * `number` - House number in given street +/// * `start_date` - Earliest date for the results +/// * `end_date` - Latest date for the results +pub fn get_pickup_times( + street: &Street, + number: &u32, + start_date: &DateTime, + end_date: &DateTime, +) -> Result, FejError> { + let client = reqwest::Client::builder().cookie_store(true).build()?; + + // This populates the cookies with the necessary values + client + .post(BASE_URL) + .form(&[ + ("garbage_type", ""), + ("ivago_street", &String::from(street)), + ("number", &number.to_string()), + ("form_id", "garbage_address_form"), + ]) + .send()?; + + let response = client + .get(CAL_URL) + .query(&[ + ("_format", "json"), + ("type", ""), + ("start", &start_date.timestamp().to_string()), + ("end", &end_date.timestamp().to_string()), + ]) + .send()?; + let data: Vec> = response.json()?; + + let mut output: Vec = Vec::new(); + + for map in data + .iter() + .filter(|m| m.contains_key("date") && m.contains_key("label")) + { + // Because we filtered the maps in the loop, we can safely use unwrap + // here + if let Ok(date) = BasicDate::try_from(map.get("date").unwrap().as_str()) { + output.push(PickupTime::new(date, map.get("label").unwrap().to_string())) + } + } + + Ok(output) } diff --git a/src/ivago/controller/pickup_time.rs b/src/ivago/pickup_time.rs similarity index 100% rename from src/ivago/controller/pickup_time.rs rename to src/ivago/pickup_time.rs diff --git a/src/ivago/controller/street.rs b/src/ivago/street.rs similarity index 100% rename from src/ivago/controller/street.rs rename to src/ivago/street.rs diff --git a/src/lib.rs b/src/lib.rs index b1e2733..761c91b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,7 @@ #![feature(proc_macro_hygiene, decl_macro)] -#[macro_use] -extern crate rocket; - // Route modules pub mod ivago; // Helper modules -pub mod catchers; pub mod errors; From 4b51ee20caad1f1b155900f457c4728cbfe11f01 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:36:03 +0200 Subject: [PATCH 2/5] [#26] Threw in the towel for now --- src/errors.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/errors.rs b/src/errors.rs index afcf04a..730714c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -10,6 +10,8 @@ pub enum FejError { FailedRequest, } +// I'd love to move this over to the server binary, but right now, error E0117 is making that +// imopssible impl From for Status { fn from(err: FejError) -> Status { match err { From 45c4a4e257efad0edba3d2f94149f33da591c1f8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:18:50 +0200 Subject: [PATCH 3/5] [#26] Moved lib & bin to own folders; Moved server tests; wrote some readmes --- Cargo.toml | 11 ++--- README.md | 60 ++++++++++++++++----------- src/{ => fej}/errors.rs | 0 src/{ => fej}/ivago/basic_date.rs | 0 src/{ => fej}/ivago/mod.rs | 0 src/{ => fej}/ivago/pickup_time.rs | 0 src/{ => fej}/ivago/street.rs | 0 src/{ => fej}/lib.rs | 0 src/{bin => }/server/catchers.rs | 0 src/{bin => }/server/main.rs | 4 ++ src/{bin => }/server/routes/ivago.rs | 0 src/{bin => }/server/routes/mod.rs | 0 tests/ivago.rs => src/server/tests.rs | 2 +- 13 files changed, 46 insertions(+), 31 deletions(-) rename src/{ => fej}/errors.rs (100%) rename src/{ => fej}/ivago/basic_date.rs (100%) rename src/{ => fej}/ivago/mod.rs (100%) rename src/{ => fej}/ivago/pickup_time.rs (100%) rename src/{ => fej}/ivago/street.rs (100%) rename src/{ => fej}/lib.rs (100%) rename src/{bin => }/server/catchers.rs (100%) rename src/{bin => }/server/main.rs (97%) rename src/{bin => }/server/routes/ivago.rs (100%) rename src/{bin => }/server/routes/mod.rs (100%) rename tests/ivago.rs => src/server/tests.rs (93%) diff --git a/Cargo.toml b/Cargo.toml index 46b01a1..4081b85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [lib] name = "fej" -src = "src/lib.rs" +path = "src/fej/lib.rs" test = true bench = true doc = true @@ -14,10 +14,11 @@ doctest = true [[bin]] name = "server" -src = "src/bin/server/main.rs" -test = false -bench = false -doc = false +path = "src/server/main.rs" +test = true +bench = true +doc = true +doctest = true [dependencies] rocket = "0.4.7" diff --git a/README.md b/README.md index 55d11e3..15e4e03 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,21 @@ # Fej -Fej is an API written in Rust. I started this project to learn the language, -and really just have some fun. +Fej is a RESTful API that does lots of different things. It started as an +experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The folder structure follows the structure of the URLs, e.g. the route for -`/hello/world` is found in the module `src/hello`. +The bulk of the project consists of the main `fej` library. The `src` folder +contains the `lib.rs` file for this library, and all other binaries import from +this main library. -Each module contains the following base files: +All binaries can be found in [`/src/bin`](src/bin), with the biggest one being +`server`. This is what actually runs as the Rocket.rs server. The other +binaries (with more possibly coming) are utility tools that will most likely be +run as cron jobs inside the containers, e.g. scrapers. -* `mod.rs`: defines the modules' content, and contains the route definitions. - The route functions themselves only contain the functionality needed to - represent the data, not acquire it. -* `controller.rs`: this file contains the actual logic of each route. If the - logic becomes too complicated to be contained inside a single file, - `controller.rs` becomes its own module folder named `controller`. -* `tests.rs`: this contains tests for the specific module. This can also be a - module directory if need be. - -Every module has a `routes` function that returns its route macros. +Version 1.1 also introduces the use of a database, namely +[PostgreSQL 13](https://www.postgresql.org/). ## Roadmap @@ -27,19 +23,33 @@ See [roadmap.md](roadmap.md). ## Development -The entire toolchain runs on Alpine inside Docker. This makes building easier, -and (hopefully) eliminates any system-specific bugs. +To make development more consistent (and keep my computer a bit cleaner) I've +decided to run the entire toolchain using Docker, with Alpine Linux base +images. This also allows me to make really small final images. Technically, +Docker is the only dependency you need to contribute to this project +(not accounting for language servers etc.). -A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as -all the commands are documented for your understanding ;) +A [Bash script](fejctl) is provided to speed up development on the various +binaries. It aids in starting up the containers, choosing which binary to run +etc. A quick rundown of its most important features: -There's also the `build` script. This script does all the "heavy" lifting. It -chooses which Dockerfile to build according to the given arguments, and -generates tags for the images (useful when pushing releases). The Makefile is -really just a wrapper around this build script, allowing you to write -`make test` instead of `./build -m dev -a run test`. +```bash +# By default, it compiles the 'server' binary (but doesn't run it) +./fejctl -tl;dr run `make run` to run your build, and `make test` to run the tests. +# The first argument is the action you wish to perform, the second on which +# binary (not all commands need a binary as input, so they just ignore it) +# For example, this will run the binary called server +./fejctl r server + +# This runs the tests (all commands also have their full name if you want) +./fejctl t +./fejctl test + +# These attach to the two containers +./fejctl sh # Opens a new root shell inside the 'fej' container +./fejctl psql # Open a new psql shell in the fej database +``` ## Docker images diff --git a/src/errors.rs b/src/fej/errors.rs similarity index 100% rename from src/errors.rs rename to src/fej/errors.rs diff --git a/src/ivago/basic_date.rs b/src/fej/ivago/basic_date.rs similarity index 100% rename from src/ivago/basic_date.rs rename to src/fej/ivago/basic_date.rs diff --git a/src/ivago/mod.rs b/src/fej/ivago/mod.rs similarity index 100% rename from src/ivago/mod.rs rename to src/fej/ivago/mod.rs diff --git a/src/ivago/pickup_time.rs b/src/fej/ivago/pickup_time.rs similarity index 100% rename from src/ivago/pickup_time.rs rename to src/fej/ivago/pickup_time.rs diff --git a/src/ivago/street.rs b/src/fej/ivago/street.rs similarity index 100% rename from src/ivago/street.rs rename to src/fej/ivago/street.rs diff --git a/src/lib.rs b/src/fej/lib.rs similarity index 100% rename from src/lib.rs rename to src/fej/lib.rs diff --git a/src/bin/server/catchers.rs b/src/server/catchers.rs similarity index 100% rename from src/bin/server/catchers.rs rename to src/server/catchers.rs diff --git a/src/bin/server/main.rs b/src/server/main.rs similarity index 97% rename from src/bin/server/main.rs rename to src/server/main.rs index 3fde2c7..58b0300 100644 --- a/src/bin/server/main.rs +++ b/src/server/main.rs @@ -2,6 +2,10 @@ #[macro_use] extern crate rocket; + +#[cfg(test)] +mod tests; + mod catchers; mod routes; diff --git a/src/bin/server/routes/ivago.rs b/src/server/routes/ivago.rs similarity index 100% rename from src/bin/server/routes/ivago.rs rename to src/server/routes/ivago.rs diff --git a/src/bin/server/routes/mod.rs b/src/server/routes/mod.rs similarity index 100% rename from src/bin/server/routes/mod.rs rename to src/server/routes/mod.rs diff --git a/tests/ivago.rs b/src/server/tests.rs similarity index 93% rename from tests/ivago.rs rename to src/server/tests.rs index d78501a..8871762 100644 --- a/tests/ivago.rs +++ b/src/server/tests.rs @@ -3,7 +3,7 @@ use rocket::http::Status; use rocket::local::Client; fn rocket() -> rocket::Rocket { - rocket::ignite().mount("/", fej_lib::ivago::routes()) + rocket::ignite().mount("/", super::routes::ivago()) } /// Test 404 response From c2fa764e8002352728030e512f6f99b15b0014f6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:22:30 +0200 Subject: [PATCH 4/5] [#26] Updated README again --- README.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 15e4e03..e2573af 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,10 @@ experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The bulk of the project consists of the main `fej` library. The `src` folder -contains the `lib.rs` file for this library, and all other binaries import from -this main library. - -All binaries can be found in [`/src/bin`](src/bin), with the biggest one being -`server`. This is what actually runs as the Rocket.rs server. The other -binaries (with more possibly coming) are utility tools that will most likely be -run as cron jobs inside the containers, e.g. scrapers. +The `src` folder contains subfolders for the various binaries and the main +library, called `fej`. The biggest binary is called `server`, which is the +binary that actually runs the Rocket.rs web server. All the others are utility +programs, mostly consisting of scrapers for various services. Version 1.1 also introduces the use of a database, namely [PostgreSQL 13](https://www.postgresql.org/). From 0e4eb6812164b52d77c4532ef53fbf79fe15048b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:30:25 +0200 Subject: [PATCH 5/5] [#26] Updated roadmap.md --- roadmap.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roadmap.md b/roadmap.md index f03e4b5..bfdd622 100644 --- a/roadmap.md +++ b/roadmap.md @@ -69,8 +69,8 @@ setting up a database for this version anyways. ## Kissanime I like watching anime from time to time, and I've always used Kissanime for -this. However, their sites can be quite slow, and riddled with ads from time to -time. That's why I'd like to create a high-speed wrapper that extracts all the -needed info from their sites, removing the need for the user to ever actually -visit their site. The API can just act as a fast search index, complete with -indexing of the links to the videos and everything. +this. However, their sites can be quite slow, and riddled with ads. That's why +I'd like to create a high-speed wrapper that extracts all the needed info from +their sites, removing the need for the user to ever actually visit their site. +The API can just act as a fast search index, complete with indexing of the +links to the videos and everything.