From 207e735556d18c30ba5a6ef3f29f07d1edbae60e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 13 Mar 2021 22:14:24 +0100 Subject: [PATCH 01/11] Added Ivago README --- src/ivago/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/ivago/README.md diff --git a/src/ivago/README.md b/src/ivago/README.md new file mode 100644 index 0000000..c69c1b4 --- /dev/null +++ b/src/ivago/README.md @@ -0,0 +1,5 @@ +# Ivago + +This part of the API is a wrapper around the Ivago website (Ivago being the +company that collects the trash in my city). Their city isn't exactly RESTful, +so this endpoint simply wraps it in a RESTful wrapper. From c6d29f329c1f5518e51b8d694d74b413ffef0757 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 21 Mar 2021 16:18:53 +0100 Subject: [PATCH 02/11] Start ivago calendar endpoint --- Cargo.lock | 33 ++++++++++++++++++++++++++++ Cargo.toml | 1 + src/ivago/controller/mod.rs | 2 ++ src/ivago/controller/pickup_times.rs | 24 +++++++++++++------- src/ivago/mod.rs | 20 ++++++++++++----- 5 files changed, 66 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85dbcb8..d528ead 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,6 +152,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time 0.1.43", + "winapi 0.3.9", +] + [[package]] name = "cipher" version = "0.2.5" @@ -902,6 +915,25 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.13.0" @@ -1258,6 +1290,7 @@ dependencies = [ name = "rust-api" version = "0.1.0" dependencies = [ + "chrono", "reqwest", "rocket", "rocket_contrib", diff --git a/Cargo.toml b/Cargo.toml index c994ff9..0442ebf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ edition = "2018" [dependencies] rocket = "0.4.7" serde = "1.0.124" +chrono = "0.4.19" [dependencies.reqwest] version = "0.11.2" diff --git a/src/ivago/controller/mod.rs b/src/ivago/controller/mod.rs index 01f00a4..e2555c5 100644 --- a/src/ivago/controller/mod.rs +++ b/src/ivago/controller/mod.rs @@ -1,5 +1,7 @@ mod search; +mod pickup_times; pub use search::{Street, search_streets}; +pub use pickup_times::{get_pickup_times, PickupTime}; ///// Return the known pickup times for the given street and/or city diff --git a/src/ivago/controller/pickup_times.rs b/src/ivago/controller/pickup_times.rs index 98b8e64..37d84c4 100644 --- a/src/ivago/controller/pickup_times.rs +++ b/src/ivago/controller/pickup_times.rs @@ -1,19 +1,27 @@ +use std::error::Error; +use chrono::NaiveDate; +use super::search::Street; + + const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; -/// Represents a timezoneless date -pub struct Date { - day: u8, - month: u8, - year: u32, -} - /// Represents a pickup time instance. All fields are a direct map of the /// original API pub struct PickupTime { - date: Date, + date: NaiveDate, label: String, classes: Vec, url: String } + + +pub fn get_pickup_times( + street: Street, + number: u64, + start_date: NaiveDate, + end_date: NaiveDate +) -> Result, Box> { + Ok(Vec::new()) +} diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index 7ba799e..8547130 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -2,10 +2,11 @@ mod controller; use rocket_contrib::json::Json; +use chrono::NaiveDate; pub fn routes() -> Vec { routes![ - search_streets_json, + route_search_streets, ] } @@ -13,12 +14,19 @@ pub fn routes() -> Vec { // TODO make this async // TODO change this so it can return errors instead of empty json #[get("/search?", format="json")] -pub fn search_streets_json(street: String) -> Json> { +pub fn route_search_streets(street: String) -> Json> { match controller::search_streets(&street) { Ok(streets) => Json(streets), - Err(err) => { - println!("{:?}", err); - Json(Vec::new()) - }, + Err(err) => Json(Vec::new()), } } + +#[get("/?&&&")] +pub fn route_get_pickup_times( + street: controller::Street, + number: u64, + start_date: NaiveDate, + end_date: NaiveDate + ) -> Json> { + +} From f804c01849c015434c31c44fdd5c7f72ea671dc5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 2 Apr 2021 21:08:50 +0200 Subject: [PATCH 03/11] Added git hooks --- .hooks/commit-msg | 23 +++++++++++++++++++++++ .hooks/pre-commit | 13 +++++++++++++ 2 files changed, 36 insertions(+) create mode 100755 .hooks/commit-msg create mode 100755 .hooks/pre-commit diff --git a/.hooks/commit-msg b/.hooks/commit-msg new file mode 100755 index 0000000..284ee8d --- /dev/null +++ b/.hooks/commit-msg @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# This hook checks if the commit message ends with an issue number, and if not, +# tries to derive that number from the branch name + +branch=`git rev-parse --abbrev-ref HEAD` + +# This check doesn't need to run when commiting to develop/master +[[ "$branch" =~ ^master|develop$ ]] && exit 0 + +issue_num=`echo "$branch" | grep -Po '^[0-9]+(?=-)'` + +# Check if issue number is already present +if ! grep -q '([0-9]\+)$' "$1"; then + # Error out if we can't derive issue number + [[ -z "$issue_num" ]] && { + >&2 echo "Couldn't derive issue number from branch. Please add one manually."; + exit 1; + } + + # Append issue number, and remove all comments + echo "[#$issue_num]" "$(cat "$1")" > "$1" +fi diff --git a/.hooks/pre-commit b/.hooks/pre-commit new file mode 100755 index 0000000..d9dd92d --- /dev/null +++ b/.hooks/pre-commit @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# This hook lints the code, and if we're on develop or master, also forces the tests to pass. + +branch=`git rev-parse --abbrev-ref HEAD` + +# TODO should we add release branches here as well? +if [[ "$branch" =~ ^master|develop$ ]]; then + make test > /dev/null 2>&1 || { + >&2 echo "Tests failed. check 'make test' for more info."; + exit 1; + } +fi From e3f134a9bf770664078e1b2ad85d57bf48a6b710 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 2 Apr 2021 21:20:36 +0200 Subject: [PATCH 04/11] Added code formatting --- .hooks/pre-commit | 1 + Makefile | 4 +++ src/hello/mod.rs | 9 +++---- src/hello/tests.rs | 2 +- src/ivago/controller/mod.rs | 7 +++-- src/ivago/controller/pickup_times.rs | 12 +++------ src/ivago/controller/search.rs | 39 +++++++++++----------------- src/ivago/mod.rs | 14 +++++----- src/ivago/tests.rs | 1 + src/main.rs | 3 ++- 10 files changed, 40 insertions(+), 52 deletions(-) diff --git a/.hooks/pre-commit b/.hooks/pre-commit index d9dd92d..a2dd254 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -1,6 +1,7 @@ #!/usr/bin/env bash # This hook lints the code, and if we're on develop or master, also forces the tests to pass. +cargo fmt -- --check branch=`git rev-parse --abbrev-ref HEAD` diff --git a/Makefile b/Makefile index 4017a77..2f9e53d 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,10 @@ test: @ cargo test .PHONY: test +format: + @ cargo fmt +.PHONY: format + # Documentation docs: diff --git a/src/hello/mod.rs b/src/hello/mod.rs index c15060b..608d234 100644 --- a/src/hello/mod.rs +++ b/src/hello/mod.rs @@ -1,11 +1,8 @@ -#[cfg(test)] mod tests; +#[cfg(test)] +mod tests; pub fn routes() -> Vec { - routes![ - world, - hello, - name_age - ] + routes![world, hello, name_age] } #[get("/world")] diff --git a/src/hello/tests.rs b/src/hello/tests.rs index d4f7096..e19264d 100644 --- a/src/hello/tests.rs +++ b/src/hello/tests.rs @@ -1,5 +1,5 @@ -use rocket::local::Client; use rocket::http::Status; +use rocket::local::Client; fn rocket() -> rocket::Rocket { rocket::ignite().mount("/", super::routes()) diff --git a/src/ivago/controller/mod.rs b/src/ivago/controller/mod.rs index e2555c5..6ff9dff 100644 --- a/src/ivago/controller/mod.rs +++ b/src/ivago/controller/mod.rs @@ -1,8 +1,7 @@ -mod search; mod pickup_times; -pub use search::{Street, search_streets}; +mod search; pub use pickup_times::{get_pickup_times, PickupTime}; - +pub use search::{search_streets, Street}; ///// Return the known pickup times for the given street and/or city ///// @@ -30,7 +29,7 @@ pub use pickup_times::{get_pickup_times, PickupTime}; // let params = [ // ("_format", "json"), // ("type", ""), - + // ] //r2 = s.get("https://www.ivago.be/nl/particulier/garbage/pick-up/pickups?", diff --git a/src/ivago/controller/pickup_times.rs b/src/ivago/controller/pickup_times.rs index 37d84c4..aa86a1d 100644 --- a/src/ivago/controller/pickup_times.rs +++ b/src/ivago/controller/pickup_times.rs @@ -1,27 +1,23 @@ -use std::error::Error; -use chrono::NaiveDate; use super::search::Street; - +use chrono::NaiveDate; +use std::error::Error; const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; - - /// Represents a pickup time instance. All fields are a direct map of the /// original API pub struct PickupTime { date: NaiveDate, label: String, classes: Vec, - url: String + url: String, } - pub fn get_pickup_times( street: Street, number: u64, start_date: NaiveDate, - end_date: NaiveDate + end_date: NaiveDate, ) -> Result, Box> { Ok(Vec::new()) } diff --git a/src/ivago/controller/search.rs b/src/ivago/controller/search.rs index d0282de..2df30b9 100644 --- a/src/ivago/controller/search.rs +++ b/src/ivago/controller/search.rs @@ -1,13 +1,11 @@ use reqwest::blocking as reqwest; +use serde::ser::{Serialize, SerializeStruct, Serializer}; use std::collections::HashMap; use std::convert::TryFrom; -use serde::ser::{Serialize, Serializer, SerializeStruct}; use std::error::Error; - /// Endpoint for the search feature -const SEARCH_URL: &str ="https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; - +const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; impl From for String { fn from(street: Street) -> String { @@ -15,44 +13,39 @@ impl From for String { } } - impl Serialize for Street { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut s = serializer.serialize_struct("Street", 2)?; - s.serialize_field("name", &self.name)?; - s.serialize_field("city", &self.city)?; - s.end() - } + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut s = serializer.serialize_struct("Street", 2)?; + s.serialize_field("name", &self.name)?; + s.serialize_field("city", &self.city)?; + s.end() + } } - impl TryFrom for Street { type Error = (); fn try_from(value: String) -> Result { if let Some(index) = value.find('(') { Ok(Street { - name: (value[0 .. index - 1].trim()).to_string(), - city: (value[index + 1 .. value.len() - 1].trim()).to_string(), + name: (value[0..index - 1].trim()).to_string(), + city: (value[index + 1..value.len() - 1].trim()).to_string(), }) - - }else { + } else { Err(()) } } } - /// Represents a street pub struct Street { pub name: String, pub city: String, } - /// Searches the Ivago API for streets in the given city /// /// # Arguments @@ -62,9 +55,7 @@ pub struct Street { // TODO find out how to do this async pub fn search_streets(street_name: &String) -> Result, Box> { let client = reqwest::Client::new(); - let response = client.get(SEARCH_URL) - .query(&[("q", street_name)]) - .send()?; + let response = client.get(SEARCH_URL).query(&[("q", street_name)]).send()?; let data: Vec> = response.json()?; let mut output: Vec = Vec::new(); diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index d24dc28..9cf77dc 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -1,13 +1,12 @@ -#[cfg(test)] mod tests; mod controller; +#[cfg(test)] +mod tests; -use rocket_contrib::json::Json; use chrono::NaiveDate; +use rocket_contrib::json::Json; pub fn routes() -> Vec { - routes![ - route_search_streets, - ] + routes![route_search_streets,] } // URL: https://www.ivago.be/nl/particulier/autocomplete/garbage/streets?q=Lange @@ -26,7 +25,6 @@ pub fn route_get_pickup_times( street: controller::Street, number: u64, start_date: NaiveDate, - end_date: NaiveDate - ) -> Json> { - + end_date: NaiveDate, +) -> Json> { } diff --git a/src/ivago/tests.rs b/src/ivago/tests.rs index e69de29..8b13789 100644 --- a/src/ivago/tests.rs +++ b/src/ivago/tests.rs @@ -0,0 +1 @@ + diff --git a/src/main.rs b/src/main.rs index 43348ab..8a5ea07 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ #![feature(proc_macro_hygiene, decl_macro)] -#[macro_use] extern crate rocket; +#[macro_use] +extern crate rocket; // Route modules mod hello; From e3174f21afeaab92924c48fe4fcf870e86d14e83 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 3 Apr 2021 20:47:39 +0200 Subject: [PATCH 05/11] [#4] We now properly return error values --- src/ivago/mod.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index 9cf77dc..386bf36 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -3,6 +3,7 @@ mod controller; mod tests; use chrono::NaiveDate; +use rocket::http::Status; use rocket_contrib::json::Json; pub fn routes() -> Vec { @@ -10,21 +11,20 @@ pub fn routes() -> Vec { } // URL: https://www.ivago.be/nl/particulier/autocomplete/garbage/streets?q=Lange -// TODO make this async -// TODO change this so it can return errors instead of empty json #[get("/search?")] -pub fn search_streets_json(street: String) -> Json> { +pub fn route_search_streets(street: String) -> Result>, Status> { match controller::search_streets(&street) { - Ok(streets) => Json(streets), - Err(err) => Json(Vec::new()), + Ok(streets) => Ok(Json(streets)), + Err(err) => Err(Status::InternalServerError), } } #[get("/?&&&")] pub fn route_get_pickup_times( street: controller::Street, - number: u64, + number: u32, start_date: NaiveDate, end_date: NaiveDate, -) -> Json> { +) -> Result>, Status> { + Err(Status::InternalServerError) } From 27a61f8a9a1571793f50880ffbad01a4a9da1551 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 3 Apr 2021 22:40:22 +0200 Subject: [PATCH 06/11] [#4] Started implementing needed traits (#4) --- src/ivago/controller/pickup_times.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/ivago/controller/pickup_times.rs b/src/ivago/controller/pickup_times.rs index aa86a1d..1f037a2 100644 --- a/src/ivago/controller/pickup_times.rs +++ b/src/ivago/controller/pickup_times.rs @@ -1,5 +1,7 @@ use super::search::Street; use chrono::NaiveDate; +use rocket::http::RawStr; +use rocket::request::FromFormValue; use std::error::Error; const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; @@ -13,6 +15,17 @@ pub struct PickupTime { url: String, } +impl<'v> FromFormValue<'v> for NaiveDate { + type Error = &'v RawStr; + + fn from_form_value(form_value: &'v RawStr) -> Result { + match NaiveDate::parse_from_str(form_value, "%Y-%m-%d") { + Ok(date) => Ok(date), + Err(_) => Err(form_value), + } + } +} + pub fn get_pickup_times( street: Street, number: u64, From c89841ad38cd28760323737c30fa6bc9f91b7e2e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 4 Apr 2021 09:57:19 +0200 Subject: [PATCH 07/11] [#4] Finally got proper form value handling (#1) --- Cargo.lock | 27 ++++++++++ Cargo.toml | 1 + Makefile | 6 ++- src/ivago/controller/mod.rs | 2 +- src/ivago/controller/pickup_times.rs | 79 ++++++++++++++++++++++------ src/ivago/controller/search.rs | 20 +++++++ src/ivago/mod.rs | 15 +++--- 7 files changed, 126 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e248c93..9c0d5ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,6 +56,15 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + [[package]] name = "atty" version = "0.2.14" @@ -331,6 +340,7 @@ name = "fej" version = "0.0.1" dependencies = [ "chrono", + "regex", "reqwest", "rocket", "rocket_contrib", @@ -1190,6 +1200,23 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" + [[package]] name = "remove_dir_all" version = "0.5.3" diff --git a/Cargo.toml b/Cargo.toml index 53b452c..6969693 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ edition = "2018" rocket = "0.4.7" serde = "1.0.124" chrono = "0.4.19" +regex = "1.4.5" [dependencies.reqwest] version = "0.11.2" diff --git a/Makefile b/Makefile index 2f9e53d..a1baf59 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ push: # Run run: - @ cargo run + @ RUST_BACKTRACE=1 cargo run .PHONY: run @@ -36,6 +36,10 @@ format: @ cargo fmt .PHONY: format +lint: + @ cargo fmt -- --check +.PHONY: lint + # Documentation docs: diff --git a/src/ivago/controller/mod.rs b/src/ivago/controller/mod.rs index 6ff9dff..8e3cb62 100644 --- a/src/ivago/controller/mod.rs +++ b/src/ivago/controller/mod.rs @@ -1,6 +1,6 @@ mod pickup_times; mod search; -pub use pickup_times::{get_pickup_times, PickupTime}; +pub use pickup_times::{get_pickup_times, BasicDate, PickupTime}; pub use search::{search_streets, Street}; ///// Return the known pickup times for the given street and/or city diff --git a/src/ivago/controller/pickup_times.rs b/src/ivago/controller/pickup_times.rs index 1f037a2..e4ce656 100644 --- a/src/ivago/controller/pickup_times.rs +++ b/src/ivago/controller/pickup_times.rs @@ -1,36 +1,83 @@ use super::search::Street; -use chrono::NaiveDate; +use regex::Regex; use rocket::http::RawStr; use rocket::request::FromFormValue; +use serde::ser::{Serialize, SerializeStruct, Serializer}; use std::error::Error; const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; +/// Represents a very simple Timezoneless date. Considering the timezone will +/// always be CEST (aka Belgium's timezone), this is good enough. +pub struct BasicDate { + year: u32, + month: u8, + day: u8, +} + +impl<'v> FromFormValue<'v> for BasicDate { + type Error = &'v RawStr; + + fn from_form_value(form_value: &'v RawStr) -> Result { + // Beautiful how this exact example is in the docs + let re = Regex::new(r"^(\d{4})-(\d{2})-(\d{2})$").unwrap(); + match re.captures(form_value) { + None => Err(form_value), + // Here, we can assume these parses will work, because the regex + // didn't fail + Some(caps) => Ok(BasicDate { + year: caps.get(1).unwrap().as_str().parse().unwrap(), + month: caps.get(2).unwrap().as_str().parse().unwrap(), + day: caps.get(3).unwrap().as_str().parse().unwrap(), + }), + } + } +} + +impl ToString for BasicDate { + fn to_string(&self) -> String { + format!("{}-{}-{}", self.year, self.month, self.day) + } +} + +impl Serialize for BasicDate { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl Serialize for PickupTime { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut s = serializer.serialize_struct("PickupTime", 4)?; + s.serialize_field("date", &self.date)?; + s.serialize_field("label", &self.label)?; + s.serialize_field("classes", &self.classes)?; + s.serialize_field("url", &self.url)?; + + s.end() + } +} + /// Represents a pickup time instance. All fields are a direct map of the /// original API pub struct PickupTime { - date: NaiveDate, + date: BasicDate, label: String, classes: Vec, url: String, } -impl<'v> FromFormValue<'v> for NaiveDate { - type Error = &'v RawStr; - - fn from_form_value(form_value: &'v RawStr) -> Result { - match NaiveDate::parse_from_str(form_value, "%Y-%m-%d") { - Ok(date) => Ok(date), - Err(_) => Err(form_value), - } - } -} - pub fn get_pickup_times( street: Street, - number: u64, - start_date: NaiveDate, - end_date: NaiveDate, + number: u32, + start_date: BasicDate, + end_date: BasicDate, ) -> Result, Box> { Ok(Vec::new()) } diff --git a/src/ivago/controller/search.rs b/src/ivago/controller/search.rs index 2df30b9..be81254 100644 --- a/src/ivago/controller/search.rs +++ b/src/ivago/controller/search.rs @@ -1,4 +1,7 @@ +use regex::Regex; use reqwest::blocking as reqwest; +use rocket::http::RawStr; +use rocket::request::FromFormValue; use serde::ser::{Serialize, SerializeStruct, Serializer}; use std::collections::HashMap; use std::convert::TryFrom; @@ -40,6 +43,23 @@ impl TryFrom for Street { } } +impl<'v> FromFormValue<'v> for Street { + type Error = &'v RawStr; + + fn from_form_value(form_value: &'v RawStr) -> Result { + // This regex is pretty loose tbh, but not sure how I can make it more + // strict right now + let re = Regex::new(r"^(.+) \((.+)\)$").unwrap(); + match re.captures(&form_value.url_decode_lossy()) { + None => Err(form_value), + Some(caps) => Ok(Street { + name: String::from(caps.get(1).unwrap().as_str()), + city: String::from(caps.get(2).unwrap().as_str()), + }), + } + } +} + /// Represents a street pub struct Street { pub name: String, diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index 386bf36..33ab64e 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -2,12 +2,11 @@ mod controller; #[cfg(test)] mod tests; -use chrono::NaiveDate; use rocket::http::Status; use rocket_contrib::json::Json; pub fn routes() -> Vec { - routes![route_search_streets,] + routes![route_search_streets, route_get_pickup_times] } // URL: https://www.ivago.be/nl/particulier/autocomplete/garbage/streets?q=Lange @@ -15,7 +14,7 @@ pub fn routes() -> Vec { pub fn route_search_streets(street: String) -> Result>, Status> { match controller::search_streets(&street) { Ok(streets) => Ok(Json(streets)), - Err(err) => Err(Status::InternalServerError), + Err(_) => Err(Status::InternalServerError), } } @@ -23,8 +22,12 @@ pub fn route_search_streets(street: String) -> Result Result>, Status> { - Err(Status::InternalServerError) + match controller::get_pickup_times(street, number, start_date, end_date) { + // TODO provide more meaningful status codes here + Err(_) => Err(Status::InternalServerError), + Ok(times) => Ok(Json(times)), + } } From 01276004e1fb20a7b4108dc1a95b1ffafbaf1bc8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 4 Apr 2021 11:39:40 +0200 Subject: [PATCH 08/11] [#4] Added BasicDate type conversions --- src/ivago/controller/pickup_times.rs | 51 ++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/src/ivago/controller/pickup_times.rs b/src/ivago/controller/pickup_times.rs index e4ce656..174e538 100644 --- a/src/ivago/controller/pickup_times.rs +++ b/src/ivago/controller/pickup_times.rs @@ -1,11 +1,17 @@ use super::search::Street; +use chrono::{FixedOffset, TimeZone}; use regex::Regex; +use reqwest::blocking as reqwest; use rocket::http::RawStr; use rocket::request::FromFormValue; -use serde::ser::{Serialize, SerializeStruct, Serializer}; +use serde::ser::{SerializeStruct, Serializer}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::convert::From; use std::error::Error; const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; +const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; /// Represents a very simple Timezoneless date. Considering the timezone will /// always be CEST (aka Belgium's timezone), this is good enough. @@ -49,6 +55,16 @@ impl Serialize for BasicDate { } } +impl From for i64 { + fn from(date: BasicDate) -> i64 { + // Timezone of Brussels is UTC + 2 hours in the western hemisphere + FixedOffset::west(7_200) + .ymd(date.year as i32, date.month as u32, date.day as u32) + .and_hms(0, 0, 0) + .timestamp() + } +} + impl Serialize for PickupTime { fn serialize(&self, serializer: S) -> Result where @@ -66,6 +82,7 @@ impl Serialize for PickupTime { /// Represents a pickup time instance. All fields are a direct map of the /// original API +#[derive(Deserialize)] pub struct PickupTime { date: BasicDate, label: String, @@ -79,5 +96,35 @@ pub fn get_pickup_times( start_date: BasicDate, end_date: BasicDate, ) -> Result, Box> { - Ok(Vec::new()) + let client = reqwest::Client::builder().cookie_store(true).build()?; + + // This populates the cookies with the necessary values + client + .post(BASE_URL) + .form(&[ + ("garbage_type", ""), + ("ivago_street", &String::from(street)), + ("number", &number.to_string()), + ("form_id", "garbage_address_form"), + ]) + .send()?; + + let response = client + .get(CAL_URL) + .query(&[ + ("_format", "json"), + ("type", ""), + ("start", &i64::from(start_date).to_string()), + ("end", &i64::from(end_date).to_string()), + ]) + .send()?; + let data: Vec> = response.json()?; + + let mut output: Vec = Vec::new(); + + for map in data.iter() { + if let Some(value) = map.get("value") {} + } + + Ok() } From 0c7d55647e08cf8d66a498c73d5664e72bbcb03b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 4 Apr 2021 12:06:33 +0200 Subject: [PATCH 09/11] [#4] JSON requests now work! --- src/ivago/controller/pickup_times.rs | 48 +++++++++++++++++++++------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/ivago/controller/pickup_times.rs b/src/ivago/controller/pickup_times.rs index 174e538..58d3f29 100644 --- a/src/ivago/controller/pickup_times.rs +++ b/src/ivago/controller/pickup_times.rs @@ -7,7 +7,7 @@ use rocket::request::FromFormValue; use serde::ser::{SerializeStruct, Serializer}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::convert::From; +use std::convert::{From, TryFrom}; use std::error::Error; const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; @@ -26,9 +26,29 @@ impl<'v> FromFormValue<'v> for BasicDate { fn from_form_value(form_value: &'v RawStr) -> Result { // Beautiful how this exact example is in the docs + match BasicDate::try_from(form_value.as_str()) { + Err(_) => Err(form_value), + // Here, we can assume these parses will work, because the regex + // didn't fail + Ok(date) => Ok(date), + } + } +} + +impl ToString for BasicDate { + fn to_string(&self) -> String { + format!("{}-{}-{}", self.year, self.month, self.day) + } +} + +impl TryFrom<&str> for BasicDate { + type Error = (); + + fn try_from(s: &str) -> Result { let re = Regex::new(r"^(\d{4})-(\d{2})-(\d{2})$").unwrap(); - match re.captures(form_value) { - None => Err(form_value), + + match re.captures(s) { + None => Err(()), // Here, we can assume these parses will work, because the regex // didn't fail Some(caps) => Ok(BasicDate { @@ -40,12 +60,6 @@ impl<'v> FromFormValue<'v> for BasicDate { } } -impl ToString for BasicDate { - fn to_string(&self) -> String { - format!("{}-{}-{}", self.year, self.month, self.day) - } -} - impl Serialize for BasicDate { fn serialize(&self, serializer: S) -> Result where @@ -82,7 +96,6 @@ impl Serialize for PickupTime { /// Represents a pickup time instance. All fields are a direct map of the /// original API -#[derive(Deserialize)] pub struct PickupTime { date: BasicDate, label: String, @@ -123,8 +136,19 @@ pub fn get_pickup_times( let mut output: Vec = Vec::new(); for map in data.iter() { - if let Some(value) = map.get("value") {} + output.push(PickupTime { + // TODO should I check here if the parsing worked? + date: BasicDate::try_from(map.get("date").unwrap().as_str()).unwrap(), + label: map.get("label").unwrap().to_string(), + classes: map + .get("classes") + .unwrap() + .split_whitespace() + .map(|x| String::from(x)) + .collect(), + url: map.get("url").unwrap().to_string(), + }) } - Ok() + Ok(output) } From eab31e5e913f4a05c5590cbe8a28de168400740c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 4 Apr 2021 13:03:08 +0200 Subject: [PATCH 10/11] [#9] Added general error module --- src/errors.rs | 23 +++++++++++++++++++++++ src/ivago/controller/search.rs | 3 ++- src/ivago/mod.rs | 7 +++---- src/main.rs | 1 + 4 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 src/errors.rs diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..6403235 --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,23 @@ +use reqwest::Error; +use rocket::http::Status; + +pub enum FejError { + InvalidArgument, + FailedRequest, +} + +impl From for Status { + fn from(err: FejError) -> Status { + match err { + FejError::InvalidArgument => Status::BadRequest, + FejError::FailedRequest => Status::InternalServerError, + } + } +} + +// TODO make this more advanced where possible +impl From for FejError { + fn from(_: Error) -> FejError { + FejError::FailedRequest + } +} diff --git a/src/ivago/controller/search.rs b/src/ivago/controller/search.rs index be81254..926d282 100644 --- a/src/ivago/controller/search.rs +++ b/src/ivago/controller/search.rs @@ -1,3 +1,4 @@ +use crate::errors::FejError; use regex::Regex; use reqwest::blocking as reqwest; use rocket::http::RawStr; @@ -73,7 +74,7 @@ pub struct Street { /// * `street` - name of the street /// * `city` - city the street is in // TODO find out how to do this async -pub fn search_streets(street_name: &String) -> Result, Box> { +pub fn search_streets(street_name: &String) -> Result, FejError> { let client = reqwest::Client::new(); let response = client.get(SEARCH_URL).query(&[("q", street_name)]).send()?; let data: Vec> = response.json()?; diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index 33ab64e..c173aab 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -2,6 +2,7 @@ mod controller; #[cfg(test)] mod tests; +use crate::errors::FejError; use rocket::http::Status; use rocket_contrib::json::Json; @@ -12,10 +13,8 @@ pub fn routes() -> Vec { // URL: https://www.ivago.be/nl/particulier/autocomplete/garbage/streets?q=Lange #[get("/search?")] pub fn route_search_streets(street: String) -> Result>, Status> { - match controller::search_streets(&street) { - Ok(streets) => Ok(Json(streets)), - Err(_) => Err(Status::InternalServerError), - } + let result = controller::search_streets(&street)?; + Ok(Json(result)) } #[get("/?&&&")] diff --git a/src/main.rs b/src/main.rs index 8a5ea07..882db7c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ extern crate rocket; // Route modules +mod errors; mod hello; mod ivago; From df6e245030e62d1bcae1f6bdfac2d56d9a5520b8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sun, 4 Apr 2021 14:14:14 +0200 Subject: [PATCH 11/11] [#9] Moved calendar function to new errors system --- src/ivago/controller/pickup_times.rs | 6 +++--- src/ivago/controller/search.rs | 1 - src/ivago/mod.rs | 13 ++++--------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/ivago/controller/pickup_times.rs b/src/ivago/controller/pickup_times.rs index 58d3f29..202f8fc 100644 --- a/src/ivago/controller/pickup_times.rs +++ b/src/ivago/controller/pickup_times.rs @@ -1,14 +1,14 @@ use super::search::Street; +use crate::errors::FejError; use chrono::{FixedOffset, TimeZone}; use regex::Regex; use reqwest::blocking as reqwest; use rocket::http::RawStr; use rocket::request::FromFormValue; use serde::ser::{SerializeStruct, Serializer}; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use std::collections::HashMap; use std::convert::{From, TryFrom}; -use std::error::Error; const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; @@ -108,7 +108,7 @@ pub fn get_pickup_times( number: u32, start_date: BasicDate, end_date: BasicDate, -) -> Result, Box> { +) -> Result, FejError> { let client = reqwest::Client::builder().cookie_store(true).build()?; // This populates the cookies with the necessary values diff --git a/src/ivago/controller/search.rs b/src/ivago/controller/search.rs index 926d282..5037bb4 100644 --- a/src/ivago/controller/search.rs +++ b/src/ivago/controller/search.rs @@ -6,7 +6,6 @@ use rocket::request::FromFormValue; use serde::ser::{Serialize, SerializeStruct, Serializer}; use std::collections::HashMap; use std::convert::TryFrom; -use std::error::Error; /// Endpoint for the search feature const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index c173aab..541b27a 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -2,7 +2,6 @@ mod controller; #[cfg(test)] mod tests; -use crate::errors::FejError; use rocket::http::Status; use rocket_contrib::json::Json; @@ -10,11 +9,9 @@ pub fn routes() -> Vec { routes![route_search_streets, route_get_pickup_times] } -// URL: https://www.ivago.be/nl/particulier/autocomplete/garbage/streets?q=Lange #[get("/search?")] pub fn route_search_streets(street: String) -> Result>, Status> { - let result = controller::search_streets(&street)?; - Ok(Json(result)) + Ok(Json(controller::search_streets(&street)?)) } #[get("/?&&&")] @@ -24,9 +21,7 @@ pub fn route_get_pickup_times( start_date: controller::BasicDate, end_date: controller::BasicDate, ) -> Result>, Status> { - match controller::get_pickup_times(street, number, start_date, end_date) { - // TODO provide more meaningful status codes here - Err(_) => Err(Status::InternalServerError), - Ok(times) => Ok(Json(times)), - } + Ok(Json(controller::get_pickup_times( + street, number, start_date, end_date, + )?)) }