[#28] Added basic ivago scraper

Cargo.toml

@@ -20,6 +20,14 @@ bench = true
 doc = true
 doctest = true
 
+[[bin]]
+name = "populate_ivago"
+path = "src/populate_ivago.rs"
+test = false
+bench = false
+doc = false
+doctest = false
+ 
 [dependencies]
 rocket = "0.4.7"
 serde = "1.0.124"

src/fej/ivago/db.rs

@@ -6,7 +6,7 @@ use diesel::PgConnection;
 
 pub fn search_streets(db_con: &PgConnection, search_term: &str) -> Result<Vec<Street>, FejError> {
     Ok(ivago_streets
-        .filter(name.like(search_term))
-        .or_filter(city.like(search_term))
+        .filter(name.ilike(format!("%{}%", search_term)))
+        .or_filter(city.ilike(format!("%{}%", search_term)))
         .load(db_con)?)
 }

src/fej/ivago/street.rs

@@ -1,4 +1,5 @@
-use diesel::Queryable;
+use crate::schema::ivago_streets;
+use diesel::{Insertable, Queryable};
 use regex::Regex;
 use rocket::http::RawStr;
 use rocket::request::FromFormValue;
@@ -6,7 +7,8 @@ use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::convert::TryFrom;
 
 /// Represents a street in a given city
-#[derive(Queryable)]
+#[derive(Queryable, Insertable)]
+#[table_name = "ivago_streets"]
 pub struct Street {
     name: String,
     city: String,

src/populate_ivago.rs

@@ -0,0 +1,19 @@
+use diesel::{insert_into, Connection, PgConnection, RunQueryDsl};
+use fej::ivago::search_streets;
+use fej::schema::ivago_streets::dsl::*;
+
+const ABC: &str = "abcdefghijklmnopqrstuvwxyz";
+
+fn main() {
+    let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
+
+    let db_conn = PgConnection::establish(&database_url)
+        .expect(&format!("Error connecting to {}", database_url));
+
+    // Doing this linearly is good enough I'd say
+    for c in ABC.chars() {
+        if let Ok(streets) = search_streets(&c.to_string()) {
+            insert_into(ivago_streets).values(streets).execute(&db_conn);
+        }
+    }
+}