From 85611e2d5929afdfb7c9c21f71f08f81166bef41 Mon Sep 17 00:00:00 2001
From: Jef Roosens <roosensjef@gmail.com>
Date: Sat, 17 Apr 2021 23:20:41 +0200
Subject: [PATCH] [#28] Added basic ivago scraper

---
 Cargo.toml              |  8 ++++++++
 src/fej/ivago/db.rs     |  4 ++--
 src/fej/ivago/street.rs |  6 ++++--
 src/populate_ivago.rs   | 19 +++++++++++++++++++
 4 files changed, 33 insertions(+), 4 deletions(-)
 create mode 100644 src/populate_ivago.rs

diff --git a/Cargo.toml b/Cargo.toml
index 4d1dd7e..4274991 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,6 +19,14 @@ test = true
 bench = true
 doc = true
 doctest = true
+
+[[bin]]
+name = "populate_ivago"
+path = "src/populate_ivago.rs"
+test = false
+bench = false
+doc = false
+doctest = false
  
 [dependencies]
 rocket = "0.4.7"
diff --git a/src/fej/ivago/db.rs b/src/fej/ivago/db.rs
index 43f94e9..6ba3174 100644
--- a/src/fej/ivago/db.rs
+++ b/src/fej/ivago/db.rs
@@ -6,7 +6,7 @@ use diesel::PgConnection;
 
 pub fn search_streets(db_con: &PgConnection, search_term: &str) -> Result<Vec<Street>, FejError> {
     Ok(ivago_streets
-        .filter(name.like(search_term))
-        .or_filter(city.like(search_term))
+        .filter(name.ilike(format!("%{}%", search_term)))
+        .or_filter(city.ilike(format!("%{}%", search_term)))
         .load(db_con)?)
 }
diff --git a/src/fej/ivago/street.rs b/src/fej/ivago/street.rs
index a4da4b5..2f30f06 100644
--- a/src/fej/ivago/street.rs
+++ b/src/fej/ivago/street.rs
@@ -1,4 +1,5 @@
-use diesel::Queryable;
+use crate::schema::ivago_streets;
+use diesel::{Insertable, Queryable};
 use regex::Regex;
 use rocket::http::RawStr;
 use rocket::request::FromFormValue;
@@ -6,7 +7,8 @@ use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::convert::TryFrom;
 
 /// Represents a street in a given city
-#[derive(Queryable)]
+#[derive(Queryable, Insertable)]
+#[table_name = "ivago_streets"]
 pub struct Street {
     name: String,
     city: String,
diff --git a/src/populate_ivago.rs b/src/populate_ivago.rs
new file mode 100644
index 0000000..9dfc876
--- /dev/null
+++ b/src/populate_ivago.rs
@@ -0,0 +1,19 @@
+use diesel::{insert_into, Connection, PgConnection, RunQueryDsl};
+use fej::ivago::search_streets;
+use fej::schema::ivago_streets::dsl::*;
+
+const ABC: &str = "abcdefghijklmnopqrstuvwxyz";
+
+fn main() {
+    let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
+
+    let db_conn = PgConnection::establish(&database_url)
+        .expect(&format!("Error connecting to {}", database_url));
+
+    // Doing this linearly is good enough I'd say
+    for c in ABC.chars() {
+        if let Ok(streets) = search_streets(&c.to_string()) {
+            insert_into(ivago_streets).values(streets).execute(&db_conn);
+        }
+    }
+}