From c8a06f7af9e0b2e5bb2d98d42b86730d78b021ed Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 23 Mar 2021 13:47:57 +0100 Subject: [PATCH 001/124] Started roadmap.md --- README.md | 4 ++++ roadmap.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 roadmap.md diff --git a/README.md b/README.md index d7dc9e6..bf60adc 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,7 @@ Each module contains the following base files: module directory if need be. Every module has a `routes` function that returns its route macros. + +## Roadmap + +See [roadmap.md](roadmap.md). diff --git a/roadmap.md b/roadmap.md new file mode 100644 index 0000000..17b0d76 --- /dev/null +++ b/roadmap.md @@ -0,0 +1,67 @@ +# Roadmap + +This file contains a listing of my current ideas/plans for this project. This +could change at any minute, but it serves as a quick overview of my plans for +the project. + +## 1.0: Ivago + +### Summary + +* Base project structure +* RESTful wrapper around Ivago's API for showing the pickup calendar + +### Description + +Version 1.0 won't be a very big one, but it's important nonetheless. It +contains a base project, on which I can build all the other modules. The only +real feature it'll have is a wrapper around the Ivago API. + +Ivago is one of the companies in Belgium that comes pick up the trash. On their +site, you can view a calendar for your street displaying when they'll come pick +up which kinds of trash. The sad part is that this site is very much not a +RESTful API, relying on cookies to know which street & city you entered. + +In one of my Web Development classes, I was given the assignment to convert +this API into a RESTful one. I ended up fully wrapping it up using Python, so +my goal is to port said code to Rust so we can then query my API for the same +information their website would give. This allows us to remain anonymous while +still collecting the same data. Furthermore, having the data available in +computer-readable format opens up ways to write a custom frontend for the API +(which I will be doing sometime in the near future). + +## 2.0: The Curse of the Forge + +### Summary + +* Curseforge site scraper +* Easy lookup of download links for modpacks & their mods +* Caching of Ivago API + +### Description + +Let's start off by saying I'm an avid Minecraft players. I still play at least +once a week, even after almost 8 years of playing. I also love doing stuff on +servers, so combining this two is quite easy: I host lots of Minecraft servers. +These servers often run modpacks, to spice up the game. This is where the +motvation for this version comes in. + +The Curseforge website hosts *a lot* of modpacks, but sadly, they're not +exactly in a computer-friendly format. Furthermore, the "server files" zips you +often see included with a modpack just contain a yaml file, containing +definitions for mods that only their launcher can use. I'm stubborn however, +and refuse to use their launcher, as it doesn't work with my Docker-based setup +scripts, and I just like doing things myself. Therefore, I want to do the +following things: + +* Periodically scrape their site for all modpacks +* Store all the information I'll ever need in a database +* Provide endpoints that can return a list of mods to download & the config + directory + +I could then possibly integrate my API into my own scripts, allowing me to spin +up a modded Minecraft server from the command line, without having to ever +touch their websites or services (besides downloading the mods of course). + +As a bonus, I want to cache the Ivago API's calendars, considering I'll be +setting up a database for this version anyways. From 93ec27bb02ff901f3e53bf13fb1761c670dbf765 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Apr 2021 17:39:52 +0200 Subject: [PATCH 002/124] [#17] Wrote dockerfiles; moved everything to Docker --- Dockerfile | 37 ---------------------- Makefile | 25 ++++++++------- build | 66 ++++++++++++++++++++++++++++++--------- docker/Dockerfile.builder | 19 +++++++++++ docker/Dockerfile.dev | 6 ++++ docker/Dockerfile.rel | 22 +++++++++++++ 6 files changed, 112 insertions(+), 63 deletions(-) delete mode 100644 Dockerfile create mode 100644 docker/Dockerfile.builder create mode 100644 docker/Dockerfile.dev create mode 100644 docker/Dockerfile.rel diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c54bb86..0000000 --- a/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# syntax = docker/dockerfile:1.2 - -# We use a multi-stage build to end up with a very small final image -FROM alpine:latest AS builder - -ENV PATH "$PATH:/root/.cargo/bin" - -WORKDIR /usr/src/app - -# Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev build-base curl && \ - { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } - -# Copy source code over to builder -COPY Cargo.toml Cargo.lock ./ -COPY src/ ./src/ - -# Run the tests, don't want no broken docker image -# And then finally, build the project -# Thank the lords that this article exists -# https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 -# TODO add what these flags do & why they work -# NOTE: cargo install auto-appends bin to the path -RUN --mount=type=cache,target=/usr/src/app/target RUSTFLAGS="-C target-feature=-crt-static" cargo test && \ - RUSTFLAGS="-C target-feature=-crt-static" cargo install --path . --bin fej --root /usr/local - - -# Now, we create the actual image -FROM alpine:latest - -# Install some dynamic libraries needed for everything to work -RUN apk update && apk add --no-cache openssl libgcc - -# Copy binary over to final image -COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej - -CMD ["/usr/local/bin/fej"] diff --git a/Makefile b/Makefile index ab50282..c276d79 100644 --- a/Makefile +++ b/Makefile @@ -1,35 +1,36 @@ -IMAGE := chewingbever/fej - - all: debug .PHONY: all # Builds debug: - @ cargo build + @ ./build -m dev .PHONY: debug release: - @ cargo build --release + @ ./build -m rel .PHONY: release -image: Dockerfile - @ ./build '$(IMAGE)' -.PHONY: image - push: - @ ./build '$(IMAGE)' push + @ ./build -m prod -a push .PHONY: push # Run run: - @ RUST_BACKTRACE=1 cargo run --bin fej + @ ./build -m dev -a run .PHONY: run +stop: + @ docker stop -t 2 fej +.PHONY: stop + +logs: + @ docker logs -f fej +.PHONY: logs + # Testing test: - @ cargo test --no-fail-fast + @ ./build -m dev -a run -l -- test --no-fail-fast .PHONY: test format: diff --git a/build b/build index 8abbc39..32b04aa 100755 --- a/build +++ b/build @@ -1,10 +1,21 @@ #!/usr/bin/env bash -# Simple guard to check input args -[[ $# -eq 1 ]] || [[ $# -eq 2 ]] || { - >&2 echo "Usage: ./build IMAGE [ACTION]" - exit 1 -} +image="chewingbever/fej" +# Should be either dev or rel +mode="dev" +action="" +attach="--detach" + +while getopts ":i:m:a:l" c; do + case $c in + i ) image="$OPTARG" ;; + m ) mode="$OPTARG" ;; + a ) action="$OPTARG" ;; + l ) attach="" ;; + ? ) exit 1 ;; + esac +done +shift $((OPTIND-1)) # Extract current version from Cargo.toml & get current branch patch_version="$(grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1)" @@ -25,31 +36,58 @@ else fi -# Run the actual build command -DOCKER_BUILDKIT=1 docker build -t "$1:$tags" . +# First, we build the builder +DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" . -if [[ "$2" = push ]]; then +# Run the actual build command +if [ "$mode" = "rel" ]; then + DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel . + +elif [[ "$mode" = "dev" ]]; then + DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev . + +else + >&2 echo "Invalid mode." + exit 1 + +fi + +if [[ "$action" = push ]]; then [[ "$branch" =~ ^develop|master$ ]] || { >&2 echo "You can only push from develop or master." exit 2 } + [[ "$mode" = "rel" ]] || { + >&2 echo "You can only push release builds." + exit 3 + } + for tag in "${tags[@]}"; do # Create the tag - docker tag "$1:$tags" "$1:$tag" + docker tag "$image:$tags" "$image:$tag" # Push the tag - docker push "$1:$tag" + docker push "$image:$tag" # Remove the tag again, if it's not the main tag - [[ "$tag" != "$tags" ]] && docker rmi "$1:$tag" + [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" done - elif [[ "$2" = run ]]; then - docker run \ + elif [[ "$action" = run ]]; then + if [[ "$mode" = "dev" ]]; then + # Create caching volumes if needed (they need to be named) + docker volume create fej_build-cache + docker volume create fej_registry-cache + + flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry" + fi + + docker run $attach $flags \ --rm \ --interactive \ --tty \ --publish 8000:8000 \ - "$1:$tags" + --name fej \ + "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" fi diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder new file mode 100644 index 0000000..84d09ff --- /dev/null +++ b/docker/Dockerfile.builder @@ -0,0 +1,19 @@ +# We use a multi-stage build to end up with a very small final image +FROM alpine:latest AS builder + +ARG MODE +ARG RUN_TESTS + +ENV PATH "$PATH:/root/.cargo/bin" +# Needed for proper compiling of openssl-dev +ENV RUSTFLAGS="-C target-feature=-crt-static" + +WORKDIR /usr/src/app + +# Install build dependencies, rustup & rust's nightly build & toolchain +RUN apk update && apk add --no-cache openssl-dev build-base curl && \ + { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } + +# Copy source code over to builder +COPY Cargo.toml Cargo.lock ./ +COPY src/ ./src/ diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 0000000..a1a1e92 --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,6 @@ +FROM chewingbever/fej-builder:latest + +ENV RUST_BACKTRACE=1 + +ENTRYPOINT ["cargo"] +CMD ["run"] diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel new file mode 100644 index 0000000..4d40ef2 --- /dev/null +++ b/docker/Dockerfile.rel @@ -0,0 +1,22 @@ +FROM chewingbever/fej-builder:latest AS builder + +# And then finally, build the project +# Thank the lords that this article exists +# https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 +# TODO add what these flags do & why they work +# NOTE: cargo install auto-appends bin to the path +RUN --mount=type=cache,target=/usr/src/app/target \ + --mount=type=cache,target=/root/.cargo/registry \ + cargo install --path . --bin fej --root /usr/local + + +# Now, we create the actual image +FROM alpine:latest + +# Install some dynamic libraries needed for everything to work +RUN apk update && apk add --no-cache openssl libgcc + +# Copy binary over to final image +COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej + +CMD ["/usr/local/bin/fej"] From 34b32a3252ab1029b28c3e7741c8b0e17fa17c29 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Apr 2021 18:19:22 +0200 Subject: [PATCH 003/124] [#17] Small changes to build script --- build | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/build b/build index 32b04aa..d7d86a3 100755 --- a/build +++ b/build @@ -18,12 +18,10 @@ done shift $((OPTIND-1)) # Extract current version from Cargo.toml & get current branch -patch_version="$(grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1)" -major_version="$(echo "$patch_version" | - sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/')" -minor_version="$(echo "$patch_version" | - sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/')" -branch="$(git branch --show-current)" +patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` +major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` +minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` +branch=`git rev-parse --abbrev-ref HEAD` if [[ "$branch" = "master" ]]; then tags=("$patch_version" "$minor_version" "$major_version" "latest") From 6d4cf6feb6aad6657d52075bb3a77978acd94be6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Apr 2021 21:33:56 +0200 Subject: [PATCH 004/124] [closes #6] added documentation everywhere --- .hooks/pre-commit | 16 ++++++++-------- src/errors.rs | 3 +++ src/ivago/README.md | 5 ----- src/ivago/controller/mod.rs | 19 +++++++++---------- src/ivago/controller/pickup_time.rs | 10 ++++++++-- src/ivago/controller/street.rs | 3 ++- src/ivago/mod.rs | 21 ++++++++++++++++++--- 7 files changed, 48 insertions(+), 29 deletions(-) delete mode 100644 src/ivago/README.md diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 825db47..709fb95 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -6,12 +6,12 @@ make lint &> /dev/null 2>&1 || { exit 1; } -branch=`git rev-parse --abbrev-ref HEAD` +# branch=`git rev-parse --abbrev-ref HEAD` -# TODO should we add release branches here as well? -if [[ "$branch" =~ ^master|develop$ ]]; then - make test > /dev/null 2>&1 || { - >&2 echo "Tests failed. check 'make test' for more info."; - exit 1; - } -fi +# # TODO should we add release branches here as well? +# if [[ "$branch" =~ ^master|develop$ ]]; then +# make test > /dev/null 2>&1 || { +# >&2 echo "Tests failed. check 'make test' for more info."; +# exit 1; +# } +# fi diff --git a/src/errors.rs b/src/errors.rs index dfb137f..afcf04a 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,6 +1,9 @@ // I can probably do this way easier using an external crate, I should do that use rocket::http::Status; +/// Represents any general error that the API can encounter during execution. +/// It allows us to more easily process errors, and hopefully allows for +/// more clear error messages. #[derive(Debug, PartialEq)] pub enum FejError { InvalidArgument, diff --git a/src/ivago/README.md b/src/ivago/README.md deleted file mode 100644 index 02d2d0e..0000000 --- a/src/ivago/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Ivago - -This part of the API is a wrapper around the Ivago website (Ivago being the -company that collects the trash in my city). Their site isn't exactly RESTful, -so this endpoint simply wraps it in a RESTful wrapper. diff --git a/src/ivago/controller/mod.rs b/src/ivago/controller/mod.rs index 0084a3d..7d3a9f9 100644 --- a/src/ivago/controller/mod.rs +++ b/src/ivago/controller/mod.rs @@ -20,15 +20,14 @@ const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; /// Endpoint for the actual calendar output const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; -/// Searches the Ivago API for streets in the given city +/// Searches the Ivago API for streets in the given city. /// /// # Arguments /// -/// * `street` - name of the street -/// * `city` - city the street is in -pub fn search_streets(street_name: &str) -> Result, FejError> { +/// * `search_term` - Search term to use to look for streets +pub fn search_streets(search_term: &str) -> Result, FejError> { let client = reqwest::Client::new(); - let response = client.get(SEARCH_URL).query(&[("q", street_name)]).send()?; + let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; let data: Vec> = response.json()?; // This is pretty cool, filter_map first does get() on all the maps, and @@ -41,14 +40,14 @@ pub fn search_streets(street_name: &str) -> Result, FejError> { .collect()) } -/// Returns the pickup times for the various trash types +/// Returns the pickup times for the various trash types. /// /// # Arguments /// -/// * `street` - desired street -/// * `number` - house number -/// * `start_date` - earliest date for the results -/// * `end_date` - latest date for the results +/// * `street` - Street to look up +/// * `number` - House number in given street +/// * `start_date` - Earliest date for the results +/// * `end_date` - Latest date for the results pub fn get_pickup_times( street: &Street, number: &u32, diff --git a/src/ivago/controller/pickup_time.rs b/src/ivago/controller/pickup_time.rs index 33718db..f70cbcf 100644 --- a/src/ivago/controller/pickup_time.rs +++ b/src/ivago/controller/pickup_time.rs @@ -1,14 +1,20 @@ use super::BasicDate; use serde::ser::{Serialize, SerializeStruct, Serializer}; -/// Represents a pickup time instance. All fields are a direct map of the -/// original API +/// Represents a date when a pickup will occur. Label describes which type of +/// trash will be picked up. pub struct PickupTime { date: BasicDate, label: String, } impl PickupTime { + /// Creates a new PickupTime instance. + /// + /// # Arguments + /// + /// * `date` - Date of pickup time + /// * `label` - Type of trash pub fn new(date: BasicDate, label: String) -> PickupTime { PickupTime { date: date, diff --git a/src/ivago/controller/street.rs b/src/ivago/controller/street.rs index 00f4de4..1a7cac9 100644 --- a/src/ivago/controller/street.rs +++ b/src/ivago/controller/street.rs @@ -4,7 +4,7 @@ use rocket::request::FromFormValue; use serde::ser::{Serialize, SerializeStruct, Serializer}; use std::convert::TryFrom; -/// Represents a street +/// Represents a street in a given city pub struct Street { name: String, city: String, @@ -76,6 +76,7 @@ impl<'v> FromFormValue<'v> for Street { mod tests { use super::*; + /// Tests the conversion to string #[test] fn test_to_string() { let street = Street::new(String::from("testname"), String::from("city")); diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index e4d1037..0e7bebc 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -9,11 +9,26 @@ pub fn routes() -> Vec { routes![route_search_streets, route_get_pickup_times] } -#[get("/search?")] -pub fn route_search_streets(street: String) -> Result>, Status> { - Ok(Json(search_streets(street.as_str())?)) +/// This route handles the Ivago search endpoint. It returns a list of streets, +/// consisting of a street name & a city. +/// +/// # Arguments +/// +/// * `search_term` - Search term to use to look for streets +#[get("/search?")] +pub fn route_search_streets(search_term: String) -> Result>, Status> { + Ok(Json(search_streets(search_term.as_str())?)) } +/// Handles returning of pickup times for a specific address. It returns a list +/// of pickup times, containing a date and a description of the trash type. +/// +/// # Arguments +/// +/// * `street` - Street to look up +/// * `number` - House number in the given street +/// * `start_date` - Earliest date that can be returned +/// * `end_date` - Latest date that can be returned #[get("/?&&&")] pub fn route_get_pickup_times( street: Street, From 552cf4bf7ba1e0892c62b756ea54b3bc8b768960 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Apr 2021 22:55:52 +0200 Subject: [PATCH 005/124] Added healthcheck to release container --- docker/Dockerfile.builder | 5 ----- docker/Dockerfile.rel | 12 +++++++++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index 84d09ff..c2fb526 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -1,12 +1,7 @@ # We use a multi-stage build to end up with a very small final image FROM alpine:latest AS builder -ARG MODE -ARG RUN_TESTS - ENV PATH "$PATH:/root/.cargo/bin" -# Needed for proper compiling of openssl-dev -ENV RUSTFLAGS="-C target-feature=-crt-static" WORKDIR /usr/src/app diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 4d40ef2..e872ab1 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -1,5 +1,8 @@ FROM chewingbever/fej-builder:latest AS builder +# Needed for proper compiling of openssl-dev +ENV RUSTFLAGS="-C target-feature=-crt-static" + # And then finally, build the project # Thank the lords that this article exists # https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 @@ -14,9 +17,16 @@ RUN --mount=type=cache,target=/usr/src/app/target \ FROM alpine:latest # Install some dynamic libraries needed for everything to work -RUN apk update && apk add --no-cache openssl libgcc +RUN apk update && apk add --no-cache openssl libgcc curl # Copy binary over to final image COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej +HEALTHCHECK \ + --interval=10s \ + --timeout=5s \ + --start-period=1s \ + --retries=3 \ + CMD curl -q localhost:8000 + CMD ["/usr/local/bin/fej"] From 3b5f1e9cf7e59429163e879c001c12dfb3fcf3ce Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Mon, 12 Apr 2021 23:03:15 +0200 Subject: [PATCH 006/124] [#19] Small changes --- docker/Dockerfile.dev | 3 ++- src/ivago/mod.rs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index a1a1e92..8ebeab3 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,6 +1,7 @@ FROM chewingbever/fej-builder:latest -ENV RUST_BACKTRACE=1 +ENV RUST_BACKTRACE 1 +ENV ROCKET_LOG debug ENTRYPOINT ["cargo"] CMD ["run"] diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index 0e7bebc..627e89b 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -15,9 +15,9 @@ pub fn routes() -> Vec { /// # Arguments /// /// * `search_term` - Search term to use to look for streets -#[get("/search?")] -pub fn route_search_streets(search_term: String) -> Result>, Status> { - Ok(Json(search_streets(search_term.as_str())?)) +#[get("/search?")] +pub fn route_search_streets(q: String) -> Result>, Status> { + Ok(Json(search_streets(q.as_str())?)) } /// Handles returning of pickup times for a specific address. It returns a list From 4a32349c9cfac368756d47d5828d7ee34f72a068 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Apr 2021 21:43:30 +0200 Subject: [PATCH 007/124] Fixed instant crash on ivago endpoint (#19) --- Makefile | 2 +- docker/Dockerfile.builder | 4 +++- docker/Dockerfile.rel | 3 --- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index c276d79..f7eaa8e 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ all: debug # Builds debug: - @ ./build -m dev + @ ./build -m dev -a run build .PHONY: debug release: diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index c2fb526..62b986e 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -1,7 +1,9 @@ -# We use a multi-stage build to end up with a very small final image +# Our entire toolchain runs in alpine FROM alpine:latest AS builder ENV PATH "$PATH:/root/.cargo/bin" +# Needed for proper compiling of openssl-dev +ENV RUSTFLAGS "-C target-feature=-crt-static" WORKDIR /usr/src/app diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index e872ab1..6b4e438 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -1,8 +1,5 @@ FROM chewingbever/fej-builder:latest AS builder -# Needed for proper compiling of openssl-dev -ENV RUSTFLAGS="-C target-feature=-crt-static" - # And then finally, build the project # Thank the lords that this article exists # https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 From 760e9fa584d472c02f8532f97349ee1652d0cfe5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Apr 2021 21:56:52 +0200 Subject: [PATCH 008/124] [#19] Fixed api not working from outside container --- docker/Dockerfile.builder | 2 ++ docker/Dockerfile.dev | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index 62b986e..b968c5b 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -4,6 +4,8 @@ FROM alpine:latest AS builder ENV PATH "$PATH:/root/.cargo/bin" # Needed for proper compiling of openssl-dev ENV RUSTFLAGS "-C target-feature=-crt-static" +# Otherwise, the debug build can't be used from the container +ENV ROCKET_ADDRESS "0.0.0.0" WORKDIR /usr/src/app diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 8ebeab3..9205fd4 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,7 +1,6 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 -ENV ROCKET_LOG debug ENTRYPOINT ["cargo"] CMD ["run"] From 00926162fa1c08d2f19072124a50e9ea4989c665 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Apr 2021 22:02:21 +0200 Subject: [PATCH 009/124] Updated depencies --- Cargo.lock | 165 ++++++++++++++++++++++++++--------------------------- 1 file changed, 82 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 532c627..d7e99a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" +checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28" [[package]] name = "cookie" @@ -438,42 +438,42 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures-channel" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" +checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" +checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" [[package]] name = "futures-io" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" +checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" [[package]] name = "futures-sink" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" +checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" [[package]] name = "futures-task" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" +checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" [[package]] name = "futures-util" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" +checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" dependencies = [ "futures-core", "futures-io", @@ -523,9 +523,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" +checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00" dependencies = [ "bytes", "fnv", @@ -577,9 +577,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" dependencies = [ "bytes", "fnv", @@ -599,9 +599,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.5" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" +checksum = "bc35c995b9d93ec174cf9a27d425c7892722101e14993cd227fdb51d70cf9589" [[package]] name = "httpdate" @@ -630,9 +630,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.4" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7" +checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1" dependencies = [ "bytes", "futures-channel", @@ -659,7 +659,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.4", + "hyper 0.14.5", "native-tls", "tokio", "tokio-native-tls", @@ -740,9 +740,9 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "js-sys" -version = "0.3.49" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc15e39392125075f60c95ba416f5381ff6c3a948ff02ab12464715adf56c821" +checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c" dependencies = [ "wasm-bindgen", ] @@ -777,9 +777,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.91" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7" +checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" [[package]] name = "log" @@ -1066,22 +1066,22 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" +checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" +checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.69", ] [[package]] @@ -1136,9 +1136,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" dependencies = [ "unicode-xid 0.2.1", ] @@ -1168,7 +1168,7 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.26", ] [[package]] @@ -1248,9 +1248,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf12057f289428dbf5c591c74bf10392e4a8003f993405a902f20117019022d4" +checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" dependencies = [ "base64 0.13.0", "bytes", @@ -1261,7 +1261,7 @@ dependencies = [ "futures-util", "http", "http-body", - "hyper 0.14.4", + "hyper 0.14.5", "hyper-tls", "ipnet", "js-sys", @@ -1392,9 +1392,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d493c5f39e02dfb062cd8f33301f90f9b13b650e8c1b1d0fd75c19dd64bff69d" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" dependencies = [ "bitflags", "core-foundation", @@ -1405,9 +1405,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee48cdde5ed250b0d3252818f646e174ab414036edb884dde62d80a3ac6082d" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" dependencies = [ "core-foundation-sys", "libc", @@ -1443,9 +1443,9 @@ version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.69", ] [[package]] @@ -1504,11 +1504,10 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "socket2" -version = "0.3.19" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" dependencies = [ - "cfg-if 1.0.0", "libc", "winapi 0.3.9", ] @@ -1548,11 +1547,11 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.26", "quote 1.0.9", "serde", "serde_derive", - "syn 1.0.64", + "syn 1.0.69", ] [[package]] @@ -1562,13 +1561,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ "base-x", - "proc-macro2 1.0.24", + "proc-macro2 1.0.26", "quote 1.0.9", "serde", "serde_derive", "serde_json", "sha1", - "syn 1.0.64", + "syn 1.0.69", ] [[package]] @@ -1596,11 +1595,11 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f" +checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.26", "quote 1.0.9", "unicode-xid 0.2.1", ] @@ -1662,17 +1661,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" dependencies = [ "proc-macro-hack", - "proc-macro2 1.0.24", + "proc-macro2 1.0.26", "quote 1.0.9", "standback", - "syn 1.0.64", + "syn 1.0.69", ] [[package]] name = "tinyvec" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" dependencies = [ "tinyvec_macros", ] @@ -1685,9 +1684,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134af885d758d645f0f0505c9a8b3f9bf8a348fd822e112ab5248138348f1722" +checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" dependencies = [ "autocfg", "bytes", @@ -1710,9 +1709,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5143d049e85af7fbc36f5454d990e62c2df705b3589f123b71f441b6b59f443f" +checksum = "940a12c99365c31ea8dd9ba04ec1be183ffe4920102bb7122c2f515437601e8e" dependencies = [ "bytes", "futures-core", @@ -1792,9 +1791,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" dependencies = [ "matches", ] @@ -1900,9 +1899,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.72" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe8f61dba8e5d645a4d8132dc7a0a66861ed5e1045d2c0ed940fab33bac0fbe" +checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" dependencies = [ "cfg-if 1.0.0", "serde", @@ -1912,24 +1911,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.72" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ceba58ff062da072c7cb4ba5b22a37f00a302483f7e2a6cdc18fedbdc1fd3" +checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae" dependencies = [ "bumpalo", "lazy_static", "log 0.4.14", - "proc-macro2 1.0.24", + "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.69", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73157efb9af26fb564bb59a009afd1c7c334a44db171d280690d0c3faaec3468" +checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -1939,9 +1938,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.72" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9aa01d36cda046f797c57959ff5f3c615c9cc63997a8d545831ec7976819b" +checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f" dependencies = [ "quote 1.0.9", "wasm-bindgen-macro-support", @@ -1949,28 +1948,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.72" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96eb45c1b2ee33545a813a92dbb53856418bf7eb54ab34f7f7ff1448a5b3735d" +checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" dependencies = [ - "proc-macro2 1.0.24", + "proc-macro2 1.0.26", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.69", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.72" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7148f4696fb4960a346eaa60bbfb42a1ac4ebba21f750f75fc1375b098d5ffa" +checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489" [[package]] name = "web-sys" -version = "0.3.49" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fe19d70f5dacc03f6e46777213facae5ac3801575d56ca6cbd4c93dcd12310" +checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" dependencies = [ "js-sys", "wasm-bindgen", From cfa0a8cf22420b90a7e34518b7b7f6b7e9ff72d6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Apr 2021 22:03:27 +0200 Subject: [PATCH 010/124] [closes #20] Switched to AGPLv3 --- LICENSE | 143 ++++++++++++++++++++++++++------------------------------ 1 file changed, 65 insertions(+), 78 deletions(-) diff --git a/LICENSE b/LICENSE index f288702..0ad25db 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies @@ -7,17 +7,15 @@ Preamble - The GNU General Public License is a free, copyleft license for -software and other kinds of works. + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to +our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. +software for all its users. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you @@ -26,44 +24,34 @@ them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. The precise terms and conditions for copying, distribution and modification follow. @@ -72,7 +60,7 @@ modification follow. 0. Definitions. - "This License" refers to version 3 of the GNU General Public License. + "This License" refers to version 3 of the GNU Affero General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. @@ -549,35 +537,45 @@ to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. - 13. Use with the GNU Affero General Public License. + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single +under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General +Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published +GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's +versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. @@ -635,40 +633,29 @@ the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. + GNU Affero General Public License for more details. - You should have received a copy of the GNU General Public License + You should have received a copy of the GNU Affero General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see +For more information on this, and how to apply and follow the GNU AGPL, see . - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. From 75ffdfdc278d0f805cb9737719811970018778c6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Apr 2021 22:05:31 +0200 Subject: [PATCH 011/124] Bumped version to 1.0.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7e99a4..0606bc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -347,7 +347,7 @@ dependencies = [ [[package]] name = "fej" -version = "0.1.0" +version = "1.0.0" dependencies = [ "chrono", "chrono-tz", diff --git a/Cargo.toml b/Cargo.toml index 97796ac..12add75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fej" -version = "0.1.0" +version = "1.0.0" authors = ["Jef Roosens "] edition = "2018" From bdd27446eb08ba3cd5888e25003169229a1d9af3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Apr 2021 22:09:50 +0200 Subject: [PATCH 012/124] Tiny fix in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f7eaa8e..fee2643 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ release: .PHONY: release push: - @ ./build -m prod -a push + @ ./build -m rel -a push .PHONY: push # Run From 5f98b659029d3cf3851cbff46d4f2404d920acd2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Apr 2021 22:35:39 +0200 Subject: [PATCH 013/124] Added lots of documentation --- Makefile | 30 ++++++++++++++++++++++++------ README.md | 16 ++++++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index fee2643..6ad1f50 100644 --- a/Makefile +++ b/Makefile @@ -1,48 +1,66 @@ all: debug .PHONY: all -# Builds +# Builds the debug release inside the Alpine container. For build caching, two +# volumes are used named `fej_build-cache` and `fej_registry-cache`. These +# images are automatically created for you if they don't exist. If you +# encounter any strange build errors, you can try removing this volumes to +# start a completely fresh build. debug: @ ./build -m dev -a run build .PHONY: debug +# Builds the release version. In contrary to the debug version, this build +# doesn't use volumes for caching, as this would require the build to happen +# during runtime instead of during the building of the image. Instead, it uses +# the new `--mount` feature from Buildkit. This does mean that only very recent +# Docker engines can build the release version (in my case, at the time of +# writing this, 20.10.5). release: @ ./build -m rel .PHONY: release +# This builds the release version, and pushes all relevant tags to my Docker +# Hub repository, namely chewingbever/fej push: @ ./build -m rel -a push .PHONY: push -# Run +# This builds the debug release, and runs it detached. The reason we detach the +# container is because Rocket has a tendency to ignore ctlr-c when inside a +# container, which gets annoying really fast. run: @ ./build -m dev -a run .PHONY: run +# As a workaround, we just have a stop command that stops the container. stop: @ docker stop -t 2 fej .PHONY: stop +# This attaches to the running container, essentially giving the same result as +# just running `cargo run` locally. logs: @ docker logs -f fej .PHONY: logs - -# Testing +# Builds the debug version, and runs the tests (but doesn't detach). test: @ ./build -m dev -a run -l -- test --no-fail-fast .PHONY: test +# Runs the cargo code formatter on your code. format: @ cargo fmt .PHONY: format +# Lints your code. This also gets run in the pre-commit hook, basically +# preventing you from committing badly-formatted code. lint: @ cargo fmt -- --check .PHONY: lint - -# Documentation +# This builds the documentation for the project, excluding the documentation. docs: @ cargo doc --no-deps .PHONY: docs diff --git a/README.md b/README.md index 3f8e6f4..1961201 100644 --- a/README.md +++ b/README.md @@ -24,3 +24,19 @@ Every module has a `routes` function that returns its route macros. ## Roadmap See [roadmap.md](roadmap.md). + +## Development + +The entire toolchain runs on Alpine inside Docker. This makes building easier, +and (hopefully) eliminates any system-specific bugs. + +A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as +all the commands are documented for your understanding ;) + +There's also the `build` script. This script does all the "heavy" lifting. It +chooses which Dockerfile to build according to the given arguments, and +generates tags for the images (useful when pushing releases). The Makefile is +really just a wrapper around this build script, allowing you to write +`make test` instead of `./build -m dev -a run test`. + +tl;dr run `make run` to run your build, and `make test` to run the tests. From 452683f4d7360c77a81e6057b9bdf83248fa6aec Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Tue, 13 Apr 2021 23:55:22 +0200 Subject: [PATCH 014/124] Added docker images link to README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 1961201..55d11e3 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,7 @@ really just a wrapper around this build script, allowing you to write `make test` instead of `./build -m dev -a run test`. tl;dr run `make run` to run your build, and `make test` to run the tests. + +## Docker images + +The images are available [here](https://hub.docker.com/r/chewingbever/fej). From 9ee542197dfe1586984f65000b096655a1df4edb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 14 Apr 2021 00:00:29 +0200 Subject: [PATCH 015/124] Added kissanime to roadmap --- roadmap.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/roadmap.md b/roadmap.md index 17b0d76..f03e4b5 100644 --- a/roadmap.md +++ b/roadmap.md @@ -4,7 +4,7 @@ This file contains a listing of my current ideas/plans for this project. This could change at any minute, but it serves as a quick overview of my plans for the project. -## 1.0: Ivago +## Ivago ### Summary @@ -30,7 +30,7 @@ still collecting the same data. Furthermore, having the data available in computer-readable format opens up ways to write a custom frontend for the API (which I will be doing sometime in the near future). -## 2.0: The Curse of the Forge +## The Curse of the Forge ### Summary @@ -65,3 +65,12 @@ touch their websites or services (besides downloading the mods of course). As a bonus, I want to cache the Ivago API's calendars, considering I'll be setting up a database for this version anyways. + +## Kissanime + +I like watching anime from time to time, and I've always used Kissanime for +this. However, their sites can be quite slow, and riddled with ads from time to +time. That's why I'd like to create a high-speed wrapper that extracts all the +needed info from their sites, removing the need for the user to ever actually +visit their site. The API can just act as a fast search index, complete with +indexing of the links to the videos and everything. From 87f2659b4800a5f9040c5363718b176e4e2284df Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 14 Apr 2021 22:00:56 +0200 Subject: [PATCH 016/124] Added temporary CORS fix --- src/main.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/main.rs b/src/main.rs index ad86d98..8b8edc7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,8 +3,36 @@ extern crate rocket; use fej_lib::{catchers, ivago}; +// Very temporary solution for CORS +// https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs +use rocket::fairing::{Fairing, Info, Kind}; +use rocket::http::Header; +use rocket::{Request, Response}; + +pub struct CORS; + +impl Fairing for CORS { + fn info(&self) -> Info { + Info { + name: "Add CORS headers to responses", + kind: Kind::Response, + } + } + + fn on_response(&self, _: &Request, response: &mut Response) { + response.set_header(Header::new("Access-Control-Allow-Origin", "*")); + response.set_header(Header::new( + "Access-Control-Allow-Methods", + "POST, GET, PATCH, OPTIONS", + )); + response.set_header(Header::new("Access-Control-Allow-Headers", "*")); + response.set_header(Header::new("Access-Control-Allow-Credentials", "true")); + } +} + fn rocket() -> rocket::Rocket { rocket::ignite() + .attach(CORS) .mount("/ivago", ivago::routes()) .register(catchers![catchers::not_found]) } From c9b7714a47576d21c0367d44dba475f892ffe90a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Wed, 14 Apr 2021 23:26:53 +0200 Subject: [PATCH 017/124] Bumped version --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0606bc8..8ab2a9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -347,7 +347,7 @@ dependencies = [ [[package]] name = "fej" -version = "1.0.0" +version = "1.0.1" dependencies = [ "chrono", "chrono-tz", diff --git a/Cargo.toml b/Cargo.toml index 12add75..83623fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fej" -version = "1.0.0" +version = "1.0.1" authors = ["Jef Roosens "] edition = "2018" From 5b515ecd91cff5bfdc0d2ec33342b0e654e2d2a1 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 17:21:32 +0200 Subject: [PATCH 018/124] [#23] Build script now starts postgres database as well --- Makefile | 6 ++++++ build | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6ad1f50..820f893 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,7 @@ run: # As a workaround, we just have a stop command that stops the container. stop: @ docker stop -t 2 fej + @ docker stop fej_db .PHONY: stop # This attaches to the running container, essentially giving the same result as @@ -44,6 +45,11 @@ logs: @ docker logs -f fej .PHONY: logs +# This just starts up a shell inside the fej container +sh: + @ docker exec -it fej sh +.PHONY: sh + # Builds the debug version, and runs the tests (but doesn't detach). test: @ ./build -m dev -a run -l -- test --no-fail-fast diff --git a/build b/build index d7d86a3..484a8ab 100755 --- a/build +++ b/build @@ -72,7 +72,21 @@ if [[ "$action" = push ]]; then [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" done - elif [[ "$action" = run ]]; then +elif [[ "$action" = run ]]; then + # Create the network & start the database container + docker network create fej + docker volume create fej_db-data + + docker run --rm \ + --detach \ + --name fej_db \ + --network fej \ + -e "POSTGRES_DB=fej" \ + -e "POSTGRES_USER=fej" \ + -e "POSTGRES_PASSWORD=fej" \ + -v 'fej_db-data:/var/lib/postgresql/data' \ + postgres:13-alpine + if [[ "$mode" = "dev" ]]; then # Create caching volumes if needed (they need to be named) docker volume create fej_build-cache @@ -87,5 +101,6 @@ if [[ "$action" = push ]]; then --tty \ --publish 8000:8000 \ --name fej \ + --network fej \ "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" fi From 6954921df622bdf5cd456154c407330be7fca837 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 17:31:45 +0200 Subject: [PATCH 019/124] [#23] Added needed env vars --- .env | 1 + build | 1 + 2 files changed, 2 insertions(+) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..880c44c --- /dev/null +++ b/.env @@ -0,0 +1 @@ +DATABASE_URL=postgres://fej:fej@fej_db:5432/fej diff --git a/build b/build index 484a8ab..962d14c 100755 --- a/build +++ b/build @@ -101,6 +101,7 @@ elif [[ "$action" = run ]]; then --tty \ --publish 8000:8000 \ --name fej \ + --env-file .env \ --network fej \ "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" fi From 727589b10f74c318e472bb3d02c899ff8ddcf6cb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 17:53:35 +0200 Subject: [PATCH 020/124] [#24] Initialized diesel project --- .env | 3 +- .env.container | 2 ++ Cargo.lock | 33 +++++++++++++++++ Cargo.toml | 7 ++-- build | 3 +- diesel.toml | 5 +++ migrations/.gitkeep | 0 .../down.sql | 6 ++++ .../up.sql | 36 +++++++++++++++++++ 9 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 .env.container create mode 100644 diesel.toml create mode 100644 migrations/.gitkeep create mode 100644 migrations/00000000000000_diesel_initial_setup/down.sql create mode 100644 migrations/00000000000000_diesel_initial_setup/up.sql diff --git a/.env b/.env index 880c44c..6e2ca99 100644 --- a/.env +++ b/.env @@ -1 +1,2 @@ -DATABASE_URL=postgres://fej:fej@fej_db:5432/fej +# This file is read by diesel +DATABASE_URL=postgres://fej:fej@localhost:5432/fej diff --git a/.env.container b/.env.container new file mode 100644 index 0000000..8b395fb --- /dev/null +++ b/.env.container @@ -0,0 +1,2 @@ +# This file is read by the container, and therefore by fej +DATABASE_URL=postgres://fej:fej@fej_db:5432/fej diff --git a/Cargo.lock b/Cargo.lock index 8ab2a9c..0006402 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -321,6 +321,29 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "diesel" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "047bfc4d5c3bd2ef6ca6f981941046113524b9a9f9a7cbdfdd7ff40f58e6f542" +dependencies = [ + "bitflags", + "byteorder", + "diesel_derives", + "pq-sys", +] + +[[package]] +name = "diesel_derives" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" +dependencies = [ + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", +] + [[package]] name = "digest" version = "0.9.0" @@ -351,6 +374,7 @@ version = "1.0.1" dependencies = [ "chrono", "chrono-tz", + "diesel", "regex", "reqwest", "rocket", @@ -1119,6 +1143,15 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "pq-sys" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda" +dependencies = [ + "vcpkg", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" diff --git a/Cargo.toml b/Cargo.toml index 83623fa..f359a51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,11 +25,8 @@ serde = "1.0.124" chrono = "0.4.19" chrono-tz = "0.5.3" regex = "1.4.5" - -[dependencies.reqwest] -version = "0.11.2" -default-features = true -features = ["blocking", "json", "cookies"] +reqwest = { version = "0.11.2", features = ["blocking", "json", "cookies"] } +diesel = { version = "1.4.6", features = ["postgres"] } [dependencies.rocket_contrib] version = "0.4.7" diff --git a/build b/build index 962d14c..271a3d9 100755 --- a/build +++ b/build @@ -81,6 +81,7 @@ elif [[ "$action" = run ]]; then --detach \ --name fej_db \ --network fej \ + -p 5432:5432 \ -e "POSTGRES_DB=fej" \ -e "POSTGRES_USER=fej" \ -e "POSTGRES_PASSWORD=fej" \ @@ -101,7 +102,7 @@ elif [[ "$action" = run ]]; then --tty \ --publish 8000:8000 \ --name fej \ - --env-file .env \ + --env-file .env.container \ --network fej \ "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" fi diff --git a/diesel.toml b/diesel.toml new file mode 100644 index 0000000..92267c8 --- /dev/null +++ b/diesel.toml @@ -0,0 +1,5 @@ +# For documentation on how to configure this file, +# see diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema.rs" diff --git a/migrations/.gitkeep b/migrations/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/migrations/00000000000000_diesel_initial_setup/down.sql b/migrations/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 0000000..a9f5260 --- /dev/null +++ b/migrations/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/migrations/00000000000000_diesel_initial_setup/up.sql b/migrations/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 0000000..d68895b --- /dev/null +++ b/migrations/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; From 81ae1a2555370da8738226946ef333d2541689bf Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 18:05:02 +0200 Subject: [PATCH 021/124] [#24] Added first ivago migration --- Makefile | 4 ++++ migrations/2021-04-15-155511_ivago_search/down.sql | 2 ++ migrations/2021-04-15-155511_ivago_search/up.sql | 9 +++++++++ src/schema.rs | 0 4 files changed, 15 insertions(+) create mode 100644 migrations/2021-04-15-155511_ivago_search/down.sql create mode 100644 migrations/2021-04-15-155511_ivago_search/up.sql create mode 100644 src/schema.rs diff --git a/Makefile b/Makefile index 820f893..5d8c22e 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,10 @@ sh: @ docker exec -it fej sh .PHONY: sh +# Starts a psql session in the database container +dbsh: + @ docker exec -it fej_db psql -U fej -d fej + # Builds the debug version, and runs the tests (but doesn't detach). test: @ ./build -m dev -a run -l -- test --no-fail-fast diff --git a/migrations/2021-04-15-155511_ivago_search/down.sql b/migrations/2021-04-15-155511_ivago_search/down.sql new file mode 100644 index 0000000..9d9d004 --- /dev/null +++ b/migrations/2021-04-15-155511_ivago_search/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP SCHEMA ivago CASCADE; diff --git a/migrations/2021-04-15-155511_ivago_search/up.sql b/migrations/2021-04-15-155511_ivago_search/up.sql new file mode 100644 index 0000000..f33eff1 --- /dev/null +++ b/migrations/2021-04-15-155511_ivago_search/up.sql @@ -0,0 +1,9 @@ +-- Your SQL goes here +CREATE SCHEMA ivago; + +CREATE TABLE ivago.streets ( + name TEXT NOT NULL, + city TEXT NOT NULL, + + PRIMARY KEY (name, city) +); diff --git a/src/schema.rs b/src/schema.rs new file mode 100644 index 0000000..e69de29 From cb78af62ba0060465d07f42493d379dce2310a1b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 19:26:37 +0200 Subject: [PATCH 022/124] [#26] Moved main binary to separate location --- Cargo.toml | 6 +++--- docker/Dockerfile.dev | 2 +- docker/Dockerfile.rel | 6 +++--- src/{ => bin/server}/main.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) rename src/{ => bin/server}/main.rs (97%) diff --git a/Cargo.toml b/Cargo.toml index f359a51..46b01a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Jef Roosens "] edition = "2018" [lib] -name = "fej_lib" +name = "fej" src = "src/lib.rs" test = true bench = true @@ -13,8 +13,8 @@ doc = true doctest = true [[bin]] -name = "fej" -src = "src/main.rs" +name = "server" +src = "src/bin/server/main.rs" test = false bench = false doc = false diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 9205fd4..d2029af 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -3,4 +3,4 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 ENTRYPOINT ["cargo"] -CMD ["run"] +CMD ["run", "--bin", "server"] diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 6b4e438..b64916a 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -7,7 +7,7 @@ FROM chewingbever/fej-builder:latest AS builder # NOTE: cargo install auto-appends bin to the path RUN --mount=type=cache,target=/usr/src/app/target \ --mount=type=cache,target=/root/.cargo/registry \ - cargo install --path . --bin fej --root /usr/local + cargo install --path . --root /usr/local # Now, we create the actual image @@ -17,7 +17,7 @@ FROM alpine:latest RUN apk update && apk add --no-cache openssl libgcc curl # Copy binary over to final image -COPY --from=builder /usr/local/bin/fej /usr/local/bin/fej +COPY --from=builder /usr/local/bin/server /usr/local/bin/server HEALTHCHECK \ --interval=10s \ @@ -26,4 +26,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -CMD ["/usr/local/bin/fej"] +CMD ["/usr/local/bin/server"] diff --git a/src/main.rs b/src/bin/server/main.rs similarity index 97% rename from src/main.rs rename to src/bin/server/main.rs index 8b8edc7..a27e00f 100644 --- a/src/main.rs +++ b/src/bin/server/main.rs @@ -1,7 +1,7 @@ #[macro_use] extern crate rocket; -use fej_lib::{catchers, ivago}; +use fej::{catchers, ivago}; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs From 21e142e9a4feaaa01fe292772b262018597c2c8e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 19:36:42 +0200 Subject: [PATCH 023/124] [#26] Added image cleaning recipe --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5d8c22e..f9c6456 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -all: debug +all: debug logs .PHONY: all # Builds the debug release inside the Alpine container. For build caching, two @@ -74,3 +74,7 @@ lint: docs: @ cargo doc --no-deps .PHONY: docs + +# This recipe removes all chewingbever/fej images from your system +clean-images: + @ docker images | grep '^chewingbever/fej' | sed 's/ \+/ /g' | cut -f3 -d' ' | xargs docker rmi From 22e9dcceaf9ce028649d70038b8e702aea571a40 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 22:46:23 +0200 Subject: [PATCH 024/124] [#26] Started writing fejctl --- fejctl | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 fejctl diff --git a/fejctl b/fejctl new file mode 100755 index 0000000..8f3eb74 --- /dev/null +++ b/fejctl @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# This script is a helpful utility for developing fej inside Docker. + +# Default values +image="chewingbever/fej" +mode="dev" # Should be either 'rel' or 'dev' +action="" +attach="--detach" +cmd="b" +bin="server" + +# Calculated variables +patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` +major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` +minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` +branch=`git rev-parse --abbrev-ref HEAD` + + + +function main() { + # Parse the flags + while getopts ":i:m:a:l" c; do + case $c in + i ) image="$OPTARG" ;; + m ) mode="$OPTARG" ;; + a ) action="$OPTARG" ;; + l ) attach="" ;; + ? ) exit 1 ;; + esac + done + shift $((OPTIND-1)) +} + +main "$@" From 0ba31bd8ba944b4bf503366654a70c16e673fff3 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 15 Apr 2021 23:59:31 +0200 Subject: [PATCH 025/124] [#26] Started fejctl --- fejctl | 160 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 136 insertions(+), 24 deletions(-) diff --git a/fejctl b/fejctl index 8f3eb74..17a43a0 100755 --- a/fejctl +++ b/fejctl @@ -1,35 +1,147 @@ #!/usr/bin/env bash -# This script is a helpful utility for developing fej inside Docker. +image='chewingbever/fej' -# Default values -image="chewingbever/fej" -mode="dev" # Should be either 'rel' or 'dev' -action="" -attach="--detach" -cmd="b" -bin="server" +# Creates the needed images +# +# $1: wether to build the debug or the release image (default debug) +function create_images() { + # First, we build the builder + DOCKER_BUILDKIT=1 docker build \ + -f docker/Dockerfile.builder \ + -t "$image-builder:latest" . -# Calculated variables -patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` -major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` -minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` -branch=`git rev-parse --abbrev-ref HEAD` + if [[ "$1" = "rel" ]]; then + DOCKER_BUILDKIT=1 docker build \ + -t "$image:latest" \ + -f docker/Dockerfile.rel . + else + # Then, we create the debug image + DOCKER_BUILDKIT=1 docker build \ + -t "$image:dev" \ + -f docker/Dockerfile.dev . + fi +} +# Execute the debug image (must be built first) +# +# $@: the arguments to pass to the image (passed as arguments to cargo) +function run_image() { + # Run the database image + docker run --rm \ + --detach \ + --name fej_db \ + --network fej \ + -p 5432:5432 \ + -e 'POSTGRES_DB=fej' \ + -e 'POSTGRES_USER=fej' \ + -e 'POSTGRES_PASSWORD=fej' \ + -v 'fej_db-data:/var/lib/postgresql/data' \ + postgres:13-alpine -function main() { - # Parse the flags - while getopts ":i:m:a:l" c; do - case $c in - i ) image="$OPTARG" ;; - m ) mode="$OPTARG" ;; - a ) action="$OPTARG" ;; - l ) attach="" ;; - ? ) exit 1 ;; - esac + # Run the binary image + docker run \ + --detach \ + --rm \ + --interactive \ + --tty \ + --publish 8000:8000 \ + --name fej \ + --env-file .env.container \ + --network fej \ + -v 'fej_build-cache:/usr/src/app/target' \ + -v 'fej_registry-cache:/root/.cargo/registry' \ + "$image:dev" "$@" +} + +# Attach to the fej container +function logs() { + docker logs -f fej +} + +# Builds the given binary +# +# $1: the binary to build +function build() { + create_images + run_image build --bin "$1" + logs +} + +# Runs the given binary +# +# $1: the binary to run +function run() { + create_images + run_image run --bin "$1" + logs +} + +# Runs the tests +function tests() { + create_images + run_image test --no-fail-fast + logs +} + +# Stops both containers +function stop() { + docker stop fej_db + docker stop -t 0 fej +} + +function run_release() { + echo "Not implemented" +} + +# Tags & pushes the release version to Docker Hub +function publish() { + branch=`git rev-parse --abbrev-ref HEAD` + + if [[ "$branch" != master ]]; then + >&2 echo "You can only publish from master." + exit 2 + fi + + patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` + major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` + minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` + tags=("latest" "$patch_version" "$minor_version" "$major_version") + + for tag in "${tags[@]}"; do + # Create the tag + docker tag "$image:$tags" "$image:$tag" + + # Push the tag + docker push "$image:$tag" + + # Remove the tag again, if it's not the main tag + [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" done - shift $((OPTIND-1)) + +} + +# Entrypoint to the script +# +# $1: action to perform, defaults to 'build' +# $2: binary to use, defaults to 'server' +function main() { + # Default values + cmd="${1:-build}" + bin="${2:-server}" + + case $cmd in + b | build ) build "$bin" ;; + br | build-release ) create_images rel ;; + r | run ) run "$bin" ;; + rr | run-release ) run_release ;; + p | push | publish ) publish ;; + t | test ) tests ;; + s | stop ) stop ;; + p | push | publish ) publish ;; + * ) >&2 echo "Invalid command."; exit 1 ;; + esac } main "$@" From 0828dd36d6b26010bb3f8b783eccf9f4fdd736b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:06:48 +0200 Subject: [PATCH 026/124] [#26] fejctl now fully replace Makefile & build --- .hooks/pre-commit | 2 +- Makefile | 80 ---------------------------------- build | 108 ---------------------------------------------- fejctl | 16 ++++++- 4 files changed, 15 insertions(+), 191 deletions(-) delete mode 100644 Makefile delete mode 100755 build diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 709fb95..6947bc7 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This hook lints the code, and if we're on develop or master, also forces the tests to pass. -make lint &> /dev/null 2>&1 || { +./fejctl lint &> /dev/null 2>&1 || { >&2 echo "Format check failed, use 'make lint' for more information."; exit 1; } diff --git a/Makefile b/Makefile deleted file mode 100644 index f9c6456..0000000 --- a/Makefile +++ /dev/null @@ -1,80 +0,0 @@ -all: debug logs -.PHONY: all - -# Builds the debug release inside the Alpine container. For build caching, two -# volumes are used named `fej_build-cache` and `fej_registry-cache`. These -# images are automatically created for you if they don't exist. If you -# encounter any strange build errors, you can try removing this volumes to -# start a completely fresh build. -debug: - @ ./build -m dev -a run build -.PHONY: debug - -# Builds the release version. In contrary to the debug version, this build -# doesn't use volumes for caching, as this would require the build to happen -# during runtime instead of during the building of the image. Instead, it uses -# the new `--mount` feature from Buildkit. This does mean that only very recent -# Docker engines can build the release version (in my case, at the time of -# writing this, 20.10.5). -release: - @ ./build -m rel -.PHONY: release - -# This builds the release version, and pushes all relevant tags to my Docker -# Hub repository, namely chewingbever/fej -push: - @ ./build -m rel -a push -.PHONY: push - -# This builds the debug release, and runs it detached. The reason we detach the -# container is because Rocket has a tendency to ignore ctlr-c when inside a -# container, which gets annoying really fast. -run: - @ ./build -m dev -a run -.PHONY: run - -# As a workaround, we just have a stop command that stops the container. -stop: - @ docker stop -t 2 fej - @ docker stop fej_db -.PHONY: stop - -# This attaches to the running container, essentially giving the same result as -# just running `cargo run` locally. -logs: - @ docker logs -f fej -.PHONY: logs - -# This just starts up a shell inside the fej container -sh: - @ docker exec -it fej sh -.PHONY: sh - -# Starts a psql session in the database container -dbsh: - @ docker exec -it fej_db psql -U fej -d fej - -# Builds the debug version, and runs the tests (but doesn't detach). -test: - @ ./build -m dev -a run -l -- test --no-fail-fast -.PHONY: test - -# Runs the cargo code formatter on your code. -format: - @ cargo fmt -.PHONY: format - -# Lints your code. This also gets run in the pre-commit hook, basically -# preventing you from committing badly-formatted code. -lint: - @ cargo fmt -- --check -.PHONY: lint - -# This builds the documentation for the project, excluding the documentation. -docs: - @ cargo doc --no-deps -.PHONY: docs - -# This recipe removes all chewingbever/fej images from your system -clean-images: - @ docker images | grep '^chewingbever/fej' | sed 's/ \+/ /g' | cut -f3 -d' ' | xargs docker rmi diff --git a/build b/build deleted file mode 100755 index 271a3d9..0000000 --- a/build +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash - -image="chewingbever/fej" -# Should be either dev or rel -mode="dev" -action="" -attach="--detach" - -while getopts ":i:m:a:l" c; do - case $c in - i ) image="$OPTARG" ;; - m ) mode="$OPTARG" ;; - a ) action="$OPTARG" ;; - l ) attach="" ;; - ? ) exit 1 ;; - esac -done -shift $((OPTIND-1)) - -# Extract current version from Cargo.toml & get current branch -patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` -major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` -minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` -branch=`git rev-parse --abbrev-ref HEAD` - -if [[ "$branch" = "master" ]]; then - tags=("$patch_version" "$minor_version" "$major_version" "latest") - -elif [[ "$branch" = "develop" ]]; then - tags=("$patch_version-dev" "$minor_version-dev" "$major_version-dev" "dev") - -else - tags=("$branch") - -fi - -# First, we build the builder -DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.builder -t "$image-builder:latest" . - -# Run the actual build command -if [ "$mode" = "rel" ]; then - DOCKER_BUILDKIT=1 docker build -t "$image:$tags" -f docker/Dockerfile.rel . - -elif [[ "$mode" = "dev" ]]; then - DOCKER_BUILDKIT=1 docker build -t "$image-dev:$tags" -f docker/Dockerfile.dev . - -else - >&2 echo "Invalid mode." - exit 1 - -fi - -if [[ "$action" = push ]]; then - [[ "$branch" =~ ^develop|master$ ]] || { - >&2 echo "You can only push from develop or master." - exit 2 - } - - [[ "$mode" = "rel" ]] || { - >&2 echo "You can only push release builds." - exit 3 - } - - for tag in "${tags[@]}"; do - # Create the tag - docker tag "$image:$tags" "$image:$tag" - - # Push the tag - docker push "$image:$tag" - - # Remove the tag again, if it's not the main tag - [[ "$tag" != "$tags" ]] && docker rmi "$image:$tag" - done - -elif [[ "$action" = run ]]; then - # Create the network & start the database container - docker network create fej - docker volume create fej_db-data - - docker run --rm \ - --detach \ - --name fej_db \ - --network fej \ - -p 5432:5432 \ - -e "POSTGRES_DB=fej" \ - -e "POSTGRES_USER=fej" \ - -e "POSTGRES_PASSWORD=fej" \ - -v 'fej_db-data:/var/lib/postgresql/data' \ - postgres:13-alpine - - if [[ "$mode" = "dev" ]]; then - # Create caching volumes if needed (they need to be named) - docker volume create fej_build-cache - docker volume create fej_registry-cache - - flags="-v fej_build-cache:/usr/src/app/target -v fej_registry-cache:/root/.cargo/registry" - fi - - docker run $attach $flags \ - --rm \ - --interactive \ - --tty \ - --publish 8000:8000 \ - --name fej \ - --env-file .env.container \ - --network fej \ - "$image$([[ "$mode" != "rel" ]] && echo "-dev"):$tags" "$@" -fi diff --git a/fejctl b/fejctl index 17a43a0..c977f79 100755 --- a/fejctl +++ b/fejctl @@ -132,14 +132,26 @@ function main() { bin="${2:-server}" case $cmd in + # Building b | build ) build "$bin" ;; br | build-release ) create_images rel ;; + + # Running r | run ) run "$bin" ;; rr | run-release ) run_release ;; + s | stop ) stop ;; + + # Ease of life + psql ) docker exec -it fej_db psql -U fej -d fej ;; + sh ) docker exec -it fej sh ;; + + # Misc + docs ) cargo doc --no-deps ;; + format ) cargo fmt ;; + l | logs ) logs ;; + lint ) cargo fmt -- --check ;; p | push | publish ) publish ;; t | test ) tests ;; - s | stop ) stop ;; - p | push | publish ) publish ;; * ) >&2 echo "Invalid command."; exit 1 ;; esac } From d19fe5c42e29af96e3be02662ca6e507b90f19b2 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:32:03 +0200 Subject: [PATCH 027/124] [#26] Moved all routing to server binary --- .hooks/pre-commit | 2 +- src/{ => bin/server}/catchers.rs | 0 src/bin/server/main.rs | 8 +- src/bin/server/routes/ivago.rs | 38 ++++++++ src/bin/server/routes/mod.rs | 5 + src/ivago/{controller => }/basic_date.rs | 0 src/ivago/controller/mod.rs | 95 ------------------ src/ivago/mod.rs | 112 ++++++++++++++++------ src/ivago/{controller => }/pickup_time.rs | 0 src/ivago/{controller => }/street.rs | 0 src/lib.rs | 4 - 11 files changed, 130 insertions(+), 134 deletions(-) rename src/{ => bin/server}/catchers.rs (100%) create mode 100644 src/bin/server/routes/ivago.rs create mode 100644 src/bin/server/routes/mod.rs rename src/ivago/{controller => }/basic_date.rs (100%) delete mode 100644 src/ivago/controller/mod.rs rename src/ivago/{controller => }/pickup_time.rs (100%) rename src/ivago/{controller => }/street.rs (100%) diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 6947bc7..fe55771 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -2,7 +2,7 @@ # This hook lints the code, and if we're on develop or master, also forces the tests to pass. ./fejctl lint &> /dev/null 2>&1 || { - >&2 echo "Format check failed, use 'make lint' for more information."; + >&2 echo "Format check failed, use './fejctl lint' for more information."; exit 1; } diff --git a/src/catchers.rs b/src/bin/server/catchers.rs similarity index 100% rename from src/catchers.rs rename to src/bin/server/catchers.rs diff --git a/src/bin/server/main.rs b/src/bin/server/main.rs index a27e00f..3fde2c7 100644 --- a/src/bin/server/main.rs +++ b/src/bin/server/main.rs @@ -1,7 +1,9 @@ +#![feature(proc_macro_hygiene, decl_macro)] + #[macro_use] extern crate rocket; - -use fej::{catchers, ivago}; +mod catchers; +mod routes; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs @@ -33,7 +35,7 @@ impl Fairing for CORS { fn rocket() -> rocket::Rocket { rocket::ignite() .attach(CORS) - .mount("/ivago", ivago::routes()) + .mount("/ivago", routes::ivago()) .register(catchers![catchers::not_found]) } diff --git a/src/bin/server/routes/ivago.rs b/src/bin/server/routes/ivago.rs new file mode 100644 index 0000000..d8318bd --- /dev/null +++ b/src/bin/server/routes/ivago.rs @@ -0,0 +1,38 @@ +use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street}; +use rocket::http::Status; +use rocket_contrib::json::Json; + +/// This route handles the Ivago search endpoint. It returns a list of streets, +/// consisting of a street name & a city. +/// +/// # Arguments +/// +/// * `search_term` - Search term to use to look for streets +#[get("/search?")] +pub fn route_search_streets(q: String) -> Result>, Status> { + Ok(Json(search_streets(q.as_str())?)) +} + +/// Handles returning of pickup times for a specific address. It returns a list +/// of pickup times, containing a date and a description of the trash type. +/// +/// # Arguments +/// +/// * `street` - Street to look up +/// * `number` - House number in the given street +/// * `start_date` - Earliest date that can be returned +/// * `end_date` - Latest date that can be returned +#[get("/?&&&")] +pub fn route_get_pickup_times( + street: Street, + number: u32, + start_date: BasicDate, + end_date: BasicDate, +) -> Result>, Status> { + Ok(Json(get_pickup_times( + &street, + &number, + &start_date.0, + &end_date.0, + )?)) +} diff --git a/src/bin/server/routes/mod.rs b/src/bin/server/routes/mod.rs new file mode 100644 index 0000000..449f82f --- /dev/null +++ b/src/bin/server/routes/mod.rs @@ -0,0 +1,5 @@ +mod ivago; + +pub fn ivago() -> Vec { + routes![ivago::route_search_streets, ivago::route_get_pickup_times] +} diff --git a/src/ivago/controller/basic_date.rs b/src/ivago/basic_date.rs similarity index 100% rename from src/ivago/controller/basic_date.rs rename to src/ivago/basic_date.rs diff --git a/src/ivago/controller/mod.rs b/src/ivago/controller/mod.rs deleted file mode 100644 index 7d3a9f9..0000000 --- a/src/ivago/controller/mod.rs +++ /dev/null @@ -1,95 +0,0 @@ -use crate::errors::FejError; -use chrono::DateTime; -use chrono_tz::Tz; -use reqwest::blocking as reqwest; -use std::collections::HashMap; -use std::convert::{From, TryFrom}; - -mod basic_date; -mod pickup_time; -mod street; - -pub use basic_date::BasicDate; -pub use pickup_time::PickupTime; -pub use street::Street; - -/// Endpoint for the search feature -const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; -/// Endpoint for populating the initial cookies -const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; -/// Endpoint for the actual calendar output -const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; - -/// Searches the Ivago API for streets in the given city. -/// -/// # Arguments -/// -/// * `search_term` - Search term to use to look for streets -pub fn search_streets(search_term: &str) -> Result, FejError> { - let client = reqwest::Client::new(); - let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; - let data: Vec> = response.json()?; - - // This is pretty cool, filter_map first does get() on all the maps, and - // then filters out any None values - // Then, we do the same thing for streets - Ok(data - .iter() - .filter_map(|m| m.get("value")) - .filter_map(|v| Street::try_from(v.as_str()).ok()) - .collect()) -} - -/// Returns the pickup times for the various trash types. -/// -/// # Arguments -/// -/// * `street` - Street to look up -/// * `number` - House number in given street -/// * `start_date` - Earliest date for the results -/// * `end_date` - Latest date for the results -pub fn get_pickup_times( - street: &Street, - number: &u32, - start_date: &DateTime, - end_date: &DateTime, -) -> Result, FejError> { - let client = reqwest::Client::builder().cookie_store(true).build()?; - - // This populates the cookies with the necessary values - client - .post(BASE_URL) - .form(&[ - ("garbage_type", ""), - ("ivago_street", &String::from(street)), - ("number", &number.to_string()), - ("form_id", "garbage_address_form"), - ]) - .send()?; - - let response = client - .get(CAL_URL) - .query(&[ - ("_format", "json"), - ("type", ""), - ("start", &start_date.timestamp().to_string()), - ("end", &end_date.timestamp().to_string()), - ]) - .send()?; - let data: Vec> = response.json()?; - - let mut output: Vec = Vec::new(); - - for map in data - .iter() - .filter(|m| m.contains_key("date") && m.contains_key("label")) - { - // Because we filtered the maps in the loop, we can safely use unwrap - // here - if let Ok(date) = BasicDate::try_from(map.get("date").unwrap().as_str()) { - output.push(PickupTime::new(date, map.get("label").unwrap().to_string())) - } - } - - Ok(output) -} diff --git a/src/ivago/mod.rs b/src/ivago/mod.rs index 627e89b..7d3a9f9 100644 --- a/src/ivago/mod.rs +++ b/src/ivago/mod.rs @@ -1,45 +1,95 @@ -mod controller; +use crate::errors::FejError; +use chrono::DateTime; +use chrono_tz::Tz; +use reqwest::blocking as reqwest; +use std::collections::HashMap; +use std::convert::{From, TryFrom}; -use controller::{get_pickup_times, search_streets}; -use controller::{BasicDate, PickupTime, Street}; -use rocket::http::Status; -use rocket_contrib::json::Json; +mod basic_date; +mod pickup_time; +mod street; -pub fn routes() -> Vec { - routes![route_search_streets, route_get_pickup_times] -} +pub use basic_date::BasicDate; +pub use pickup_time::PickupTime; +pub use street::Street; -/// This route handles the Ivago search endpoint. It returns a list of streets, -/// consisting of a street name & a city. +/// Endpoint for the search feature +const SEARCH_URL: &str = "https://www.ivago.be/nl/particulier/autocomplete/garbage/streets"; +/// Endpoint for populating the initial cookies +const BASE_URL: &str = "https://www.ivago.be/nl/particulier/afval/ophaling"; +/// Endpoint for the actual calendar output +const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/pickups"; + +/// Searches the Ivago API for streets in the given city. /// /// # Arguments /// /// * `search_term` - Search term to use to look for streets -#[get("/search?")] -pub fn route_search_streets(q: String) -> Result>, Status> { - Ok(Json(search_streets(q.as_str())?)) +pub fn search_streets(search_term: &str) -> Result, FejError> { + let client = reqwest::Client::new(); + let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; + let data: Vec> = response.json()?; + + // This is pretty cool, filter_map first does get() on all the maps, and + // then filters out any None values + // Then, we do the same thing for streets + Ok(data + .iter() + .filter_map(|m| m.get("value")) + .filter_map(|v| Street::try_from(v.as_str()).ok()) + .collect()) } -/// Handles returning of pickup times for a specific address. It returns a list -/// of pickup times, containing a date and a description of the trash type. +/// Returns the pickup times for the various trash types. /// /// # Arguments /// /// * `street` - Street to look up -/// * `number` - House number in the given street -/// * `start_date` - Earliest date that can be returned -/// * `end_date` - Latest date that can be returned -#[get("/?&&&")] -pub fn route_get_pickup_times( - street: Street, - number: u32, - start_date: BasicDate, - end_date: BasicDate, -) -> Result>, Status> { - Ok(Json(get_pickup_times( - &street, - &number, - &start_date.0, - &end_date.0, - )?)) +/// * `number` - House number in given street +/// * `start_date` - Earliest date for the results +/// * `end_date` - Latest date for the results +pub fn get_pickup_times( + street: &Street, + number: &u32, + start_date: &DateTime, + end_date: &DateTime, +) -> Result, FejError> { + let client = reqwest::Client::builder().cookie_store(true).build()?; + + // This populates the cookies with the necessary values + client + .post(BASE_URL) + .form(&[ + ("garbage_type", ""), + ("ivago_street", &String::from(street)), + ("number", &number.to_string()), + ("form_id", "garbage_address_form"), + ]) + .send()?; + + let response = client + .get(CAL_URL) + .query(&[ + ("_format", "json"), + ("type", ""), + ("start", &start_date.timestamp().to_string()), + ("end", &end_date.timestamp().to_string()), + ]) + .send()?; + let data: Vec> = response.json()?; + + let mut output: Vec = Vec::new(); + + for map in data + .iter() + .filter(|m| m.contains_key("date") && m.contains_key("label")) + { + // Because we filtered the maps in the loop, we can safely use unwrap + // here + if let Ok(date) = BasicDate::try_from(map.get("date").unwrap().as_str()) { + output.push(PickupTime::new(date, map.get("label").unwrap().to_string())) + } + } + + Ok(output) } diff --git a/src/ivago/controller/pickup_time.rs b/src/ivago/pickup_time.rs similarity index 100% rename from src/ivago/controller/pickup_time.rs rename to src/ivago/pickup_time.rs diff --git a/src/ivago/controller/street.rs b/src/ivago/street.rs similarity index 100% rename from src/ivago/controller/street.rs rename to src/ivago/street.rs diff --git a/src/lib.rs b/src/lib.rs index b1e2733..761c91b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,7 @@ #![feature(proc_macro_hygiene, decl_macro)] -#[macro_use] -extern crate rocket; - // Route modules pub mod ivago; // Helper modules -pub mod catchers; pub mod errors; From 4b51ee20caad1f1b155900f457c4728cbfe11f01 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 00:36:03 +0200 Subject: [PATCH 028/124] [#26] Threw in the towel for now --- src/errors.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/errors.rs b/src/errors.rs index afcf04a..730714c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -10,6 +10,8 @@ pub enum FejError { FailedRequest, } +// I'd love to move this over to the server binary, but right now, error E0117 is making that +// imopssible impl From for Status { fn from(err: FejError) -> Status { match err { From 45c4a4e257efad0edba3d2f94149f33da591c1f8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:18:50 +0200 Subject: [PATCH 029/124] [#26] Moved lib & bin to own folders; Moved server tests; wrote some readmes --- Cargo.toml | 11 ++--- README.md | 60 ++++++++++++++++----------- src/{ => fej}/errors.rs | 0 src/{ => fej}/ivago/basic_date.rs | 0 src/{ => fej}/ivago/mod.rs | 0 src/{ => fej}/ivago/pickup_time.rs | 0 src/{ => fej}/ivago/street.rs | 0 src/{ => fej}/lib.rs | 0 src/{bin => }/server/catchers.rs | 0 src/{bin => }/server/main.rs | 4 ++ src/{bin => }/server/routes/ivago.rs | 0 src/{bin => }/server/routes/mod.rs | 0 tests/ivago.rs => src/server/tests.rs | 2 +- 13 files changed, 46 insertions(+), 31 deletions(-) rename src/{ => fej}/errors.rs (100%) rename src/{ => fej}/ivago/basic_date.rs (100%) rename src/{ => fej}/ivago/mod.rs (100%) rename src/{ => fej}/ivago/pickup_time.rs (100%) rename src/{ => fej}/ivago/street.rs (100%) rename src/{ => fej}/lib.rs (100%) rename src/{bin => }/server/catchers.rs (100%) rename src/{bin => }/server/main.rs (97%) rename src/{bin => }/server/routes/ivago.rs (100%) rename src/{bin => }/server/routes/mod.rs (100%) rename tests/ivago.rs => src/server/tests.rs (93%) diff --git a/Cargo.toml b/Cargo.toml index 46b01a1..4081b85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [lib] name = "fej" -src = "src/lib.rs" +path = "src/fej/lib.rs" test = true bench = true doc = true @@ -14,10 +14,11 @@ doctest = true [[bin]] name = "server" -src = "src/bin/server/main.rs" -test = false -bench = false -doc = false +path = "src/server/main.rs" +test = true +bench = true +doc = true +doctest = true [dependencies] rocket = "0.4.7" diff --git a/README.md b/README.md index 55d11e3..15e4e03 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,21 @@ # Fej -Fej is an API written in Rust. I started this project to learn the language, -and really just have some fun. +Fej is a RESTful API that does lots of different things. It started as an +experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The folder structure follows the structure of the URLs, e.g. the route for -`/hello/world` is found in the module `src/hello`. +The bulk of the project consists of the main `fej` library. The `src` folder +contains the `lib.rs` file for this library, and all other binaries import from +this main library. -Each module contains the following base files: +All binaries can be found in [`/src/bin`](src/bin), with the biggest one being +`server`. This is what actually runs as the Rocket.rs server. The other +binaries (with more possibly coming) are utility tools that will most likely be +run as cron jobs inside the containers, e.g. scrapers. -* `mod.rs`: defines the modules' content, and contains the route definitions. - The route functions themselves only contain the functionality needed to - represent the data, not acquire it. -* `controller.rs`: this file contains the actual logic of each route. If the - logic becomes too complicated to be contained inside a single file, - `controller.rs` becomes its own module folder named `controller`. -* `tests.rs`: this contains tests for the specific module. This can also be a - module directory if need be. - -Every module has a `routes` function that returns its route macros. +Version 1.1 also introduces the use of a database, namely +[PostgreSQL 13](https://www.postgresql.org/). ## Roadmap @@ -27,19 +23,33 @@ See [roadmap.md](roadmap.md). ## Development -The entire toolchain runs on Alpine inside Docker. This makes building easier, -and (hopefully) eliminates any system-specific bugs. +To make development more consistent (and keep my computer a bit cleaner) I've +decided to run the entire toolchain using Docker, with Alpine Linux base +images. This also allows me to make really small final images. Technically, +Docker is the only dependency you need to contribute to this project +(not accounting for language servers etc.). -A [Makefile wrapper](Makefile) is provided for ease of use. Do check it out, as -all the commands are documented for your understanding ;) +A [Bash script](fejctl) is provided to speed up development on the various +binaries. It aids in starting up the containers, choosing which binary to run +etc. A quick rundown of its most important features: -There's also the `build` script. This script does all the "heavy" lifting. It -chooses which Dockerfile to build according to the given arguments, and -generates tags for the images (useful when pushing releases). The Makefile is -really just a wrapper around this build script, allowing you to write -`make test` instead of `./build -m dev -a run test`. +```bash +# By default, it compiles the 'server' binary (but doesn't run it) +./fejctl -tl;dr run `make run` to run your build, and `make test` to run the tests. +# The first argument is the action you wish to perform, the second on which +# binary (not all commands need a binary as input, so they just ignore it) +# For example, this will run the binary called server +./fejctl r server + +# This runs the tests (all commands also have their full name if you want) +./fejctl t +./fejctl test + +# These attach to the two containers +./fejctl sh # Opens a new root shell inside the 'fej' container +./fejctl psql # Open a new psql shell in the fej database +``` ## Docker images diff --git a/src/errors.rs b/src/fej/errors.rs similarity index 100% rename from src/errors.rs rename to src/fej/errors.rs diff --git a/src/ivago/basic_date.rs b/src/fej/ivago/basic_date.rs similarity index 100% rename from src/ivago/basic_date.rs rename to src/fej/ivago/basic_date.rs diff --git a/src/ivago/mod.rs b/src/fej/ivago/mod.rs similarity index 100% rename from src/ivago/mod.rs rename to src/fej/ivago/mod.rs diff --git a/src/ivago/pickup_time.rs b/src/fej/ivago/pickup_time.rs similarity index 100% rename from src/ivago/pickup_time.rs rename to src/fej/ivago/pickup_time.rs diff --git a/src/ivago/street.rs b/src/fej/ivago/street.rs similarity index 100% rename from src/ivago/street.rs rename to src/fej/ivago/street.rs diff --git a/src/lib.rs b/src/fej/lib.rs similarity index 100% rename from src/lib.rs rename to src/fej/lib.rs diff --git a/src/bin/server/catchers.rs b/src/server/catchers.rs similarity index 100% rename from src/bin/server/catchers.rs rename to src/server/catchers.rs diff --git a/src/bin/server/main.rs b/src/server/main.rs similarity index 97% rename from src/bin/server/main.rs rename to src/server/main.rs index 3fde2c7..58b0300 100644 --- a/src/bin/server/main.rs +++ b/src/server/main.rs @@ -2,6 +2,10 @@ #[macro_use] extern crate rocket; + +#[cfg(test)] +mod tests; + mod catchers; mod routes; diff --git a/src/bin/server/routes/ivago.rs b/src/server/routes/ivago.rs similarity index 100% rename from src/bin/server/routes/ivago.rs rename to src/server/routes/ivago.rs diff --git a/src/bin/server/routes/mod.rs b/src/server/routes/mod.rs similarity index 100% rename from src/bin/server/routes/mod.rs rename to src/server/routes/mod.rs diff --git a/tests/ivago.rs b/src/server/tests.rs similarity index 93% rename from tests/ivago.rs rename to src/server/tests.rs index d78501a..8871762 100644 --- a/tests/ivago.rs +++ b/src/server/tests.rs @@ -3,7 +3,7 @@ use rocket::http::Status; use rocket::local::Client; fn rocket() -> rocket::Rocket { - rocket::ignite().mount("/", fej_lib::ivago::routes()) + rocket::ignite().mount("/", super::routes::ivago()) } /// Test 404 response From c2fa764e8002352728030e512f6f99b15b0014f6 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:22:30 +0200 Subject: [PATCH 030/124] [#26] Updated README again --- README.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 15e4e03..e2573af 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,10 @@ experiment to learn Rust, but has grown into a full-on passion project. ## Project Structure -The bulk of the project consists of the main `fej` library. The `src` folder -contains the `lib.rs` file for this library, and all other binaries import from -this main library. - -All binaries can be found in [`/src/bin`](src/bin), with the biggest one being -`server`. This is what actually runs as the Rocket.rs server. The other -binaries (with more possibly coming) are utility tools that will most likely be -run as cron jobs inside the containers, e.g. scrapers. +The `src` folder contains subfolders for the various binaries and the main +library, called `fej`. The biggest binary is called `server`, which is the +binary that actually runs the Rocket.rs web server. All the others are utility +programs, mostly consisting of scrapers for various services. Version 1.1 also introduces the use of a database, namely [PostgreSQL 13](https://www.postgresql.org/). From 0e4eb6812164b52d77c4532ef53fbf79fe15048b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 09:30:25 +0200 Subject: [PATCH 031/124] [#26] Updated roadmap.md --- roadmap.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roadmap.md b/roadmap.md index f03e4b5..bfdd622 100644 --- a/roadmap.md +++ b/roadmap.md @@ -69,8 +69,8 @@ setting up a database for this version anyways. ## Kissanime I like watching anime from time to time, and I've always used Kissanime for -this. However, their sites can be quite slow, and riddled with ads from time to -time. That's why I'd like to create a high-speed wrapper that extracts all the -needed info from their sites, removing the need for the user to ever actually -visit their site. The API can just act as a fast search index, complete with -indexing of the links to the videos and everything. +this. However, their sites can be quite slow, and riddled with ads. That's why +I'd like to create a high-speed wrapper that extracts all the needed info from +their sites, removing the need for the user to ever actually visit their site. +The API can just act as a fast search index, complete with indexing of the +links to the videos and everything. From 14321db3165797e6b170eace5977d7826fe6c6fd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 20:17:00 +0200 Subject: [PATCH 032/124] [#24] Added diesel_postgres_pool to deps --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4081b85..886d903 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,4 +32,4 @@ diesel = { version = "1.4.6", features = ["postgres"] } [dependencies.rocket_contrib] version = "0.4.7" default-features = false -features = ["json"] +features = ["json", "diesel_postgres_pool"] From d5a513482a41b48d0452449241b6d2244b4de3db Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 20:39:09 +0200 Subject: [PATCH 033/124] [#24] First try at compiling diesel in container (DOESN'T WORK) --- Cargo.lock | 85 +++++++++++++++++++++++++++++++++++++++ Rocket.toml | 11 +++++ docker/Dockerfile.builder | 4 +- fejctl | 1 + 4 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 Rocket.toml diff --git a/Cargo.lock b/Cargo.lock index 0006402..55062dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -331,6 +331,7 @@ dependencies = [ "byteorder", "diesel_derives", "pq-sys", + "r2d2", ] [[package]] @@ -741,6 +742,15 @@ dependencies = [ "libc", ] +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -805,6 +815,15 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +[[package]] +name = "lock_api" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.3.9" @@ -1045,6 +1064,31 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + [[package]] name = "parse-zoneinfo" version = "0.3.0" @@ -1204,6 +1248,17 @@ dependencies = [ "proc-macro2 1.0.26", ] +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log 0.4.14", + "parking_lot", + "scheduled-thread-pool", +] + [[package]] name = "rand" version = "0.8.3" @@ -1359,13 +1414,28 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7954a707f9ca18aa74ca8c1f5d1f900f52a4dceb68e96e3112143f759cfd20e" dependencies = [ + "diesel", "log 0.4.14", "notify", + "r2d2", "rocket", + "rocket_contrib_codegen", "serde", "serde_json", ] +[[package]] +name = "rocket_contrib_codegen" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30deb6dec53b91fac3538a2a3935cf13e0f462745f9f33bf27bedffbe7265b5d" +dependencies = [ + "devise", + "quote 0.6.13", + "version_check 0.9.3", + "yansi", +] + [[package]] name = "rocket_http" version = "0.4.7" @@ -1423,6 +1493,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "security-framework" version = "2.2.0" diff --git a/Rocket.toml b/Rocket.toml new file mode 100644 index 0000000..2eba66c --- /dev/null +++ b/Rocket.toml @@ -0,0 +1,11 @@ +[development] +address = "localhost" +port = 7999 +keep_alive = 5 +read_timeout = 5 +write_timeout = 5 +log = "normal" +limits = { forms = 32768 } + +[global.databases] +postgres_fej = { url = "postgres://fej:fej@fej_db:5432/fej" } diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index b968c5b..4d04e9b 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -4,13 +4,11 @@ FROM alpine:latest AS builder ENV PATH "$PATH:/root/.cargo/bin" # Needed for proper compiling of openssl-dev ENV RUSTFLAGS "-C target-feature=-crt-static" -# Otherwise, the debug build can't be used from the container -ENV ROCKET_ADDRESS "0.0.0.0" WORKDIR /usr/src/app # Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev build-base curl && \ +RUN apk update && apk add --no-cache openssl-dev build-base curl libpq && \ { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } # Copy source code over to builder diff --git a/fejctl b/fejctl index c977f79..acaaeee 100755 --- a/fejctl +++ b/fejctl @@ -52,6 +52,7 @@ function run_image() { --network fej \ -v 'fej_build-cache:/usr/src/app/target' \ -v 'fej_registry-cache:/root/.cargo/registry' \ + -v "$PWD/Rocket.toml:/usr/src/app/Rocket.toml:ro" \ "$image:dev" "$@" } From fd7a3e23317d00f655cca87b1d1ba1214969af19 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 16 Apr 2021 23:20:22 +0200 Subject: [PATCH 034/124] [#24] Second attempt (failed) --- .dockerignore | 3 +++ docker/Dockerfile.builder | 5 +++-- docker/Dockerfile.dev | 4 +++- docker/entrypoint_dev.sh | 4 ++++ fejctl | 4 ++++ 5 files changed, 17 insertions(+), 3 deletions(-) create mode 100755 docker/entrypoint_dev.sh diff --git a/.dockerignore b/.dockerignore index 614c7dd..aa9bcf8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,6 @@ # Cargo files !Cargo.toml !Cargo.lock + +# Entrypoint for devop container +!docker/entrypoint_dev.sh diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index 4d04e9b..126a302 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -8,8 +8,9 @@ ENV RUSTFLAGS "-C target-feature=-crt-static" WORKDIR /usr/src/app # Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev build-base curl libpq && \ - { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } +RUN apk update && apk add --no-cache openssl-dev curl libpq libgcc musl-dev gcc && \ + { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ + rustup target add x86_64-unknown-linux-musl --toolchain nightly # Copy source code over to builder COPY Cargo.toml Cargo.lock ./ diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index d2029af..7281bc3 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -2,5 +2,7 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 -ENTRYPOINT ["cargo"] +COPY ./docker/entrypoint_dev.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] CMD ["run", "--bin", "server"] diff --git a/docker/entrypoint_dev.sh b/docker/entrypoint_dev.sh new file mode 100755 index 0000000..0838bce --- /dev/null +++ b/docker/entrypoint_dev.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh + +# All this file does is inject the target +cargo "$@" --target x86_64-unknown-linux-musl diff --git a/fejctl b/fejctl index acaaeee..ee4cf8e 100755 --- a/fejctl +++ b/fejctl @@ -28,6 +28,10 @@ function create_images() { # # $@: the arguments to pass to the image (passed as arguments to cargo) function run_image() { + docker volume create fej_build-cache + docker volume create fej_registry-cache + docker volume create fej_db-data + # Run the database image docker run --rm \ --detach \ From f381c5b9108a09c3e612b4d2ee64ace4d1e5eb0f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 10:12:37 +0200 Subject: [PATCH 035/124] [#24] Fixed libpq not wanting to link --- Rocket.toml | 4 ++-- docker/Dockerfile.builder | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Rocket.toml b/Rocket.toml index 2eba66c..20ca98c 100644 --- a/Rocket.toml +++ b/Rocket.toml @@ -1,6 +1,6 @@ [development] -address = "localhost" -port = 7999 +address = "0.0.0.0" +port = 8000 keep_alive = 5 read_timeout = 5 write_timeout = 5 diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index 126a302..dc28716 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -8,7 +8,7 @@ ENV RUSTFLAGS "-C target-feature=-crt-static" WORKDIR /usr/src/app # Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev curl libpq libgcc musl-dev gcc && \ +RUN apk update && apk add --no-cache openssl-dev curl postgresql-dev libgcc musl-dev gcc && \ { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ rustup target add x86_64-unknown-linux-musl --toolchain nightly From 5746d295564aabff2410344f278184e4bb85a28a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 10:32:16 +0200 Subject: [PATCH 036/124] [#24] Added musl target to release builder --- docker/Dockerfile.rel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index b64916a..e293ccc 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -7,7 +7,10 @@ FROM chewingbever/fej-builder:latest AS builder # NOTE: cargo install auto-appends bin to the path RUN --mount=type=cache,target=/usr/src/app/target \ --mount=type=cache,target=/root/.cargo/registry \ - cargo install --path . --root /usr/local + cargo install \ + --path . \ + --root /usr/local \ + --target x86_64-unknown-linux-musl # Now, we create the actual image From 3a490fca353fa6d955a3d44e125bd031aeaca70c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 10:52:55 +0200 Subject: [PATCH 037/124] [#24] Added db connection to server --- src/server/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/server/main.rs b/src/server/main.rs index 58b0300..9e01983 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -2,6 +2,8 @@ #[macro_use] extern crate rocket; +#[macro_use] +extern crate rocket_contrib; #[cfg(test)] mod tests; @@ -14,6 +16,7 @@ mod routes; use rocket::fairing::{Fairing, Info, Kind}; use rocket::http::Header; use rocket::{Request, Response}; +use rocket_contrib::databases::diesel; pub struct CORS; @@ -36,9 +39,14 @@ impl Fairing for CORS { } } +// This defines a connection to the database +#[database("postgres_fej")] +struct FejDbConn(diesel::PgConnection); + fn rocket() -> rocket::Rocket { rocket::ignite() .attach(CORS) + .attach(FejDbConn::fairing()) .mount("/ivago", routes::ivago()) .register(catchers![catchers::not_found]) } From 0b2b986205dbf3c4e3aa84f80ae3db61b745b9fb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 13:46:38 +0200 Subject: [PATCH 038/124] [#31] Debug build is now rootless --- docker/Dockerfile.builder | 34 ++++++++++++++++++++++++++-------- docker/Dockerfile.dev | 3 ++- docker/Dockerfile.rel | 1 + fejctl | 29 ++++++++++++++++++++--------- 4 files changed, 49 insertions(+), 18 deletions(-) diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index dc28716..4790f7c 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -1,17 +1,35 @@ +# vim: filetype=dockerfile # Our entire toolchain runs in alpine FROM alpine:latest AS builder -ENV PATH "$PATH:/root/.cargo/bin" +ENV PATH "$PATH:/app/.cargo/bin" # Needed for proper compiling of openssl-dev ENV RUSTFLAGS "-C target-feature=-crt-static" -WORKDIR /usr/src/app +# Add the build user +# Install dependencies +RUN addgroup -S builder && \ + adduser -S builder -G builder -h /app && \ + apk update && \ + apk add --no-cache \ + curl \ + gcc \ + libgcc \ + musl-dev \ + openssl-dev \ + postgresql-dev -# Install build dependencies, rustup & rust's nightly build & toolchain -RUN apk update && apk add --no-cache openssl-dev curl postgresql-dev libgcc musl-dev gcc && \ - { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ - rustup target add x86_64-unknown-linux-musl --toolchain nightly +# Switch to the non-root user +USER builder + +WORKDIR /app + +# Install rustup in the new user's home +# Create mountpoints for volumes with correct permissions +RUN { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ + rustup target add x86_64-unknown-linux-musl --toolchain nightly && \ + mkdir -p .cargo/registry target # Copy source code over to builder -COPY Cargo.toml Cargo.lock ./ -COPY src/ ./src/ +COPY --chown=builder:builder Cargo.toml Cargo.lock ./ +COPY --chown=builder:builder src/ ./src/ diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 7281bc3..28004ac 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,8 +1,9 @@ +# vim: filetype=dockerfile FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 -COPY ./docker/entrypoint_dev.sh /entrypoint.sh +COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] CMD ["run", "--bin", "server"] diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index e293ccc..0fa71ad 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -1,3 +1,4 @@ +# vim: filetype=dockerfile FROM chewingbever/fej-builder:latest AS builder # And then finally, build the project diff --git a/fejctl b/fejctl index ee4cf8e..688395c 100755 --- a/fejctl +++ b/fejctl @@ -9,18 +9,27 @@ function create_images() { # First, we build the builder DOCKER_BUILDKIT=1 docker build \ -f docker/Dockerfile.builder \ - -t "$image-builder:latest" . + -t "$image-builder:latest" . || { + >&2 echo "Failed to build builder."; + exit 1; + } if [[ "$1" = "rel" ]]; then DOCKER_BUILDKIT=1 docker build \ -t "$image:latest" \ - -f docker/Dockerfile.rel . + -f docker/Dockerfile.rel . || { + >&2 echo "Failed to build release image."; + exit 1; + } else # Then, we create the debug image DOCKER_BUILDKIT=1 docker build \ -t "$image:dev" \ - -f docker/Dockerfile.dev . + -f docker/Dockerfile.dev . || { + >&2 echo "Failed to build debug image."; + exit 1; + } fi } @@ -28,9 +37,9 @@ function create_images() { # # $@: the arguments to pass to the image (passed as arguments to cargo) function run_image() { - docker volume create fej_build-cache - docker volume create fej_registry-cache - docker volume create fej_db-data + docker volume create fej_build-cache > /dev/null + docker volume create fej_registry-cache > /dev/null + docker volume create fej_db-data > /dev/null # Run the database image docker run --rm \ @@ -54,9 +63,9 @@ function run_image() { --name fej \ --env-file .env.container \ --network fej \ - -v 'fej_build-cache:/usr/src/app/target' \ - -v 'fej_registry-cache:/root/.cargo/registry' \ - -v "$PWD/Rocket.toml:/usr/src/app/Rocket.toml:ro" \ + -v 'fej_build-cache:/app/target' \ + -v 'fej_registry-cache:/app/.cargo/registry' \ + -v "$PWD/Rocket.toml:/app/Rocket.toml:ro" \ "$image:dev" "$@" } @@ -109,6 +118,8 @@ function publish() { exit 2 fi + create_images rel + patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` From adbe6d1fb7e566af58fb433039132ec723613b4b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 14:09:31 +0200 Subject: [PATCH 039/124] [#31] Release is now rootless (rip caching) --- .dockerignore | 3 +++ docker/Dockerfile.rel | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index aa9bcf8..41f29f5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,6 @@ # Entrypoint for devop container !docker/entrypoint_dev.sh + +# Config file +!Rocket.toml diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 0fa71ad..2d3672c 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -6,22 +6,42 @@ FROM chewingbever/fej-builder:latest AS builder # https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 # TODO add what these flags do & why they work # NOTE: cargo install auto-appends bin to the path -RUN --mount=type=cache,target=/usr/src/app/target \ - --mount=type=cache,target=/root/.cargo/registry \ - cargo install \ - --path . \ - --root /usr/local \ - --target x86_64-unknown-linux-musl + +# RUN --mount=type=cache,mode=0777,target=/app/target \ +# --mount=type=cache,mode=0777,target=/app/.cargo/registry \ + +# Buildkit cache mounts really don't like it when you're not root, +# so I guess we're building release without a cache for now +RUN cargo install \ + --path . \ + --root /app/output \ + --target x86_64-unknown-linux-musl # Now, we create the actual image FROM alpine:latest # Install some dynamic libraries needed for everything to work -RUN apk update && apk add --no-cache openssl libgcc curl +# Create -non-root user +RUN apk update && \ + apk add --no-cache \ + curl \ + libgcc \ + libpq \ + openssl && \ + addgroup -S fej && \ + adduser -S fej -G fej -h /app + +# Switch to non-root user +USER fej:fej # Copy binary over to final image -COPY --from=builder /usr/local/bin/server /usr/local/bin/server +COPY --from=builder --chown=fej:fej /app/output/bin /app/bin + +# Embed config file inside container +# The workdir is changed so that the config file is read properly +WORKDIR /app +COPY --chown=fej:fej Rocket.toml /app/Rocket.toml HEALTHCHECK \ --interval=10s \ @@ -30,4 +50,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -CMD ["/usr/local/bin/server"] +CMD ["/app/bin/server"] From 0bd09db295b96ec2dcec23fc92caf25353697f5a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 14:19:53 +0200 Subject: [PATCH 040/124] [#31] Config file is now copied into images --- docker/Dockerfile.dev | 1 + fejctl | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 28004ac..ea63e1e 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -4,6 +4,7 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh +COPY --chown=builder:builder ./Rocket.toml /app/Rocket.toml ENTRYPOINT ["/entrypoint.sh"] CMD ["run", "--bin", "server"] diff --git a/fejctl b/fejctl index 688395c..879419e 100755 --- a/fejctl +++ b/fejctl @@ -65,7 +65,6 @@ function run_image() { --network fej \ -v 'fej_build-cache:/app/target' \ -v 'fej_registry-cache:/app/.cargo/registry' \ - -v "$PWD/Rocket.toml:/app/Rocket.toml:ro" \ "$image:dev" "$@" } From 65c3d616de73b65f503a5cbc3e9808477e6f26dd Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 14:43:24 +0200 Subject: [PATCH 041/124] Migrations should now run on startup --- .dockerignore | 3 +++ Cargo.toml | 1 + docker/Dockerfile.builder | 1 + src/server/main.rs | 19 ++++++++++++++++++- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 41f29f5..b7c1108 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,3 +12,6 @@ # Config file !Rocket.toml + +# Database migrations +!migrations/ diff --git a/Cargo.toml b/Cargo.toml index 886d903..9677304 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ chrono-tz = "0.5.3" regex = "1.4.5" reqwest = { version = "0.11.2", features = ["blocking", "json", "cookies"] } diesel = { version = "1.4.6", features = ["postgres"] } +diesel_migrations = "1.4.0" [dependencies.rocket_contrib] version = "0.4.7" diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index 4790f7c..b4152fa 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -33,3 +33,4 @@ RUN { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -- # Copy source code over to builder COPY --chown=builder:builder Cargo.toml Cargo.lock ./ COPY --chown=builder:builder src/ ./src/ +COPY --chown=builder:builder migrations/ ./migrations/ diff --git a/src/server/main.rs b/src/server/main.rs index 9e01983..4267776 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -4,6 +4,8 @@ extern crate rocket; #[macro_use] extern crate rocket_contrib; +#[macro_use] +extern crate diesel_migrations; #[cfg(test)] mod tests; @@ -13,9 +15,10 @@ mod routes; // Very temporary solution for CORS // https://stackoverflow.com/questions/62412361/how-to-set-up-cors-or-options-for-rocket-rs +use rocket::fairing::AdHoc; use rocket::fairing::{Fairing, Info, Kind}; use rocket::http::Header; -use rocket::{Request, Response}; +use rocket::{Request, Response, Rocket}; use rocket_contrib::databases::diesel; pub struct CORS; @@ -39,14 +42,28 @@ impl Fairing for CORS { } } +// Macro from diesel_migrations that sets up migrations +embed_migrations!(); + // This defines a connection to the database #[database("postgres_fej")] struct FejDbConn(diesel::PgConnection); +// I'd like to thank Stackoverflow for helping me with this +// https://stackoverflow.com/questions/61047355/how-to-run-diesel-migration-with-rocket-in-production +fn run_db_migrations(rocket: Rocket) -> Result { + let conn = FejDbConn::get_one(&rocket).expect("database connection"); + match embedded_migrations::run(&*conn) { + Ok(()) => Ok(rocket), + Err(e) => Err(rocket), + } +} + fn rocket() -> rocket::Rocket { rocket::ignite() .attach(CORS) .attach(FejDbConn::fairing()) + .attach(AdHoc::on_attach("Database Migrations", run_db_migrations)) .mount("/ivago", routes::ivago()) .register(catchers![catchers::not_found]) } From 00bf8501dd5f3e0e966ab9923f19e8be14bab1eb Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 15:10:42 +0200 Subject: [PATCH 042/124] [closes #33] Added compose file; ./fejctl rr works now --- Rocket.toml | 5 ++++- docker/docker-compose.yml | 35 +++++++++++++++++++++++++++++++++++ fejctl | 14 +++++++++++++- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 docker/docker-compose.yml diff --git a/Rocket.toml b/Rocket.toml index 20ca98c..56291e7 100644 --- a/Rocket.toml +++ b/Rocket.toml @@ -7,5 +7,8 @@ write_timeout = 5 log = "normal" limits = { forms = 32768 } -[global.databases] +[development.databases] postgres_fej = { url = "postgres://fej:fej@fej_db:5432/fej" } + +[production.databases] +postgres_fej = { url = "postgres://fej:fej@db:5432/fej" } diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..c6d6066 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,35 @@ +version: '2.4' + +services: + app: + image: 'chewingbever/fej:latest' + restart: 'always' + + depends_on: + db: + condition: 'service_healthy' + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' + + db: + image: 'postgres:13-alpine' + restart: 'always' + + environment: + - 'POSTGRES_DB=fej' + - 'POSTGRES_USER=fej' + - 'POSTGRES_PASSWORD=fej' + healthcheck: + test: 'pg_isready -U fej' + interval: '30s' + timeout: '5s' + retries: 3 + start_period: '0s' + + volumes: + - 'db-data:/var/lib/postgresql/data' + + +volumes: + db-data: diff --git a/fejctl b/fejctl index 879419e..b60be50 100755 --- a/fejctl +++ b/fejctl @@ -105,7 +105,18 @@ function stop() { } function run_release() { - echo "Not implemented" + docker-compose \ + --file docker/docker-compose.yml \ + --project-name fej-release \ + up \ + --detach +} + +function stop_release() { + docker-compose \ + --file docker/docker-compose.yml \ + --project-name fej-release \ + down } # Tags & pushes the release version to Docker Hub @@ -155,6 +166,7 @@ function main() { r | run ) run "$bin" ;; rr | run-release ) run_release ;; s | stop ) stop ;; + sr | stop-release ) stop_release ;; # Ease of life psql ) docker exec -it fej_db psql -U fej -d fej ;; From 43e26191ccd816224ce259982dcc35a1332c40e8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 16:04:04 +0200 Subject: [PATCH 043/124] Moved development stuff to docker-compose --- Cargo.lock | 32 +++++++ docker/Dockerfile.dev | 1 - docker/docker-compose.dev.yml | 31 +++++++ docker/docker-compose.yml | 8 +- fejctl | 158 ++++++++++------------------------ 5 files changed, 112 insertions(+), 118 deletions(-) create mode 100644 docker/docker-compose.dev.yml diff --git a/Cargo.lock b/Cargo.lock index 55062dc..9c229ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,6 +345,16 @@ dependencies = [ "syn 1.0.69", ] +[[package]] +name = "diesel_migrations" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" +dependencies = [ + "migrations_internals", + "migrations_macros", +] + [[package]] name = "digest" version = "0.9.0" @@ -376,6 +386,7 @@ dependencies = [ "chrono", "chrono-tz", "diesel", + "diesel_migrations", "regex", "reqwest", "rocket", @@ -854,6 +865,27 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "migrations_internals" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" +dependencies = [ + "diesel", +] + +[[package]] +name = "migrations_macros" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" +dependencies = [ + "migrations_internals", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.69", +] + [[package]] name = "mime" version = "0.2.6" diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index ea63e1e..bac8b44 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -7,4 +7,3 @@ COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh COPY --chown=builder:builder ./Rocket.toml /app/Rocket.toml ENTRYPOINT ["/entrypoint.sh"] -CMD ["run", "--bin", "server"] diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 0000000..bf7fb6f --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,31 @@ +version: '2.4' +services: + app: + build: + # Make sure the build context is one directory up + context: '..' + dockerfile: './docker/Dockerfile.dev' + image: 'chewingbever/fej:dev' + restart: 'no' + + container_name: 'fej_app' + volumes: + - 'build-cache:/app/target' + - 'registry-cache:/app/.cargo/registry' + ports: + - '8000:8000' + + command: "${CMD}" + + db: + container_name: 'fej_db' + restart: 'no' + + # the devop environment exposes the database so we can use the Diesel cli + ports: + - '5432:5432' + +volumes: + build-cache: + registry-cache: + diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c6d6066..14e90d2 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,13 +2,13 @@ version: '2.4' services: app: + build: + context: '..' + dockerfile: 'docker/Dockerfile.rel' + image: 'chewingbever/fej:latest' restart: 'always' - depends_on: - db: - condition: 'service_healthy' - environment: - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' diff --git a/fejctl b/fejctl index b60be50..d55e090 100755 --- a/fejctl +++ b/fejctl @@ -2,121 +2,52 @@ image='chewingbever/fej' -# Creates the needed images +# Small wrapper around the docker-compose command # -# $1: wether to build the debug or the release image (default debug) -function create_images() { - # First, we build the builder - DOCKER_BUILDKIT=1 docker build \ - -f docker/Dockerfile.builder \ - -t "$image-builder:latest" . || { - >&2 echo "Failed to build builder."; - exit 1; - } +# Flags: +# -b: build the builder +# -r: use the release image instead +function dc() { + while getopts ":br" c; do + case $c in + b ) build_builder=1 ;; + r ) release=1 ;; + esac + done + shift $((OPTIND-1)) - if [[ "$1" = "rel" ]]; then + if [[ "$build_builder" -eq 1 ]]; then + # We always rebuild the builder before we run any compose command DOCKER_BUILDKIT=1 docker build \ - -t "$image:latest" \ - -f docker/Dockerfile.rel . || { - >&2 echo "Failed to build release image."; + -f docker/Dockerfile.builder \ + -t "$image-builder:latest" . || { + >&2 echo "Failed to build builder."; exit 1; } + fi + + if [[ "$release" -eq 1 ]]; then + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ + --file docker/docker-compose.yml \ + --project-name fej \ + "$@" else - # Then, we create the debug image - DOCKER_BUILDKIT=1 docker build \ - -t "$image:dev" \ - -f docker/Dockerfile.dev . || { - >&2 echo "Failed to build debug image."; - exit 1; - } + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ + --file docker/docker-compose.yml \ + --file docker/docker-compose.dev.yml \ + --project-name fej-dev \ + "$@" fi } # Execute the debug image (must be built first) # # $@: the arguments to pass to the image (passed as arguments to cargo) -function run_image() { - docker volume create fej_build-cache > /dev/null - docker volume create fej_registry-cache > /dev/null - docker volume create fej_db-data > /dev/null - - # Run the database image - docker run --rm \ - --detach \ - --name fej_db \ - --network fej \ - -p 5432:5432 \ - -e 'POSTGRES_DB=fej' \ - -e 'POSTGRES_USER=fej' \ - -e 'POSTGRES_PASSWORD=fej' \ - -v 'fej_db-data:/var/lib/postgresql/data' \ - postgres:13-alpine - - # Run the binary image - docker run \ - --detach \ - --rm \ - --interactive \ - --tty \ - --publish 8000:8000 \ - --name fej \ - --env-file .env.container \ - --network fej \ - -v 'fej_build-cache:/app/target' \ - -v 'fej_registry-cache:/app/.cargo/registry' \ - "$image:dev" "$@" -} - -# Attach to the fej container -function logs() { - docker logs -f fej -} - -# Builds the given binary -# -# $1: the binary to build -function build() { - create_images - run_image build --bin "$1" - logs -} - -# Runs the given binary -# -# $1: the binary to run -function run() { - create_images - run_image run --bin "$1" - logs -} - -# Runs the tests -function tests() { - create_images - run_image test --no-fail-fast - logs -} - -# Stops both containers -function stop() { - docker stop fej_db - docker stop -t 0 fej -} - -function run_release() { - docker-compose \ - --file docker/docker-compose.yml \ - --project-name fej-release \ - up \ - --detach -} - -function stop_release() { - docker-compose \ - --file docker/docker-compose.yml \ - --project-name fej-release \ - down +function dcr() { + CMD="$@" dc -b -- up \ + --build \ + --detach } # Tags & pushes the release version to Docker Hub @@ -128,7 +59,8 @@ function publish() { exit 2 fi - create_images rel + # Build the release images + dc -br build patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` @@ -159,26 +91,26 @@ function main() { case $cmd in # Building - b | build ) build "$bin" ;; - br | build-release ) create_images rel ;; + b | build ) dcr build --bin "$bin" && dc -- logs -f app ;; + br | build-release ) dc -br build ;; # Running - r | run ) run "$bin" ;; - rr | run-release ) run_release ;; - s | stop ) stop ;; - sr | stop-release ) stop_release ;; + r | run ) dcr run --bin "$bin" && dc -- logs -f app ;; + rr | run-release ) dc -br -- run --build --detach ;; + s | stop ) dc down ;; + sr | stop-release ) dc -r stop ;; # Ease of life - psql ) docker exec -it fej_db psql -U fej -d fej ;; - sh ) docker exec -it fej sh ;; + psql ) dc -- exec db psql -U fej -d fej ;; + sh ) dc -- exec app sh ;; # Misc docs ) cargo doc --no-deps ;; format ) cargo fmt ;; - l | logs ) logs ;; + l | logs ) dc -- logs -f app ;; lint ) cargo fmt -- --check ;; p | push | publish ) publish ;; - t | test ) tests ;; + t | test ) dc test --no-fail-fast ;; * ) >&2 echo "Invalid command."; exit 1 ;; esac } From 95f45cab76f6909a738dc128ed5f962b1cad16c9 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 17:46:16 +0200 Subject: [PATCH 044/124] Fixed strange dc behavior --- fejctl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fejctl b/fejctl index d55e090..7ee69df 100755 --- a/fejctl +++ b/fejctl @@ -8,7 +8,10 @@ image='chewingbever/fej' # -b: build the builder # -r: use the release image instead function dc() { + local OPTIND c build_builder release + while getopts ":br" c; do + echo "$c" case $c in b ) build_builder=1 ;; r ) release=1 ;; @@ -52,7 +55,7 @@ function dcr() { # Tags & pushes the release version to Docker Hub function publish() { - branch=`git rev-parse --abbrev-ref HEAD` + local branch=`git rev-parse --abbrev-ref HEAD` if [[ "$branch" != master ]]; then >&2 echo "You can only publish from master." @@ -62,10 +65,10 @@ function publish() { # Build the release images dc -br build - patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` - major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` - minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` - tags=("latest" "$patch_version" "$minor_version" "$major_version") + local patch_version=`grep -Po '(?<=version = ").*(?=")' Cargo.toml | head -n1` + local major_version=`echo "$patch_version" | sed -E 's/([0-9]+)\.([0-9]+)\.([0-9]+)/\1/'` + local minor_version=`echo "$patch_version" | sed -E 's/([0-9]+).([0-9]+).([0-9]+)/\1.\2/'` + local tags=("latest" "$patch_version" "$minor_version" "$major_version") for tag in "${tags[@]}"; do # Create the tag @@ -86,8 +89,8 @@ function publish() { # $2: binary to use, defaults to 'server' function main() { # Default values - cmd="${1:-build}" - bin="${2:-server}" + local cmd="${1:-build}" + local bin="${2:-server}" case $cmd in # Building From a072941ec3a3d7692dac98c115c6320803ca947e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 17:48:02 +0200 Subject: [PATCH 045/124] Removed unneeded file --- .env.container | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .env.container diff --git a/.env.container b/.env.container deleted file mode 100644 index 8b395fb..0000000 --- a/.env.container +++ /dev/null @@ -1,2 +0,0 @@ -# This file is read by the container, and therefore by fej -DATABASE_URL=postgres://fej:fej@fej_db:5432/fej From 6beca85154112bb17ad2bdb3869183fcbf6da3e1 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 18:06:31 +0200 Subject: [PATCH 046/124] Added cron as rel dependency --- docker/Dockerfile.rel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 2d3672c..90cefde 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -26,6 +26,7 @@ FROM alpine:latest RUN apk update && \ apk add --no-cache \ curl \ + cron \ libgcc \ libpq \ openssl && \ @@ -50,4 +51,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -CMD ["/app/bin/server"] +ENTRYPOINT ["/app/bin/server"] From e62cd0bb9308af8c2c5246b10cde20b544c5639f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 18:20:59 +0200 Subject: [PATCH 047/124] Crond now runs in container (#29) --- .dockerignore | 1 + docker/Dockerfile.rel | 4 ++-- docker/entrypoint.sh | 5 +++++ fejctl | 3 +-- 4 files changed, 9 insertions(+), 4 deletions(-) create mode 100755 docker/entrypoint.sh diff --git a/.dockerignore b/.dockerignore index b7c1108..29fa987 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ # Entrypoint for devop container !docker/entrypoint_dev.sh +!docker/entrypoint.sh # Config file !Rocket.toml diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 90cefde..acb3cb4 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -26,7 +26,6 @@ FROM alpine:latest RUN apk update && \ apk add --no-cache \ curl \ - cron \ libgcc \ libpq \ openssl && \ @@ -43,6 +42,7 @@ COPY --from=builder --chown=fej:fej /app/output/bin /app/bin # The workdir is changed so that the config file is read properly WORKDIR /app COPY --chown=fej:fej Rocket.toml /app/Rocket.toml +COPY ./docker/entrypoint.sh /entrypoint.sh HEALTHCHECK \ --interval=10s \ @@ -51,4 +51,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -ENTRYPOINT ["/app/bin/server"] +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..5c345c7 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +crond -b + +exec "/app/bin/server" diff --git a/fejctl b/fejctl index 7ee69df..a8fc12d 100755 --- a/fejctl +++ b/fejctl @@ -11,7 +11,6 @@ function dc() { local OPTIND c build_builder release while getopts ":br" c; do - echo "$c" case $c in b ) build_builder=1 ;; r ) release=1 ;; @@ -99,7 +98,7 @@ function main() { # Running r | run ) dcr run --bin "$bin" && dc -- logs -f app ;; - rr | run-release ) dc -br -- run --build --detach ;; + rr | run-release ) dc -br -- up --build --detach && dc -r -- logs -f app ;; s | stop ) dc down ;; sr | stop-release ) dc -r stop ;; From fee37420cda3675b0ac5ec6f0fd9cddebc1a9787 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 20:52:29 +0200 Subject: [PATCH 048/124] Added cronjob container (closes #29) --- .dockerignore | 2 ++ docker/Dockerfile.rel | 5 +++-- docker/crontab | 1 + docker/docker-compose.yml | 10 ++++++++++ docker/entrypoint.sh | 5 ----- 5 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 docker/crontab delete mode 100755 docker/entrypoint.sh diff --git a/.dockerignore b/.dockerignore index 29fa987..a96bd42 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,5 @@ # Database migrations !migrations/ + +!docker/crontab diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index acb3cb4..4f4c54d 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -20,9 +20,11 @@ RUN cargo install \ # Now, we create the actual image FROM alpine:latest +COPY ./docker/crontab /var/spool/cron/crontabs/fej # Install some dynamic libraries needed for everything to work # Create -non-root user +# Change permissions for crontab file RUN apk update && \ apk add --no-cache \ curl \ @@ -42,7 +44,6 @@ COPY --from=builder --chown=fej:fej /app/output/bin /app/bin # The workdir is changed so that the config file is read properly WORKDIR /app COPY --chown=fej:fej Rocket.toml /app/Rocket.toml -COPY ./docker/entrypoint.sh /entrypoint.sh HEALTHCHECK \ --interval=10s \ @@ -51,4 +52,4 @@ HEALTHCHECK \ --retries=3 \ CMD curl -q localhost:8000 -ENTRYPOINT ["/entrypoint.sh"] +ENTRYPOINT ["/app/bin/server"] diff --git a/docker/crontab b/docker/crontab new file mode 100644 index 0000000..16fbbcd --- /dev/null +++ b/docker/crontab @@ -0,0 +1 @@ +# This'll be filled up later diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 14e90d2..4e1f9fb 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,6 +12,16 @@ services: environment: - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' + cron: + image: 'chewingbever/fej:latest' + restart: 'always' + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' + + entrypoint: 'crond -f' + user: 'root' + db: image: 'postgres:13-alpine' restart: 'always' diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index 5c345c7..0000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env sh - -crond -b - -exec "/app/bin/server" From 2c8f1ac601a55974f7fd49c9abdcec037751128c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 20:59:54 +0200 Subject: [PATCH 049/124] Removed unneeded healthcheck --- docker/docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4e1f9fb..55c5990 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -16,11 +16,13 @@ services: image: 'chewingbever/fej:latest' restart: 'always' - environment: - - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' - entrypoint: 'crond -f' user: 'root' + healthcheck: + disable: true + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' db: image: 'postgres:13-alpine' From a0e55e4830f00ef4b890afe92b16b9674e0dab9c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 21:39:29 +0200 Subject: [PATCH 050/124] Removed cron container from debug build --- docker/docker-compose.override.yml | 14 ++++++++++++++ docker/docker-compose.yml | 13 ------------- fejctl | 3 ++- 3 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 docker/docker-compose.override.yml diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml new file mode 100644 index 0000000..88c3a22 --- /dev/null +++ b/docker/docker-compose.override.yml @@ -0,0 +1,14 @@ +version: '2.4' + +services: + cron: + image: 'chewingbever/fej:latest' + restart: 'always' + + entrypoint: 'crond -f' + user: 'root' + healthcheck: + disable: true + + environment: + - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 55c5990..f2a19e0 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,18 +12,6 @@ services: environment: - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' - cron: - image: 'chewingbever/fej:latest' - restart: 'always' - - entrypoint: 'crond -f' - user: 'root' - healthcheck: - disable: true - - environment: - - 'DATABASE_URL=postgres://fej:fej@db:5432/fej' - db: image: 'postgres:13-alpine' restart: 'always' @@ -42,6 +30,5 @@ services: volumes: - 'db-data:/var/lib/postgresql/data' - volumes: db-data: diff --git a/fejctl b/fejctl index a8fc12d..6701343 100755 --- a/fejctl +++ b/fejctl @@ -31,6 +31,7 @@ function dc() { if [[ "$release" -eq 1 ]]; then DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ --file docker/docker-compose.yml \ + --file docker/docker-compose.override.yml \ --project-name fej \ "$@" @@ -112,7 +113,7 @@ function main() { l | logs ) dc -- logs -f app ;; lint ) cargo fmt -- --check ;; p | push | publish ) publish ;; - t | test ) dc test --no-fail-fast ;; + t | test ) dcr -- test --no-fail-fast && dc -- logs -f app ;; * ) >&2 echo "Invalid command."; exit 1 ;; esac } From 95564e81119a4bc937476dee60521306d5348cfa Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 21:44:20 +0200 Subject: [PATCH 051/124] Updated deps; bumped version --- Cargo.lock | 40 ++++++++++++++++++++-------------------- Cargo.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c229ee..acb8e84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28" +checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec" [[package]] name = "cookie" @@ -233,7 +233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3" dependencies = [ "cookie 0.14.4", - "idna 0.2.2", + "idna 0.2.3", "log 0.4.14", "publicsuffix", "serde", @@ -381,7 +381,7 @@ dependencies = [ [[package]] name = "fej" -version = "1.0.1" +version = "1.0.2" dependencies = [ "chrono", "chrono-tz", @@ -635,9 +635,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.6" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc35c995b9d93ec174cf9a27d425c7892722101e14993cd227fdb51d70cf9589" +checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" [[package]] name = "httpdate" @@ -714,9 +714,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", @@ -997,9 +997,9 @@ dependencies = [ [[package]] name = "notify" -version = "4.0.15" +version = "4.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd" +checksum = "2599080e87c9bd051ddb11b10074f4da7b1223298df65d4c2ec5bcf309af1533" dependencies = [ "bitflags", "filetime", @@ -1166,18 +1166,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc174859768806e91ae575187ada95c91a29e96a98dc5d2cd9a1fed039501ba6" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2 1.0.26", "quote 1.0.9", @@ -1258,7 +1258,7 @@ version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f" dependencies = [ - "idna 0.2.2", + "idna 0.2.3", "url 2.2.1", ] @@ -1333,9 +1333,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041" dependencies = [ "bitflags", ] @@ -1997,16 +1997,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" dependencies = [ "form_urlencoded", - "idna 0.2.2", + "idna 0.2.3", "matches", "percent-encoding 2.1.0", ] [[package]] name = "vcpkg" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" +checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" [[package]] name = "version_check" diff --git a/Cargo.toml b/Cargo.toml index 9677304..4d1dd7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fej" -version = "1.0.1" +version = "1.0.2" authors = ["Jef Roosens "] edition = "2018" From 91985a1710f395d313a22d19cf2ff030df7ff41b Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 22:39:22 +0200 Subject: [PATCH 052/124] [#28] Switched search_streets to database --- diesel.toml | 2 +- .../2021-04-15-155511_ivago_search/down.sql | 2 +- .../2021-04-15-155511_ivago_search/up.sql | 5 +-- src/fej/errors.rs | 8 +++++ src/fej/ivago/mod.rs | 33 ++++++++++++------- src/fej/ivago/street.rs | 2 ++ src/fej/lib.rs | 5 ++- src/fej/schema.rs | 6 ++++ src/schema.rs | 0 src/server/main.rs | 2 +- src/server/routes/ivago.rs | 5 +-- 11 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 src/fej/schema.rs delete mode 100644 src/schema.rs diff --git a/diesel.toml b/diesel.toml index 92267c8..779714e 100644 --- a/diesel.toml +++ b/diesel.toml @@ -2,4 +2,4 @@ # see diesel.rs/guides/configuring-diesel-cli [print_schema] -file = "src/schema.rs" +file = "src/fej/schema.rs" diff --git a/migrations/2021-04-15-155511_ivago_search/down.sql b/migrations/2021-04-15-155511_ivago_search/down.sql index 9d9d004..a5beedb 100644 --- a/migrations/2021-04-15-155511_ivago_search/down.sql +++ b/migrations/2021-04-15-155511_ivago_search/down.sql @@ -1,2 +1,2 @@ -- This file should undo anything in `up.sql` -DROP SCHEMA ivago CASCADE; +DROP table ivago_streets; diff --git a/migrations/2021-04-15-155511_ivago_search/up.sql b/migrations/2021-04-15-155511_ivago_search/up.sql index f33eff1..d4928af 100644 --- a/migrations/2021-04-15-155511_ivago_search/up.sql +++ b/migrations/2021-04-15-155511_ivago_search/up.sql @@ -1,7 +1,4 @@ --- Your SQL goes here -CREATE SCHEMA ivago; - -CREATE TABLE ivago.streets ( +CREATE TABLE ivago_streets ( name TEXT NOT NULL, city TEXT NOT NULL, diff --git a/src/fej/errors.rs b/src/fej/errors.rs index 730714c..3174b92 100644 --- a/src/fej/errors.rs +++ b/src/fej/errors.rs @@ -8,6 +8,7 @@ use rocket::http::Status; pub enum FejError { InvalidArgument, FailedRequest, + DatabaseError, } // I'd love to move this over to the server binary, but right now, error E0117 is making that @@ -17,6 +18,7 @@ impl From for Status { match err { FejError::InvalidArgument => Status::BadRequest, FejError::FailedRequest => Status::InternalServerError, + FejError::DatabaseError => Status::InternalServerError, } } } @@ -33,3 +35,9 @@ impl From for FejError { FejError::InvalidArgument } } + +impl From for FejError { + fn from(_: diesel::result::Error) -> FejError { + FejError::DatabaseError + } +} diff --git a/src/fej/ivago/mod.rs b/src/fej/ivago/mod.rs index 7d3a9f9..5879cef 100644 --- a/src/fej/ivago/mod.rs +++ b/src/fej/ivago/mod.rs @@ -1,6 +1,9 @@ use crate::errors::FejError; +use crate::schema::ivago_streets::dsl::*; use chrono::DateTime; use chrono_tz::Tz; +use diesel::prelude::*; +use diesel::PgConnection; use reqwest::blocking as reqwest; use std::collections::HashMap; use std::convert::{From, TryFrom}; @@ -25,19 +28,25 @@ const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/picku /// # Arguments /// /// * `search_term` - Search term to use to look for streets -pub fn search_streets(search_term: &str) -> Result, FejError> { - let client = reqwest::Client::new(); - let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; - let data: Vec> = response.json()?; +// pub fn search_streets(search_term: &str) -> Result, FejError> { +// let client = reqwest::Client::new(); +// let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; +// let data: Vec> = response.json()?; - // This is pretty cool, filter_map first does get() on all the maps, and - // then filters out any None values - // Then, we do the same thing for streets - Ok(data - .iter() - .filter_map(|m| m.get("value")) - .filter_map(|v| Street::try_from(v.as_str()).ok()) - .collect()) +// // This is pretty cool, filter_map first does get() on all the maps, and +// // then filters out any None values +// // Then, we do the same thing for streets +// Ok(data +// .iter() +// .filter_map(|m| m.get("value")) +// .filter_map(|v| Street::try_from(v.as_str()).ok()) +// .collect()) +// } +pub fn search_streets(db_con: &PgConnection, search_term: &str) -> Result, FejError> { + Ok(ivago_streets + .filter(name.like(search_term)) + .or_filter(city.like(search_term)) + .load(db_con)?) } /// Returns the pickup times for the various trash types. diff --git a/src/fej/ivago/street.rs b/src/fej/ivago/street.rs index 1a7cac9..a4da4b5 100644 --- a/src/fej/ivago/street.rs +++ b/src/fej/ivago/street.rs @@ -1,3 +1,4 @@ +use diesel::Queryable; use regex::Regex; use rocket::http::RawStr; use rocket::request::FromFormValue; @@ -5,6 +6,7 @@ use serde::ser::{Serialize, SerializeStruct, Serializer}; use std::convert::TryFrom; /// Represents a street in a given city +#[derive(Queryable)] pub struct Street { name: String, city: String, diff --git a/src/fej/lib.rs b/src/fej/lib.rs index 761c91b..f84d5d6 100644 --- a/src/fej/lib.rs +++ b/src/fej/lib.rs @@ -1,7 +1,10 @@ -#![feature(proc_macro_hygiene, decl_macro)] +#[macro_use] +extern crate diesel; // Route modules pub mod ivago; // Helper modules pub mod errors; + +pub mod schema; diff --git a/src/fej/schema.rs b/src/fej/schema.rs new file mode 100644 index 0000000..3fe23f2 --- /dev/null +++ b/src/fej/schema.rs @@ -0,0 +1,6 @@ +table! { + ivago_streets (name, city) { + name -> Text, + city -> Text, + } +} diff --git a/src/schema.rs b/src/schema.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/server/main.rs b/src/server/main.rs index 4267776..aa9de11 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -47,7 +47,7 @@ embed_migrations!(); // This defines a connection to the database #[database("postgres_fej")] -struct FejDbConn(diesel::PgConnection); +pub struct FejDbConn(diesel::PgConnection); // I'd like to thank Stackoverflow for helping me with this // https://stackoverflow.com/questions/61047355/how-to-run-diesel-migration-with-rocket-in-production diff --git a/src/server/routes/ivago.rs b/src/server/routes/ivago.rs index d8318bd..a9b9336 100644 --- a/src/server/routes/ivago.rs +++ b/src/server/routes/ivago.rs @@ -1,3 +1,4 @@ +use crate::FejDbConn; use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street}; use rocket::http::Status; use rocket_contrib::json::Json; @@ -9,8 +10,8 @@ use rocket_contrib::json::Json; /// /// * `search_term` - Search term to use to look for streets #[get("/search?")] -pub fn route_search_streets(q: String) -> Result>, Status> { - Ok(Json(search_streets(q.as_str())?)) +pub fn route_search_streets(db_conn: FejDbConn, q: String) -> Result>, Status> { + Ok(Json(search_streets(&db_conn.0, q.as_str())?)) } /// Handles returning of pickup times for a specific address. It returns a list From 0877cc590cd671916192495f0eb92cf1b83af811 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 22:44:15 +0200 Subject: [PATCH 053/124] [#28] Moved db stuff to own module --- src/fej/ivago/db.rs | 12 ++++++++++++ src/fej/ivago/mod.rs | 34 +++++++++++++--------------------- src/server/main.rs | 2 +- src/server/routes/ivago.rs | 3 ++- 4 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 src/fej/ivago/db.rs diff --git a/src/fej/ivago/db.rs b/src/fej/ivago/db.rs new file mode 100644 index 0000000..43f94e9 --- /dev/null +++ b/src/fej/ivago/db.rs @@ -0,0 +1,12 @@ +use super::Street; +use crate::errors::FejError; +use crate::schema::ivago_streets::dsl::*; +use diesel::prelude::*; +use diesel::PgConnection; + +pub fn search_streets(db_con: &PgConnection, search_term: &str) -> Result, FejError> { + Ok(ivago_streets + .filter(name.like(search_term)) + .or_filter(city.like(search_term)) + .load(db_con)?) +} diff --git a/src/fej/ivago/mod.rs b/src/fej/ivago/mod.rs index 5879cef..a6d8295 100644 --- a/src/fej/ivago/mod.rs +++ b/src/fej/ivago/mod.rs @@ -1,14 +1,12 @@ use crate::errors::FejError; -use crate::schema::ivago_streets::dsl::*; use chrono::DateTime; use chrono_tz::Tz; -use diesel::prelude::*; -use diesel::PgConnection; use reqwest::blocking as reqwest; use std::collections::HashMap; use std::convert::{From, TryFrom}; mod basic_date; +pub mod db; mod pickup_time; mod street; @@ -28,25 +26,19 @@ const CAL_URL: &str = "https://www.ivago.be/nl/particulier/garbage/pick-up/picku /// # Arguments /// /// * `search_term` - Search term to use to look for streets -// pub fn search_streets(search_term: &str) -> Result, FejError> { -// let client = reqwest::Client::new(); -// let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; -// let data: Vec> = response.json()?; +pub fn search_streets(search_term: &str) -> Result, FejError> { + let client = reqwest::Client::new(); + let response = client.get(SEARCH_URL).query(&[("q", search_term)]).send()?; + let data: Vec> = response.json()?; -// // This is pretty cool, filter_map first does get() on all the maps, and -// // then filters out any None values -// // Then, we do the same thing for streets -// Ok(data -// .iter() -// .filter_map(|m| m.get("value")) -// .filter_map(|v| Street::try_from(v.as_str()).ok()) -// .collect()) -// } -pub fn search_streets(db_con: &PgConnection, search_term: &str) -> Result, FejError> { - Ok(ivago_streets - .filter(name.like(search_term)) - .or_filter(city.like(search_term)) - .load(db_con)?) + // This is pretty cool, filter_map first does get() on all the maps, and + // then filters out any None values + // Then, we do the same thing for streets + Ok(data + .iter() + .filter_map(|m| m.get("value")) + .filter_map(|v| Street::try_from(v.as_str()).ok()) + .collect()) } /// Returns the pickup times for the various trash types. diff --git a/src/server/main.rs b/src/server/main.rs index aa9de11..0e84ff8 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -55,7 +55,7 @@ fn run_db_migrations(rocket: Rocket) -> Result { let conn = FejDbConn::get_one(&rocket).expect("database connection"); match embedded_migrations::run(&*conn) { Ok(()) => Ok(rocket), - Err(e) => Err(rocket), + Err(_) => Err(rocket), } } diff --git a/src/server/routes/ivago.rs b/src/server/routes/ivago.rs index a9b9336..e8c0a9e 100644 --- a/src/server/routes/ivago.rs +++ b/src/server/routes/ivago.rs @@ -1,5 +1,6 @@ use crate::FejDbConn; -use fej::ivago::{get_pickup_times, search_streets, BasicDate, PickupTime, Street}; +use fej::ivago::db::search_streets; +use fej::ivago::{get_pickup_times, BasicDate, PickupTime, Street}; use rocket::http::Status; use rocket_contrib::json::Json; From 85611e2d5929afdfb7c9c21f71f08f81166bef41 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 23:20:41 +0200 Subject: [PATCH 054/124] [#28] Added basic ivago scraper --- Cargo.toml | 8 ++++++++ src/fej/ivago/db.rs | 4 ++-- src/fej/ivago/street.rs | 6 ++++-- src/populate_ivago.rs | 19 +++++++++++++++++++ 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 src/populate_ivago.rs diff --git a/Cargo.toml b/Cargo.toml index 4d1dd7e..4274991 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,14 @@ test = true bench = true doc = true doctest = true + +[[bin]] +name = "populate_ivago" +path = "src/populate_ivago.rs" +test = false +bench = false +doc = false +doctest = false [dependencies] rocket = "0.4.7" diff --git a/src/fej/ivago/db.rs b/src/fej/ivago/db.rs index 43f94e9..6ba3174 100644 --- a/src/fej/ivago/db.rs +++ b/src/fej/ivago/db.rs @@ -6,7 +6,7 @@ use diesel::PgConnection; pub fn search_streets(db_con: &PgConnection, search_term: &str) -> Result, FejError> { Ok(ivago_streets - .filter(name.like(search_term)) - .or_filter(city.like(search_term)) + .filter(name.ilike(format!("%{}%", search_term))) + .or_filter(city.ilike(format!("%{}%", search_term))) .load(db_con)?) } diff --git a/src/fej/ivago/street.rs b/src/fej/ivago/street.rs index a4da4b5..2f30f06 100644 --- a/src/fej/ivago/street.rs +++ b/src/fej/ivago/street.rs @@ -1,4 +1,5 @@ -use diesel::Queryable; +use crate::schema::ivago_streets; +use diesel::{Insertable, Queryable}; use regex::Regex; use rocket::http::RawStr; use rocket::request::FromFormValue; @@ -6,7 +7,8 @@ use serde::ser::{Serialize, SerializeStruct, Serializer}; use std::convert::TryFrom; /// Represents a street in a given city -#[derive(Queryable)] +#[derive(Queryable, Insertable)] +#[table_name = "ivago_streets"] pub struct Street { name: String, city: String, diff --git a/src/populate_ivago.rs b/src/populate_ivago.rs new file mode 100644 index 0000000..9dfc876 --- /dev/null +++ b/src/populate_ivago.rs @@ -0,0 +1,19 @@ +use diesel::{insert_into, Connection, PgConnection, RunQueryDsl}; +use fej::ivago::search_streets; +use fej::schema::ivago_streets::dsl::*; + +const ABC: &str = "abcdefghijklmnopqrstuvwxyz"; + +fn main() { + let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + + let db_conn = PgConnection::establish(&database_url) + .expect(&format!("Error connecting to {}", database_url)); + + // Doing this linearly is good enough I'd say + for c in ABC.chars() { + if let Ok(streets) = search_streets(&c.to_string()) { + insert_into(ivago_streets).values(streets).execute(&db_conn); + } + } +} From ecec03134ad4205505055e6f9386150006fa0ab4 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 17 Apr 2021 23:29:32 +0200 Subject: [PATCH 055/124] Added populate_ivago to crontab --- docker/crontab | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/crontab b/docker/crontab index 16fbbcd..1a9b072 100644 --- a/docker/crontab +++ b/docker/crontab @@ -1 +1,2 @@ -# This'll be filled up later +# Runs the ivago scraper +*/30 * * * * /app/bin/populate_ivago From bf8a0ae666f24d132c0ace8810b696f20f2680e7 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 20 Apr 2021 08:00:45 +0000 Subject: [PATCH 056/124] Add renovate.json --- renovate.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..7190a60 --- /dev/null +++ b/renovate.json @@ -0,0 +1,3 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json" +} From b93d7713725f5cb9ddf9563e537324b0b31de3c3 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 20 Apr 2021 09:09:25 +0000 Subject: [PATCH 057/124] Update Rust crate reqwest to 0.11.3 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4274991..c3e5c06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ serde = "1.0.124" chrono = "0.4.19" chrono-tz = "0.5.3" regex = "1.4.5" -reqwest = { version = "0.11.2", features = ["blocking", "json", "cookies"] } +reqwest = { version = "0.11.3", features = ["blocking", "json", "cookies"] } diesel = { version = "1.4.6", features = ["postgres"] } diesel_migrations = "1.4.0" From 26ec8b4eba64992cb1cf398f2e53ab09f4befa71 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 20 Apr 2021 09:09:29 +0000 Subject: [PATCH 058/124] Update Rust crate serde to 1.0.125 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4274991..8737109 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ doctest = false [dependencies] rocket = "0.4.7" -serde = "1.0.124" +serde = "1.0.125" chrono = "0.4.19" chrono-tz = "0.5.3" regex = "1.4.5" From 00820765273ce86b3cd8c9f00d656ccfd6c6c595 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Thu, 22 Apr 2021 22:04:08 +0000 Subject: [PATCH 059/124] Update Rust crate regex to 1.4.6 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acb8e84..1d51c09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1342,9 +1342,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.5" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759" dependencies = [ "aho-corasick", "memchr", diff --git a/Cargo.toml b/Cargo.toml index cf11ad9..97668d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ rocket = "0.4.7" serde = "1.0.125" chrono = "0.4.19" chrono-tz = "0.5.3" -regex = "1.4.5" +regex = "1.4.6" reqwest = { version = "0.11.3", features = ["blocking", "json", "cookies"] } diesel = { version = "1.4.6", features = ["postgres"] } diesel_migrations = "1.4.0" From 9e428b18821ca2f6d791c46cc34b01cd4ff69ea8 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 24 Apr 2021 00:20:20 +0200 Subject: [PATCH 060/124] Added test .drone.yml --- .drone.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..0627887 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,5 @@ +pipeline: + first-test: + image: alpine + commands: + - echo "woodpecker is very cool" From 62738354a0f44107e3fd0c71b0f27c7466260677 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 24 Apr 2021 00:27:43 +0200 Subject: [PATCH 061/124] Just a commit --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index 0627887..01d2202 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3,3 +3,4 @@ pipeline: image: alpine commands: - echo "woodpecker is very cool" + From 0abfaca967b0e7bae408ee41c841ce4f464c6b3c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 24 Apr 2021 09:33:42 +0200 Subject: [PATCH 062/124] Added test cicd job --- .drone.yml | 6 ------ .woodpecker.yml | 5 +++++ docker/Dockerfile.builder | 7 +------ docker/Dockerfile.dev | 4 ++++ docker/Dockerfile.rel | 6 +++++- 5 files changed, 15 insertions(+), 13 deletions(-) delete mode 100644 .drone.yml create mode 100644 .woodpecker.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 01d2202..0000000 --- a/.drone.yml +++ /dev/null @@ -1,6 +0,0 @@ -pipeline: - first-test: - image: alpine - commands: - - echo "woodpecker is very cool" - diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..2703d03 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,5 @@ +pipeline: + test: + image: chewingbever/fej-builder:latest + commands: + - cargo test diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index b4152fa..d50e8e4 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -1,6 +1,6 @@ # vim: filetype=dockerfile # Our entire toolchain runs in alpine -FROM alpine:latest AS builder +FROM alpine:3.13.5 AS builder ENV PATH "$PATH:/app/.cargo/bin" # Needed for proper compiling of openssl-dev @@ -29,8 +29,3 @@ WORKDIR /app RUN { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ rustup target add x86_64-unknown-linux-musl --toolchain nightly && \ mkdir -p .cargo/registry target - -# Copy source code over to builder -COPY --chown=builder:builder Cargo.toml Cargo.lock ./ -COPY --chown=builder:builder src/ ./src/ -COPY --chown=builder:builder migrations/ ./migrations/ diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index bac8b44..7aa9d09 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -3,6 +3,10 @@ FROM chewingbever/fej-builder:latest ENV RUST_BACKTRACE 1 +# Copy source code over to builder +COPY --chown=builder:builder Cargo.toml Cargo.lock ./ +COPY --chown=builder:builder src/ ./src/ +COPY --chown=builder:builder migrations/ ./migrations/ COPY --chown=builder:builder ./docker/entrypoint_dev.sh /entrypoint.sh COPY --chown=builder:builder ./Rocket.toml /app/Rocket.toml diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 4f4c54d..79bd7bb 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -1,6 +1,10 @@ # vim: filetype=dockerfile FROM chewingbever/fej-builder:latest AS builder +COPY --chown=builder:builder Cargo.toml Cargo.lock ./ +COPY --chown=builder:builder src/ ./src/ +COPY --chown=builder:builder migrations/ ./migrations/ + # And then finally, build the project # Thank the lords that this article exists # https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 @@ -19,7 +23,7 @@ RUN cargo install \ # Now, we create the actual image -FROM alpine:latest +FROM alpine:3.12.5 COPY ./docker/crontab /var/spool/cron/crontabs/fej # Install some dynamic libraries needed for everything to work From 3bdf79e824d2f13efdaff6d97d8e930ea0336bf9 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 24 Apr 2021 09:47:28 +0200 Subject: [PATCH 063/124] Made builder root again --- docker/Dockerfile.builder | 10 ++-------- fejctl | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder index d50e8e4..1073062 100644 --- a/docker/Dockerfile.builder +++ b/docker/Dockerfile.builder @@ -2,7 +2,7 @@ # Our entire toolchain runs in alpine FROM alpine:3.13.5 AS builder -ENV PATH "$PATH:/app/.cargo/bin" +ENV PATH "$PATH:/root/.cargo/bin" # Needed for proper compiling of openssl-dev ENV RUSTFLAGS "-C target-feature=-crt-static" @@ -19,13 +19,7 @@ RUN addgroup -S builder && \ openssl-dev \ postgresql-dev -# Switch to the non-root user -USER builder - -WORKDIR /app - # Install rustup in the new user's home # Create mountpoints for volumes with correct permissions RUN { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly; } && \ - rustup target add x86_64-unknown-linux-musl --toolchain nightly && \ - mkdir -p .cargo/registry target + rustup target add x86_64-unknown-linux-musl --toolchain nightly diff --git a/fejctl b/fejctl index 6701343..6f5c8a9 100755 --- a/fejctl +++ b/fejctl @@ -18,16 +18,6 @@ function dc() { done shift $((OPTIND-1)) - if [[ "$build_builder" -eq 1 ]]; then - # We always rebuild the builder before we run any compose command - DOCKER_BUILDKIT=1 docker build \ - -f docker/Dockerfile.builder \ - -t "$image-builder:latest" . || { - >&2 echo "Failed to build builder."; - exit 1; - } - fi - if [[ "$release" -eq 1 ]]; then DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose \ --file docker/docker-compose.yml \ @@ -53,6 +43,18 @@ function dcr() { --detach } +# Publish the builder image +function publish_builder() { + DOCKER_BUILDKIT=1 docker build \ + -f docker/Dockerfile.builder \ + -t "$image-builder:latest" . || { + >&2 echo "Failed to build builder."; + exit 1; + } + + docker push "$image-builder:latest" +} + # Tags & pushes the release version to Docker Hub function publish() { local branch=`git rev-parse --abbrev-ref HEAD` @@ -113,6 +115,7 @@ function main() { l | logs ) dc -- logs -f app ;; lint ) cargo fmt -- --check ;; p | push | publish ) publish ;; + pb ) publish_builder ;; t | test ) dcr -- test --no-fail-fast && dc -- logs -f app ;; * ) >&2 echo "Invalid command."; exit 1 ;; esac From caa92377f6087ee2b8231a2c79faaff5bef27839 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sat, 24 Apr 2021 08:03:09 +0000 Subject: [PATCH 064/124] Update alpine Docker tag to v3.13.5 --- docker/Dockerfile.rel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 79bd7bb..4bffeae 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -23,7 +23,7 @@ RUN cargo install \ # Now, we create the actual image -FROM alpine:3.12.5 +FROM alpine:3.13.5 COPY ./docker/crontab /var/spool/cron/crontabs/fej # Install some dynamic libraries needed for everything to work From 77e04d5b6bf844c520e91ede382fc2264f3bad23 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 24 Apr 2021 12:44:53 +0200 Subject: [PATCH 065/124] Added cicd restriction [CI SKIP] --- .woodpecker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 2703d03..78e9241 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,5 +1,9 @@ pipeline: test: image: chewingbever/fej-builder:latest + # Always update the builder image + pull: true commands: - cargo test + +branches: [ master, develop ] From 687632a7045d03d3bec2da87d81961d08196e96f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 29 Apr 2021 09:58:29 +0200 Subject: [PATCH 066/124] [#44] Added frontend source code --- web/.env | 1 + web/.env.production | 1 + web/.gitignore | 5 +++ web/README.md | 27 ++++++++++++++++ web/index.html | 13 ++++++++ web/package.json | 20 ++++++++++++ web/public/favicon.ico | Bin 0 -> 4286 bytes web/src/App.vue | 27 ++++++++++++++++ web/src/api/ivago.ts | 32 ++++++++++++++++++ web/src/assets/logo.png | Bin 0 -> 6849 bytes web/src/components/HelloWorld.vue | 51 +++++++++++++++++++++++++++++ web/src/components/Home.vue | 13 ++++++++ web/src/components/Ivago.vue | 52 ++++++++++++++++++++++++++++++ web/src/components/Nav.vue | 43 ++++++++++++++++++++++++ web/src/main.ts | 8 +++++ web/src/router.ts | 21 ++++++++++++ web/src/shims-vue.d.ts | 5 +++ web/tsconfig.json | 15 +++++++++ web/vite.config.ts | 7 ++++ 19 files changed, 341 insertions(+) create mode 100644 web/.env create mode 100644 web/.env.production create mode 100644 web/.gitignore create mode 100644 web/README.md create mode 100644 web/index.html create mode 100644 web/package.json create mode 100644 web/public/favicon.ico create mode 100644 web/src/App.vue create mode 100644 web/src/api/ivago.ts create mode 100644 web/src/assets/logo.png create mode 100644 web/src/components/HelloWorld.vue create mode 100644 web/src/components/Home.vue create mode 100644 web/src/components/Ivago.vue create mode 100644 web/src/components/Nav.vue create mode 100644 web/src/main.ts create mode 100644 web/src/router.ts create mode 100644 web/src/shims-vue.d.ts create mode 100644 web/tsconfig.json create mode 100644 web/vite.config.ts diff --git a/web/.env b/web/.env new file mode 100644 index 0000000..88668b6 --- /dev/null +++ b/web/.env @@ -0,0 +1 @@ +VITE_ENDPOINT=http://localhost:8000 diff --git a/web/.env.production b/web/.env.production new file mode 100644 index 0000000..cf6e652 --- /dev/null +++ b/web/.env.production @@ -0,0 +1 @@ +VITE_ENDPOINT=https://fej.roosens.me/api diff --git a/web/.gitignore b/web/.gitignore new file mode 100644 index 0000000..d451ff1 --- /dev/null +++ b/web/.gitignore @@ -0,0 +1,5 @@ +node_modules +.DS_Store +dist +dist-ssr +*.local diff --git a/web/README.md b/web/README.md new file mode 100644 index 0000000..a797a27 --- /dev/null +++ b/web/README.md @@ -0,0 +1,27 @@ +# Vue 3 + Typescript + Vite + +This template should help get you started developing with Vue 3 and Typescript in Vite. + +## Recommended IDE Setup + +[VSCode](https://code.visualstudio.com/) + [Vetur](https://marketplace.visualstudio.com/items?itemName=octref.vetur). Make sure to enable `vetur.experimental.templateInterpolationService` in settings! + +### If Using ` + + diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000..ae1647d --- /dev/null +++ b/web/package.json @@ -0,0 +1,20 @@ +{ + "name": "fej-frontend", + "version": "0.0.0", + "scripts": { + "dev": "vite", + "build": "vue-tsc --noEmit && vite build", + "serve": "vite preview" + }, + "dependencies": { + "vue": "^3.0.5", + "vue-router": "^4.0.6" + }, + "devDependencies": { + "@vitejs/plugin-vue": "^1.2.1", + "@vue/compiler-sfc": "^3.0.5", + "typescript": "^4.1.3", + "vite": "^2.2.1", + "vue-tsc": "^0.0.25" + } +} diff --git a/web/public/favicon.ico b/web/public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..df36fcfb72584e00488330b560ebcf34a41c64c2 GIT binary patch literal 4286 zcmds*O-Phc6o&64GDVCEQHxsW(p4>LW*W<827=Unuo8sGpRux(DN@jWP-e29Wl%wj zY84_aq9}^Am9-cWTD5GGEo#+5Fi2wX_P*bo+xO!)p*7B;iKlbFd(U~_d(U?#hLj56 zPhFkj-|A6~Qk#@g^#D^U0XT1cu=c-vu1+SElX9NR;kzAUV(q0|dl0|%h|dI$%VICy zJnu2^L*Te9JrJMGh%-P79CL0}dq92RGU6gI{v2~|)p}sG5x0U*z<8U;Ij*hB9z?ei z@g6Xq-pDoPl=MANPiR7%172VA%r)kevtV-_5H*QJKFmd;8yA$98zCxBZYXTNZ#QFk2(TX0;Y2dt&WitL#$96|gJY=3xX zpCoi|YNzgO3R`f@IiEeSmKrPSf#h#Qd<$%Ej^RIeeYfsxhPMOG`S`Pz8q``=511zm zAm)MX5AV^5xIWPyEu7u>qYs?pn$I4nL9J!=K=SGlKLXpE<5x+2cDTXq?brj?n6sp= zphe9;_JHf40^9~}9i08r{XM$7HB!`{Ys~TK0kx<}ZQng`UPvH*11|q7&l9?@FQz;8 zx!=3<4seY*%=OlbCbcae?5^V_}*K>Uo6ZWV8mTyE^B=DKy7-sdLYkR5Z?paTgK-zyIkKjIcpyO z{+uIt&YSa_$QnN_@t~L014dyK(fOOo+W*MIxbA6Ndgr=Y!f#Tokqv}n<7-9qfHkc3 z=>a|HWqcX8fzQCT=dqVbogRq!-S>H%yA{1w#2Pn;=e>JiEj7Hl;zdt-2f+j2%DeVD zsW0Ab)ZK@0cIW%W7z}H{&~yGhn~D;aiP4=;m-HCo`BEI+Kd6 z={Xwx{TKxD#iCLfl2vQGDitKtN>z|-AdCN|$jTFDg0m3O`WLD4_s#$S literal 0 HcmV?d00001 diff --git a/web/src/App.vue b/web/src/App.vue new file mode 100644 index 0000000..ca4df88 --- /dev/null +++ b/web/src/App.vue @@ -0,0 +1,27 @@ + + + + + diff --git a/web/src/api/ivago.ts b/web/src/api/ivago.ts new file mode 100644 index 0000000..662e531 --- /dev/null +++ b/web/src/api/ivago.ts @@ -0,0 +1,32 @@ +export class Street { + name: string; + city: string; + + constructor(name: string, city: string) { + this.name = name; + this.city = city; + } +} + +export class Ivago { + base_url: string; + + constructor(url: string) { + this.base_url = url; + } + + async search(search_term: string): Promise { + var r = await fetch(`${this.base_url}/ivago/search?` + new URLSearchParams({ + q: search_term, + })); + + if (!r.ok) { + return Promise.reject(); + } + + var json = await r.json(); + + return json.map((o: {name: string, city: string}) => new Street(o.name, o.city)); + } +} + diff --git a/web/src/assets/logo.png b/web/src/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..f3d2503fc2a44b5053b0837ebea6e87a2d339a43 GIT binary patch literal 6849 zcmaKRcUV(fvo}bjDT-7nLI_nlK}sT_69H+`qzVWDA|yaU?}j417wLi^B1KB1SLsC& zL0ag7$U(XW5YR7p&Ux?sP$d4lvMt8C^+TcQu4F zQqv!UF!I+kw)c0jhd6+g6oCr9P?7)?!qX1ui*iL{p}sKCAGuJ{{W)0z1pLF|=>h}& zt(2Lr0Z`2ig8<5i%Zk}cO5Fm=LByqGWaS`oqChZdEFmc`0hSb#gg|Aap^{+WKOYcj zHjINK)KDG%&s?Mt4CL(T=?;~U@bU2x_mLKN!#GJuK_CzbNw5SMEJorG!}_5;?R>@1 zSl)jns3WlU7^J%=(hUtfmuUCU&C3%8B5C^f5>W2Cy8jW3#{Od{lF1}|?c61##3dzA zsPlFG;l_FzBK}8>|H_Ru_H#!_7$UH4UKo3lKOA}g1(R&|e@}GINYVzX?q=_WLZCgh z)L|eJMce`D0EIwgRaNETDsr+?vQknSGAi=7H00r`QnI%oQnFxm`G2umXso9l+8*&Q z7WqF|$p49js$mdzo^BXpH#gURy=UO;=IMrYc5?@+sR4y_?d*~0^YP7d+y0{}0)zBM zIKVM(DBvICK#~7N0a+PY6)7;u=dutmNqK3AlsrUU9U`d;msiucB_|8|2kY=(7XA;G zwDA8AR)VCA#JOkxm#6oHNS^YVuOU;8p$N)2{`;oF|rQ?B~K$%rHDxXs+_G zF5|-uqHZvSzq}L;5Kcy_P+x0${33}Ofb6+TX&=y;;PkEOpz%+_bCw_{<&~ zeLV|!bP%l1qxywfVr9Z9JI+++EO^x>ZuCK);=$VIG1`kxK8F2M8AdC$iOe3cj1fo(ce4l-9 z7*zKy3={MixvUk=enQE;ED~7tv%qh&3lR<0m??@w{ILF|e#QOyPkFYK!&Up7xWNtL zOW%1QMC<3o;G9_S1;NkPB6bqbCOjeztEc6TsBM<(q9((JKiH{01+Ud=uw9B@{;(JJ z-DxI2*{pMq`q1RQc;V8@gYAY44Z!%#W~M9pRxI(R?SJ7sy7em=Z5DbuDlr@*q|25V)($-f}9c#?D%dU^RS<(wz?{P zFFHtCab*!rl(~j@0(Nadvwg8q|4!}L^>d?0al6}Rrv9$0M#^&@zjbfJy_n!%mVHK4 z6pLRIQ^Uq~dnyy$`ay51Us6WaP%&O;@49m&{G3z7xV3dLtt1VTOMYl3UW~Rm{Eq4m zF?Zl_v;?7EFx1_+#WFUXxcK78IV)FO>42@cm@}2I%pVbZqQ}3;p;sDIm&knay03a^ zn$5}Q$G!@fTwD$e(x-~aWP0h+4NRz$KlnO_H2c< z(XX#lPuW_%H#Q+c&(nRyX1-IadKR-%$4FYC0fsCmL9ky3 zKpxyjd^JFR+vg2!=HWf}2Z?@Td`0EG`kU?{8zKrvtsm)|7>pPk9nu@2^z96aU2<#` z2QhvH5w&V;wER?mopu+nqu*n8p~(%QkwSs&*0eJwa zMXR05`OSFpfyRb!Y_+H@O%Y z0=K^y6B8Gcbl?SA)qMP3Z+=C(?8zL@=74R=EVnE?vY!1BQy2@q*RUgRx4yJ$k}MnL zs!?74QciNb-LcG*&o<9=DSL>1n}ZNd)w1z3-0Pd^4ED1{qd=9|!!N?xnXjM!EuylY z5=!H>&hSofh8V?Jofyd!h`xDI1fYAuV(sZwwN~{$a}MX^=+0TH*SFp$vyxmUv7C*W zv^3Gl0+eTFgBi3FVD;$nhcp)ka*4gSskYIqQ&+M}xP9yLAkWzBI^I%zR^l1e?bW_6 zIn{mo{dD=)9@V?s^fa55jh78rP*Ze<3`tRCN4*mpO$@7a^*2B*7N_|A(Ve2VB|)_o z$=#_=aBkhe(ifX}MLT()@5?OV+~7cXC3r!%{QJxriXo9I%*3q4KT4Xxzyd{ z9;_%=W%q!Vw$Z7F3lUnY+1HZ*lO;4;VR2+i4+D(m#01OYq|L_fbnT;KN<^dkkCwtd zF7n+O7KvAw8c`JUh6LmeIrk4`F3o|AagKSMK3))_5Cv~y2Bb2!Ibg9BO7Vkz?pAYX zoI=B}+$R22&IL`NCYUYjrdhwjnMx_v=-Qcx-jmtN>!Zqf|n1^SWrHy zK|MwJ?Z#^>)rfT5YSY{qjZ&`Fjd;^vv&gF-Yj6$9-Dy$<6zeP4s+78gS2|t%Z309b z0^fp~ue_}i`U9j!<|qF92_3oB09NqgAoehQ`)<)dSfKoJl_A6Ec#*Mx9Cpd-p#$Ez z={AM*r-bQs6*z$!*VA4|QE7bf@-4vb?Q+pPKLkY2{yKsw{&udv_2v8{Dbd zm~8VAv!G~s)`O3|Q6vFUV%8%+?ZSVUa(;fhPNg#vab@J*9XE4#D%)$UU-T5`fwjz! z6&gA^`OGu6aUk{l*h9eB?opVdrHK>Q@U>&JQ_2pR%}TyOXGq_6s56_`U(WoOaAb+K zXQr#6H}>a-GYs9^bGP2Y&hSP5gEtW+GVC4=wy0wQk=~%CSXj=GH6q z-T#s!BV`xZVxm{~jr_ezYRpqqIcXC=Oq`b{lu`Rt(IYr4B91hhVC?yg{ol4WUr3v9 zOAk2LG>CIECZ-WIs0$N}F#eoIUEtZudc7DPYIjzGqDLWk_A4#(LgacooD z2K4IWs@N`Bddm-{%oy}!k0^i6Yh)uJ1S*90>|bm3TOZxcV|ywHUb(+CeX-o1|LTZM zwU>dY3R&U)T(}5#Neh?-CWT~@{6Ke@sI)uSuzoah8COy)w)B)aslJmp`WUcjdia-0 zl2Y}&L~XfA`uYQboAJ1;J{XLhYjH){cObH3FDva+^8ioOQy%Z=xyjGLmWMrzfFoH; zEi3AG`_v+%)&lDJE;iJWJDI@-X9K5O)LD~j*PBe(wu+|%ar~C+LK1+-+lK=t# z+Xc+J7qp~5q=B~rD!x78)?1+KUIbYr^5rcl&tB-cTtj+e%{gpZZ4G~6r15+d|J(ky zjg@@UzMW0k9@S#W(1H{u;Nq(7llJbq;;4t$awM;l&(2s+$l!Ay9^Ge|34CVhr7|BG z?dAR83smef^frq9V(OH+a+ki#q&-7TkWfFM=5bsGbU(8mC;>QTCWL5ydz9s6k@?+V zcjiH`VI=59P-(-DWXZ~5DH>B^_H~;4$)KUhnmGo*G!Tq8^LjfUDO)lASN*=#AY_yS zqW9UX(VOCO&p@kHdUUgsBO0KhXxn1sprK5h8}+>IhX(nSXZKwlNsjk^M|RAaqmCZB zHBolOHYBas@&{PT=R+?d8pZu zUHfyucQ`(umXSW7o?HQ3H21M`ZJal+%*)SH1B1j6rxTlG3hx1IGJN^M7{$j(9V;MZ zRKybgVuxKo#XVM+?*yTy{W+XHaU5Jbt-UG33x{u(N-2wmw;zzPH&4DE103HV@ER86 z|FZEmQb|&1s5#`$4!Cm}&`^{(4V}OP$bk`}v6q6rm;P!H)W|2i^e{7lTk2W@jo_9q z*aw|U7#+g59Fv(5qI`#O-qPj#@_P>PC#I(GSp3DLv7x-dmYK=C7lPF8a)bxb=@)B1 zUZ`EqpXV2dR}B&r`uM}N(TS99ZT0UB%IN|0H%DcVO#T%L_chrgn#m6%x4KE*IMfjX zJ%4veCEqbXZ`H`F_+fELMC@wuy_ch%t*+Z+1I}wN#C+dRrf2X{1C8=yZ_%Pt6wL_~ zZ2NN-hXOT4P4n$QFO7yYHS-4wF1Xfr-meG9Pn;uK51?hfel`d38k{W)F*|gJLT2#T z<~>spMu4(mul-8Q3*pf=N4DcI)zzjqAgbE2eOT7~&f1W3VsdD44Ffe;3mJp-V@8UC z)|qnPc12o~$X-+U@L_lWqv-RtvB~%hLF($%Ew5w>^NR82qC_0FB z)=hP1-OEx?lLi#jnLzH}a;Nvr@JDO-zQWd}#k^an$Kwml;MrD&)sC5b`s0ZkVyPkb zt}-jOq^%_9>YZe7Y}PhW{a)c39G`kg(P4@kxjcYfgB4XOOcmezdUI7j-!gs7oAo2o zx(Ph{G+YZ`a%~kzK!HTAA5NXE-7vOFRr5oqY$rH>WI6SFvWmahFav!CfRMM3%8J&c z*p+%|-fNS_@QrFr(at!JY9jCg9F-%5{nb5Bo~z@Y9m&SHYV`49GAJjA5h~h4(G!Se zZmK{Bo7ivCfvl}@A-ptkFGcWXAzj3xfl{evi-OG(TaCn1FAHxRc{}B|x+Ua1D=I6M z!C^ZIvK6aS_c&(=OQDZfm>O`Nxsw{ta&yiYPA~@e#c%N>>#rq)k6Aru-qD4(D^v)y z*>Rs;YUbD1S8^D(ps6Jbj0K3wJw>L4m)0e(6Pee3Y?gy9i0^bZO?$*sv+xKV?WBlh zAp*;v6w!a8;A7sLB*g-^<$Z4L7|5jXxxP1}hQZ<55f9<^KJ>^mKlWSGaLcO0=$jem zWyZkRwe~u{{tU63DlCaS9$Y4CP4f?+wwa(&1ou)b>72ydrFvm`Rj-0`kBJgK@nd(*Eh!(NC{F-@=FnF&Y!q`7){YsLLHf0_B6aHc# z>WIuHTyJwIH{BJ4)2RtEauC7Yq7Cytc|S)4^*t8Va3HR zg=~sN^tp9re@w=GTx$;zOWMjcg-7X3Wk^N$n;&Kf1RgVG2}2L-(0o)54C509C&77i zrjSi{X*WV=%C17((N^6R4Ya*4#6s_L99RtQ>m(%#nQ#wrRC8Y%yxkH;d!MdY+Tw@r zjpSnK`;C-U{ATcgaxoEpP0Gf+tx);buOMlK=01D|J+ROu37qc*rD(w`#O=3*O*w9?biwNoq3WN1`&Wp8TvKj3C z3HR9ssH7a&Vr<6waJrU zdLg!ieYz%U^bmpn%;(V%%ugMk92&?_XX1K@mwnVSE6!&%P%Wdi7_h`CpScvspMx?N zQUR>oadnG17#hNc$pkTp+9lW+MBKHRZ~74XWUryd)4yd zj98$%XmIL4(9OnoeO5Fnyn&fpQ9b0h4e6EHHw*l68j;>(ya`g^S&y2{O8U>1*>4zR zq*WSI_2o$CHQ?x0!wl9bpx|Cm2+kFMR)oMud1%n2=qn5nE&t@Fgr#=Zv2?}wtEz^T z9rrj=?IH*qI5{G@Rn&}^Z{+TW}mQeb9=8b<_a`&Cm#n%n~ zU47MvCBsdXFB1+adOO)03+nczfWa#vwk#r{o{dF)QWya9v2nv43Zp3%Ps}($lA02*_g25t;|T{A5snSY?3A zrRQ~(Ygh_ebltHo1VCbJb*eOAr;4cnlXLvI>*$-#AVsGg6B1r7@;g^L zFlJ_th0vxO7;-opU@WAFe;<}?!2q?RBrFK5U{*ai@NLKZ^};Ul}beukveh?TQn;$%9=R+DX07m82gP$=}Uo_%&ngV`}Hyv8g{u z3SWzTGV|cwQuFIs7ZDOqO_fGf8Q`8MwL}eUp>q?4eqCmOTcwQuXtQckPy|4F1on8l zP*h>d+cH#XQf|+6c|S{7SF(Lg>bR~l(0uY?O{OEVlaxa5@e%T&xju=o1`=OD#qc16 zSvyH*my(dcp6~VqR;o(#@m44Lug@~_qw+HA=mS#Z^4reBy8iV?H~I;{LQWk3aKK8$bLRyt$g?- +

Fej Frontend

+ + + + +
    +
  • {{res.name}} ({{res.city}})
  • +
+ + + + + diff --git a/web/src/components/Home.vue b/web/src/components/Home.vue new file mode 100644 index 0000000..f5cc87a --- /dev/null +++ b/web/src/components/Home.vue @@ -0,0 +1,13 @@ + + + diff --git a/web/src/components/Ivago.vue b/web/src/components/Ivago.vue new file mode 100644 index 0000000..923f85c --- /dev/null +++ b/web/src/components/Ivago.vue @@ -0,0 +1,52 @@ + + + + + diff --git a/web/src/components/Nav.vue b/web/src/components/Nav.vue new file mode 100644 index 0000000..fcf9c26 --- /dev/null +++ b/web/src/components/Nav.vue @@ -0,0 +1,43 @@ + + + + + diff --git a/web/src/main.ts b/web/src/main.ts new file mode 100644 index 0000000..af7463e --- /dev/null +++ b/web/src/main.ts @@ -0,0 +1,8 @@ +import { createApp } from 'vue'; +import App from './App.vue'; +import router from './router'; + +const app = createApp(App) +app.use(router) + +app.mount('#app') diff --git a/web/src/router.ts b/web/src/router.ts new file mode 100644 index 0000000..2d7fa58 --- /dev/null +++ b/web/src/router.ts @@ -0,0 +1,21 @@ +import { createWebHistory, createRouter } from "vue-router"; +import Home from './components/Home.vue'; +import Ivago from './components/Ivago.vue'; + +const routes = [ + { + path: "/", + component: Home, + }, + { + path: "/ivago", + component: Ivago, + }, +]; + +const router = createRouter({ + history: createWebHistory(), + routes, +}); + +export default router; diff --git a/web/src/shims-vue.d.ts b/web/src/shims-vue.d.ts new file mode 100644 index 0000000..ac1ded7 --- /dev/null +++ b/web/src/shims-vue.d.ts @@ -0,0 +1,5 @@ +declare module '*.vue' { + import { DefineComponent } from 'vue' + const component: DefineComponent<{}, {}, any> + export default component +} diff --git a/web/tsconfig.json b/web/tsconfig.json new file mode 100644 index 0000000..e754e65 --- /dev/null +++ b/web/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "esnext", + "moduleResolution": "node", + "strict": true, + "jsx": "preserve", + "sourceMap": true, + "resolveJsonModule": true, + "esModuleInterop": true, + "lib": ["esnext", "dom"], + "types": ["vite/client"] + }, + "include": ["src/**/*.ts", "src/**/*.d.ts", "src/**/*.tsx", "src/**/*.vue"] +} diff --git a/web/vite.config.ts b/web/vite.config.ts new file mode 100644 index 0000000..315212d --- /dev/null +++ b/web/vite.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vite' +import vue from '@vitejs/plugin-vue' + +// https://vitejs.dev/config/ +export default defineConfig({ + plugins: [vue()] +}) From 0055667877b0b94c288e7628eff8ddf4968aa565 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 29 Apr 2021 10:02:44 +0200 Subject: [PATCH 067/124] [#44] [#44] Added proper Node.js gitignore --- web/.gitignore | 133 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 129 insertions(+), 4 deletions(-) diff --git a/web/.gitignore b/web/.gitignore index d451ff1..04459e4 100644 --- a/web/.gitignore +++ b/web/.gitignore @@ -1,5 +1,130 @@ -node_modules -.DS_Store +# Created by https://www.toptal.com/developers/gitignore/api/node +# Edit at https://www.toptal.com/developers/gitignore?templates=node + +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test +.env*.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next + +# Nuxt.js build / generate output +.nuxt dist -dist-ssr -*.local + +# Storybook build outputs +.out +.storybook-out +storybook-static + +# rollup.js default build output +dist/ + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# Temporary folders +tmp/ +temp/ + +# End of https://www.toptal.com/developers/gitignore/api/node From 628a918682afd68702efb6807fa2c484a36a7d6d Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 29 Apr 2021 10:23:07 +0200 Subject: [PATCH 068/124] [#44] Fej can now easily startup a Vite development server --- fejctl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fejctl b/fejctl index 6f5c8a9..57f7443 100755 --- a/fejctl +++ b/fejctl @@ -1,6 +1,7 @@ #!/usr/bin/env bash image='chewingbever/fej' +web_dir='web' # Small wrapper around the docker-compose command # @@ -98,10 +99,12 @@ function main() { # Building b | build ) dcr build --bin "$bin" && dc -- logs -f app ;; br | build-release ) dc -br build ;; + bf | build-frontend ) cd "$web_dir" && yarn run build ;; # Running r | run ) dcr run --bin "$bin" && dc -- logs -f app ;; rr | run-release ) dc -br -- up --build --detach && dc -r -- logs -f app ;; + rf | run-frontend ) dcr run --bin server && cd "$web_dir" && yarn run dev ;; s | stop ) dc down ;; sr | stop-release ) dc -r stop ;; From e0ac8ddd240f28056e753498eb0f41ed95323771 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 29 Apr 2021 10:45:42 +0200 Subject: [PATCH 069/124] [#44] [#44] Release container now builds & packages frontend as well --- .dockerignore | 32 +++++++++++++++++++++++++++----- docker/Dockerfile.rel | 17 ++++++++++++++--- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.dockerignore b/.dockerignore index a96bd42..b143673 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,42 @@ * -# Source code +# =====BACKEND===== +## Source code !src/ -# Cargo files +## Cargo files !Cargo.toml !Cargo.lock -# Entrypoint for devop container +## Entrypoint for devop container !docker/entrypoint_dev.sh !docker/entrypoint.sh -# Config file +## Config file !Rocket.toml -# Database migrations +## Database migrations !migrations/ +## Crontab for release container !docker/crontab + + +# =====FRONTEND===== +## Source code +!web/src/ + +## Public assets +!web/public/ +!web/index.html + +## Production env file +!web/.env.production + +## Package manager stuff +!web/package.json +!web/yarn.lock + +## Project configs +!web/tsconfig.json +!web/vite.config.ts diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index 4bffeae..a01d343 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -1,5 +1,5 @@ # vim: filetype=dockerfile -FROM chewingbever/fej-builder:latest AS builder +FROM chewingbever/fej-builder:latest AS backend-builder COPY --chown=builder:builder Cargo.toml Cargo.lock ./ COPY --chown=builder:builder src/ ./src/ @@ -22,6 +22,16 @@ RUN cargo install \ --target x86_64-unknown-linux-musl +FROM node:15-alpine3.13 AS frontend-builder + +COPY ./web /app +WORKDIR /app + +# Build the frontend +RUN yarn install && \ + yarn run build + + # Now, we create the actual image FROM alpine:3.13.5 COPY ./docker/crontab /var/spool/cron/crontabs/fej @@ -41,8 +51,9 @@ RUN apk update && \ # Switch to non-root user USER fej:fej -# Copy binary over to final image -COPY --from=builder --chown=fej:fej /app/output/bin /app/bin +# Copy binary & frontend over to final image +COPY --from=backend-builder --chown=fej:fej /app/output/bin /app/bin +COPY --from=frontend-builder --chown=fej:fej /app/dist /app/dist # Embed config file inside container # The workdir is changed so that the config file is read properly From 5a95ee5270db5a7cbbb88282b564cbaeb61701c9 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 29 Apr 2021 11:34:35 +0200 Subject: [PATCH 070/124] [#44] Added frontend hosting using feature --- Cargo.toml | 6 +++++- docker/Dockerfile.rel | 3 ++- src/server/main.rs | 17 ++++++++++++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 97668d2..cf6db71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,10 @@ version = "1.0.2" authors = ["Jef Roosens "] edition = "2018" +[features] +# Enables hosting of the frontend +frontend = [] + [lib] name = "fej" path = "src/fej/lib.rs" @@ -41,4 +45,4 @@ diesel_migrations = "1.4.0" [dependencies.rocket_contrib] version = "0.4.7" default-features = false -features = ["json", "diesel_postgres_pool"] +features = ["json", "diesel_postgres_pool", "serve"] diff --git a/docker/Dockerfile.rel b/docker/Dockerfile.rel index a01d343..5a17f05 100644 --- a/docker/Dockerfile.rel +++ b/docker/Dockerfile.rel @@ -19,7 +19,8 @@ COPY --chown=builder:builder migrations/ ./migrations/ RUN cargo install \ --path . \ --root /app/output \ - --target x86_64-unknown-linux-musl + --target x86_64-unknown-linux-musl \ + --features frontend FROM node:15-alpine3.13 AS frontend-builder diff --git a/src/server/main.rs b/src/server/main.rs index 0e84ff8..2a5bd2f 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -20,6 +20,8 @@ use rocket::fairing::{Fairing, Info, Kind}; use rocket::http::Header; use rocket::{Request, Response, Rocket}; use rocket_contrib::databases::diesel; +#[cfg(feature = "frontend")] +use rocket_contrib::serve::StaticFiles; pub struct CORS; @@ -60,12 +62,21 @@ fn run_db_migrations(rocket: Rocket) -> Result { } fn rocket() -> rocket::Rocket { - rocket::ignite() + // This needs to be muted for the frontend feature + let mut rocket = rocket::ignite() .attach(CORS) .attach(FejDbConn::fairing()) .attach(AdHoc::on_attach("Database Migrations", run_db_migrations)) - .mount("/ivago", routes::ivago()) - .register(catchers![catchers::not_found]) + .mount("/api/ivago", routes::ivago()) // /api being hardcoded is temporary + .register(catchers![catchers::not_found]); + + // TODO make all of this not hard-coded + #[cfg(feature = "frontend")] + { + rocket = rocket.mount("/", StaticFiles::from("/app/dist")); + } + + rocket } fn main() { From 28bb959c1227fac5f4a88d2455174bd9f8ad649f Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 29 Apr 2021 11:37:35 +0200 Subject: [PATCH 071/124] [#44] Fixed development .env file --- web/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/.env b/web/.env index 88668b6..18f38c1 100644 --- a/web/.env +++ b/web/.env @@ -1 +1 @@ -VITE_ENDPOINT=http://localhost:8000 +VITE_ENDPOINT=http://localhost:8000/api From d2ad0591f6f2b62880157fa9989c1253699623a5 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Thu, 29 Apr 2021 11:43:53 +0200 Subject: [PATCH 072/124] [#44] Added some stuff to cicd file --- .woodpecker.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 78e9241..6199967 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,9 +1,18 @@ pipeline: - test: + build-frontend: + image: node:15-alpine3.13 + pull: true + commands: + - cd web && yarn install + - cd web && yarn run build + + test-backend: image: chewingbever/fej-builder:latest # Always update the builder image pull: true commands: - cargo test + # TODO build dev & rel image, deploy these images + branches: [ master, develop ] From f27775910f72cc1669eca112c97acae1c005bc8a Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Fri, 30 Apr 2021 00:24:19 +0200 Subject: [PATCH 073/124] Simplified Ivago API wrapper in frontend --- web/index.html | 2 +- web/src/api/ivago.ts | 38 ++++++++++++------------------------ web/src/components/Ivago.vue | 5 ++--- 3 files changed, 15 insertions(+), 30 deletions(-) diff --git a/web/index.html b/web/index.html index 11603f8..f6a2ff1 100644 --- a/web/index.html +++ b/web/index.html @@ -4,7 +4,7 @@ - Vite App + Fej
diff --git a/web/src/api/ivago.ts b/web/src/api/ivago.ts index 662e531..3d9375d 100644 --- a/web/src/api/ivago.ts +++ b/web/src/api/ivago.ts @@ -1,32 +1,18 @@ -export class Street { - name: string; - city: string; - - constructor(name: string, city: string) { - this.name = name; - this.city = city; - } +export interface Street { + name: string + city: string } -export class Ivago { - base_url: string; +export async function search(search_term: string): Promise { + var base_url = import.meta.env.VITE_ENDPOINT as string - constructor(url: string) { - this.base_url = url; + var r = await fetch(`${base_url}/ivago/search?` + new URLSearchParams({ + q: search_term, + })) + + if (!r.ok) { + return Promise.reject() } - async search(search_term: string): Promise { - var r = await fetch(`${this.base_url}/ivago/search?` + new URLSearchParams({ - q: search_term, - })); - - if (!r.ok) { - return Promise.reject(); - } - - var json = await r.json(); - - return json.map((o: {name: string, city: string}) => new Street(o.name, o.city)); - } + return r.json() } - diff --git a/web/src/components/Ivago.vue b/web/src/components/Ivago.vue index 923f85c..77c6042 100644 --- a/web/src/components/Ivago.vue +++ b/web/src/components/Ivago.vue @@ -11,7 +11,7 @@ diff --git a/web/src/api/ivago.ts b/web/src/api/ivago.ts index 3d9375d..4c01856 100644 --- a/web/src/api/ivago.ts +++ b/web/src/api/ivago.ts @@ -3,11 +3,11 @@ export interface Street { city: string } -export async function search(search_term: string): Promise { - var base_url = import.meta.env.VITE_ENDPOINT as string +export async function search (search_term: string): Promise { + const base_url = import.meta.env.VITE_ENDPOINT as string - var r = await fetch(`${base_url}/ivago/search?` + new URLSearchParams({ - q: search_term, + const r = await fetch(`${base_url}/ivago/search?` + new URLSearchParams({ + q: search_term })) if (!r.ok) { diff --git a/web/src/components/HelloWorld.vue b/web/src/components/HelloWorld.vue index 12f05c8..1ddbac8 100644 --- a/web/src/components/HelloWorld.vue +++ b/web/src/components/HelloWorld.vue @@ -15,18 +15,18 @@ import { Street, Ivago } from '../api/ivago' export default defineComponent({ name: 'HelloWorld', - data() { + data () { return { - street: "", - results: [] as Street[], + street: '', + results: [] as Street[] } }, methods: { - onSubmit() { + onSubmit () { new Ivago(import.meta.env.VITE_ENDPOINT as string).search(this.$data.street) .then(res => { - this.$data.results = res; - }); + this.$data.results = res + }) } } }) diff --git a/web/src/components/Home.vue b/web/src/components/Home.vue index f5cc87a..cdb41ed 100644 --- a/web/src/components/Home.vue +++ b/web/src/components/Home.vue @@ -8,6 +8,6 @@ import { defineComponent } from 'vue' export default defineComponent({ - name: 'Home', + name: 'Home' }) diff --git a/web/src/components/Ivago.vue b/web/src/components/Ivago.vue index 77c6042..be2779b 100644 --- a/web/src/components/Ivago.vue +++ b/web/src/components/Ivago.vue @@ -3,7 +3,7 @@
    -
  • {{ item.name }}
  • +
  • {{ item.name }} ({{ item.city }})

{{ msg }}

@@ -11,31 +11,31 @@ diff --git a/web/src/main.ts b/web/src/main.ts index af7463e..daff2db 100644 --- a/web/src/main.ts +++ b/web/src/main.ts @@ -1,6 +1,6 @@ -import { createApp } from 'vue'; -import App from './App.vue'; -import router from './router'; +import { createApp } from 'vue' +import App from './App.vue' +import router from './router' const app = createApp(App) app.use(router) diff --git a/web/src/router.ts b/web/src/router.ts index 2d7fa58..2668326 100644 --- a/web/src/router.ts +++ b/web/src/router.ts @@ -1,21 +1,21 @@ -import { createWebHistory, createRouter } from "vue-router"; -import Home from './components/Home.vue'; -import Ivago from './components/Ivago.vue'; +import { createWebHistory, createRouter } from 'vue-router' +import Home from './components/Home.vue' +import Ivago from './components/Ivago.vue' const routes = [ { - path: "/", - component: Home, + path: '/', + component: Home }, { - path: "/ivago", - component: Ivago, - }, -]; + path: '/ivago', + component: Ivago + } +] const router = createRouter({ history: createWebHistory(), - routes, -}); + routes +}) -export default router; +export default router From 39ad811429da5935bcc813b93bdb7e07d4b2df46 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sat, 1 May 2021 18:00:28 +0000 Subject: [PATCH 078/124] Update dependency vue-tsc to ^0.0.26 --- web/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/package.json b/web/package.json index 3bbe454..740439c 100644 --- a/web/package.json +++ b/web/package.json @@ -25,6 +25,6 @@ "eslint-plugin-vue": "^7.9.0", "typescript": "^4.1.3", "vite": "^2.2.1", - "vue-tsc": "^0.0.25" + "vue-tsc": "^0.0.26" } } From 9f31c697774295a80c09481a2e6f675bd4b71c68 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sat, 1 May 2021 18:00:34 +0000 Subject: [PATCH 079/124] Update dependency eslint-plugin-promise to v5 --- web/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/package.json b/web/package.json index 3bbe454..e37101c 100644 --- a/web/package.json +++ b/web/package.json @@ -21,7 +21,7 @@ "eslint-config-standard": "^16.0.2", "eslint-plugin-import": "^2.22.1", "eslint-plugin-node": "^11.1.0", - "eslint-plugin-promise": "^4.3.1", + "eslint-plugin-promise": "^5.0.0", "eslint-plugin-vue": "^7.9.0", "typescript": "^4.1.3", "vite": "^2.2.1", From 6532fbebb116a1630a36f2b0a2e82e1d2a541981 Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 1 May 2021 21:11:13 +0200 Subject: [PATCH 080/124] Fixed frontend linter --- web/package.json | 4 ++-- web/src/api/ivago.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/web/package.json b/web/package.json index e19388b..0534f03 100644 --- a/web/package.json +++ b/web/package.json @@ -5,8 +5,8 @@ "dev": "vite", "build": "vue-tsc --noEmit && vite build", "serve": "vite preview", - "lint": "eslint \"src/**\"", - "format": "eslint --fix \"src/**\"" + "lint": "eslint --ext .js,.vue src", + "format": "yarn run lint --fix" }, "dependencies": { "vue": "^3.0.5", diff --git a/web/src/api/ivago.ts b/web/src/api/ivago.ts index 4c01856..7fa15a4 100644 --- a/web/src/api/ivago.ts +++ b/web/src/api/ivago.ts @@ -3,11 +3,11 @@ export interface Street { city: string } -export async function search (search_term: string): Promise { - const base_url = import.meta.env.VITE_ENDPOINT as string +export async function search (searchTerm: string): Promise { + const baseURL = import.meta.env.VITE_ENDPOINT as string - const r = await fetch(`${base_url}/ivago/search?` + new URLSearchParams({ - q: search_term + const r = await fetch(`${baseURL}/ivago/search?` + new URLSearchParams({ + q: searchTerm })) if (!r.ok) { From 1da94c5211daa394dbaa7ca9e1edb2c20b1ea08e Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 1 May 2021 21:22:12 +0200 Subject: [PATCH 081/124] Fixed eslint errors --- web/src/App.vue | 6 ++-- web/src/components/HelloWorld.vue | 51 ------------------------------- web/src/components/Home.vue | 8 +++-- web/src/components/Ivago.vue | 28 +++++++++++------ 4 files changed, 28 insertions(+), 65 deletions(-) delete mode 100644 web/src/components/HelloWorld.vue diff --git a/web/src/App.vue b/web/src/App.vue index f19b1c3..f629716 100644 --- a/web/src/App.vue +++ b/web/src/App.vue @@ -1,6 +1,8 @@ - - diff --git a/web/src/components/Home.vue b/web/src/components/Home.vue index cdb41ed..181d6e9 100644 --- a/web/src/components/Home.vue +++ b/web/src/components/Home.vue @@ -1,7 +1,9 @@