Merge pull request 'Update alpine Docker tag to v3.15' (#9) from renovate/alpine-3.x into develop

Reviewed-on: #9

.dockerignore

@@ -0,0 +1,5 @@
+*
+
+!Cargo.toml
+!Cargo.lock
+!src/

.editorconfig

@@ -0,0 +1,13 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = false
+indent_size = 4
+indent_style = space
+
+[*.{yml,yaml}]
+indent_size = 2
+
+[Makefile]
+indent_style = tab

.env

@@ -0,0 +1,2 @@
+# This file is solely used by Diesel
+DATABASE_URL=postgres://hilde:hilde@localhost/hilde

.woodpecker.yml

@@ -0,0 +1,77 @@
+pipeline:
+  # Download the cache from S3
+  restore-cache:
+    image: plugins/s3-cache
+    pull: true
+
+    endpoint: https://s3.roosens.me
+    root: build-cache/
+    restore: true
+
+    secrets: [ cache_s3_access_key, cache_s3_secret_key ]
+
+
+  # =====BUILDING=====
+  build-backend:
+    image: rust:1.53-alpine
+    pull: true
+    group: build
+    environment:
+      - CARGO_HOME=.cargo
+    commands:
+      - apk add musl-dev postgresql-dev
+      - cargo build
+
+
+  # =====TESTING=====
+  test-backend:
+    image: rust:1.53-alpine
+    environment:
+      - CARGO_HOME=.cargo
+    commands:
+      - apk add musl-dev postgresql-dev
+      - cargo test
+
+
+  # =====LINTING=====
+  lint-backend:
+    image: rust:1.53-alpine
+    group: lint
+    environment:
+      - CARGO_HOME=.cargo
+    commands:
+      - apk add musl-dev postgresql-dev
+      - rustup component add rustfmt clippy
+      - cargo fmt -- --check
+      # This is run here because it requires compilation
+      - cargo clippy --all-targets -- -D warnings
+  
+
+  # =====REBUILD & FLUSH CACHE=====
+  rebuild-cache:
+    image: plugins/s3-cache
+
+    endpoint: https://s3.roosens.me
+    root: build-cache/
+    rebuild: true
+    mount:
+      - target
+      - .cargo
+
+    secrets: [ cache_s3_access_key, cache_s3_secret_key ]
+    # Push the cache, even on failure
+    when:
+      status: [ success, failure ]
+
+  flush-cache:
+    image: plugins/s3-cache
+
+    endpoint: https://s3.roosens.me
+    root: build-cache/
+    flush: true
+    flush_age: 14
+
+    secrets: [ cache_s3_access_key, cache_s3_secret_key ]
+    # Push the cache, even on failure
+    when:
+      status: [ success, failure ]

Cargo.lock

@@ -84,6 +84,12 @@ version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
 
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+
 [[package]]
 name = "bytes"
 version = "1.0.1"
@@ -170,6 +176,40 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "diesel"
+version = "1.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d"
+dependencies = [
+ "bitflags",
+ "byteorder",
+ "diesel_derives",
+ "pq-sys",
+ "r2d2",
+]
+
+[[package]]
+name = "diesel_derives"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "diesel_migrations"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c"
+dependencies = [
+ "migrations_internals",
+ "migrations_macros",
+]
+
 [[package]]
 name = "discard"
 version = "1.0.4"
@@ -193,9 +233,9 @@ dependencies = [
 
 [[package]]
 name = "figment"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ca029e813a72b7526d28273d25f3e4a2f365d1b7a1018a6f93ec9053a119763"
+checksum = "790b4292c72618abbab50f787a477014fe15634f96291de45672ce46afe122df"
 dependencies = [
  "atomic",
  "pear",
@@ -362,9 +402,9 @@ checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
 dependencies = [
  "libc",
 ]
@@ -373,7 +413,10 @@ dependencies = [
 name = "hilde"
 version = "0.1.0"
 dependencies = [
+ "diesel",
+ "diesel_migrations",
  "rocket",
+ "rocket_sync_db_pools",
 ]
 
 [[package]]
@@ -515,6 +558,27 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
 
+[[package]]
+name = "migrations_internals"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860"
+dependencies = [
+ "diesel",
+]
+
+[[package]]
+name = "migrations_macros"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c"
+dependencies = [
+ "migrations_internals",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -645,9 +709,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.6"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905"
+checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443"
 
 [[package]]
 name = "pin-utils"
@@ -661,6 +725,15 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
 
+[[package]]
+name = "pq-sys"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda"
+dependencies = [
+ "vcpkg",
+]
+
 [[package]]
 name = "proc-macro-hack"
 version = "0.5.19"
@@ -704,6 +777,17 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "r2d2"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f"
+dependencies = [
+ "log",
+ "parking_lot",
+ "scheduled-thread-pool",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.4"
@@ -746,9 +830,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
+checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
 dependencies = [
  "bitflags",
 ]
@@ -864,6 +948,30 @@ dependencies = [
  "uncased",
 ]
 
+[[package]]
+name = "rocket_sync_db_pools"
+version = "0.1.0-rc.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38cfdfebd552d075c368e641c88a5cd6ce1c58c5c710548aeb777abb48830f4b"
+dependencies = [
+ "diesel",
+ "r2d2",
+ "rocket",
+ "rocket_sync_db_pools_codegen",
+ "serde",
+ "tokio",
+]
+
+[[package]]
+name = "rocket_sync_db_pools_codegen"
+version = "0.1.0-rc.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5267808c094db5366e1d8925aaf9f2ce05ff9b3bd92cb18c7040a1fe219c2e25"
+dependencies = [
+ "devise",
+ "quote",
+]
+
 [[package]]
 name = "rustc_version"
 version = "0.2.3"
@@ -885,6 +993,15 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
+[[package]]
+name = "scheduled-thread-pool"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7"
+dependencies = [
+ "parking_lot",
+]
+
 [[package]]
 name = "scoped-tls"
 version = "1.0.0"
@@ -982,9 +1099,9 @@ dependencies = [
 
 [[package]]
 name = "spin"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b87bbf98cb81332a56c1ee8929845836f85e8ddd693157c30d76660196014478"
+checksum = "f5fdd7196b4ae35a111c6dc97f9cc152ca3ea8ad744f7cb46a9f27b3ef8f2f54"
 
 [[package]]
 name = "stable-pattern"
@@ -1006,9 +1123,9 @@ dependencies = [
 
 [[package]]
 name = "state"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b54c22963194db84a59ee48e1fa9ed6c1fa9909ad5db92a700aa6fe956d632b"
+checksum = "87cf4f5369e6d3044b5e365c9690f451516ac8f0954084622b49ea3fde2f6de5"
 dependencies = [
  "loom",
 ]
@@ -1127,9 +1244,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.7.0"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c79ba603c337335df6ba6dd6afc38c38a7d5e1b0c871678439ea973cd62a118e"
+checksum = "5fb2ed024293bb19f7a5dc54fe83bf86532a44c12a2bb8ba40d64a4509395ca2"
 dependencies = [
  "autocfg",
  "bytes",
@@ -1262,6 +1379,12 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.3"

Cargo.toml

@@ -1,10 +1,17 @@
 [package]
 name = "hilde"
 version = "0.1.0"
-authors = ["jjr <roosensjef@gmail.com>"]
+authors = ["Jef Roosens <roosensjef@gmail.com>"]
 edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 rocket = "0.5.0-rc.1"
+diesel = { version = "1.4.8", features = ["postgres"] }
+diesel_migrations = "1.4.0"
+
+[dependencies.rocket_sync_db_pools]
+version = "0.1.0-rc.1"
+default_features = false
+features = ["diesel_postgres_pool"]

Dockerfile

@@ -0,0 +1,22 @@
+FROM rust:1.57-alpine3.13 AS builder
+
+WORKDIR /src
+
+# Install build dependencies
+RUN apk add --no-cache \
+    musl-dev \
+    postgresql-dev
+
+# Copy over source code
+COPY Cargo.toml Cargo.lock ./
+COPY src ./src/
+
+RUN cargo build \
+    --release
+
+
+FROM alpine:3.15
+
+COPY --from=builder /src/target/release/hilde /usr/local/bin/hilde
+
+ENTRYPOINT [ "/usr/local/bin/hilde" ]

Makefile

@@ -0,0 +1,59 @@
+# The main usecase for this Makefile is to simplify database management
+
+
+all: build
+.PHONY: all
+
+
+# =====CARGO STUFF=====
+build:
+	@ cargo build
+.PHONY: build
+
+run:
+	@ cargo run
+.PHONY: run
+
+test:
+	@ cargo test
+.PHONY: test
+
+lint:
+	@ cargo fmt -- --check
+	@ cargo clippy --all-targets -- -D warnings
+.PHONY: lint
+
+format:
+	@ cargo fmt
+.PHONY: format
+
+
+# =====DATABASE STUFF=====
+db:
+	@ docker run \
+		--rm \
+		-itd \
+		-v hilde_db-data:/var/lib/postgresql/data \
+		-e POSTGRES_USER=hilde \
+		-e POSTGRES_PASSWORD=hilde \
+		-e POSTGRES_DB=hilde \
+		--name hilde_db \
+		-p 5432:5432 \
+		postgres:13-alpine
+.PHONY: db
+
+psql:
+	@ docker exec \
+		-it hilde_db \
+		psql -U hilde
+.PHONY: psql
+
+stop-db:
+	@ docker stop hilde_db
+.PHONY: stop-db
+
+
+# =====DOCKER STUFF=====
+image:
+	@ docker build -t hilde:latest .
+.PHONY: image

README.md

@@ -1,11 +1,19 @@
 # hilde
 
-## What?
+A server implementation of a Debian repository.
 
-A server implementation of a Debian repository, written in Rust.
+## Development
 
-## Why?
+To develop this project, you'll need a few things:
 
-The original reason was that I wanted to create my own "AUR" for Debian,
+* Docker (for setting up the database)
-populating it using my CI pipeline. And of course, the only true valid reason,
+* Cargo & Rust stable (all hail Rocket v0.5)
-it sounded fun :)
+* The PostgreSQL dev libraries (to compile Diesel)
+
+That's the ones I can think of right now. If I think of any others, I'll be
+sure to add them to the list.
+
+For ease of development, there's a Makefile wrapper that can do everything you
+need to do to work on Hilde. I recommend reading through it (it's not long),
+but the important ones are `make` & `make run`, which are just Cargo aliases,
+and `make db`, which starts a local PostgreSQL database using Docker.

ROADMAP.md

@@ -1,86 +0,0 @@
-# Roadmap
-
-This file describes a broad roadmap of what's currently planned with the
-project.
-
-## Readable repository
-
-The first goal is to make the server a fully functional Debian package
-repository. This includes:
-
-### Setting up a database
-
-Hilde stores an index of the stored deb files in a PostgreSQL database. Any
-data that can be calculated beforehand and/or stored efficiently inside a
-database (e.g. hashes, package names & versions, descriptions etc.) will be
-stored there. Any request that isn't a deb file will be queried and/or
-generated from this database. For example, the `Packages` file can be
-generated on the fly instead of storing it on disk. The goal is for the data
-storage to only include deb files.
-
-### Indexing the directory on startup
-
-On startup, Hilde will scan the packages directory for deb files. These will
-then all be parsed, and synced with the database.
-
-### Functioning as a repository
-
-Using this database backend, the server should fully mimic a Debian repository
-server ([an example](http://ftp.debian.org/debian/)). As there's no real reason
-not to, we should implement the
-[repository spec](https://wiki.debian.org/DebianRepository) as complete as
-possible.
-
-## The ideas
-
-Now that we have a repository that we can use, it's time to add some extra
-functionality.
-
-### Uploading versions
-
-This is the initial reason I wanted to start this project. My goal is to add a
-way to send POST requests to the server containing new versions of a new or
-existing package. These packages can be from any distribution, architecture, or
-name, and the server will just dynamically create the directories n stuff when
-needed. This feature can then be used in CI pipelines to simplify uploading
-versions to a repository.
-
-### User management & JWT tokens
-
-Considering we can't just make an upload endpoint publicly available, we'll
-need a system to log in. There will be an admin user that can do everything,
-but I want to add a detailed user management system. This system will have
-fine-grained controls over what user can do what, and each user is able to
-create JWT tokens that further narrows down the functionality. For example, a
-user could have read & write access to the stable distribution's updates
-section. Then they can create a token that only allows to write a certain
-package, and use it in their CI pipeline for that particular package.
-
-### Federation
-
-I think the idea of being able to spread a service across multiple hosts is
-really cool, so I've thought about adding this to Hilde as well. I have never
-created anything like this before, but I think the idea is really cool. My
-current approach would be as follows:
-
-The nodes communicate using a common secret key. Each node only requires the
-location of a single other node. When a new node connects to the network, it
-asks the node it's been given about which nodes it knows. These nodes are then
-added to the node's list of known nodes. These new nodes are then also
-notified, and this process is repeated until all nodes are discovered. Each
-node also keeps an index of which packages can be found on which node. This
-makes it so that every node can be used as an entrypoint to the network, making
-it easy to load balance.
-
-Each node could maybe send a notification to all the other nodes when it
-discovers/loses a node, or receives a new package.
-
-When uploading a new package, we could query the system to see which node has
-the most space left, as to more efficiently distribute the packages, if
-possible.
-
-### Mirroring repositories
-
-It might be useful to add functionality to mirror an existing repo. This could
-for example make a home setup fully self-sufficient, by mirroring the official
-Debian repositories.

Rocket.toml

@@ -0,0 +1,2 @@
+[debug.databases]
+postgres_hilde = { url = "postgres://hilde:hilde@localhost/hilde" }

diesel.toml

@@ -0,0 +1,5 @@
+# For documentation on how to configure this file,
+# see diesel.rs/guides/configuring-diesel-cli
+
+[print_schema]
+file = "src/schema.rs"

migrations/00000000000000_diesel_initial_setup/down.sql

@@ -0,0 +1,6 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
+DROP FUNCTION IF EXISTS diesel_set_updated_at();

migrations/00000000000000_diesel_initial_setup/up.sql

@@ -0,0 +1,36 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+
+
+
+-- Sets up a trigger for the given table to automatically set a column called
+-- `updated_at` whenever the row is modified (unless `updated_at` was included
+-- in the modified columns)
+--
+-- # Example
+--
+-- ```sql
+-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
+--
+-- SELECT diesel_manage_updated_at('users');
+-- ```
+CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
+BEGIN
+    EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
+                    FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
+BEGIN
+    IF (
+        NEW IS DISTINCT FROM OLD AND
+        NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
+    ) THEN
+        NEW.updated_at := current_timestamp;
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;

src/main.rs

@@ -1,3 +1,34 @@
-fn main() {
+#[macro_use]
-    println!("Hello, world!");
+extern crate rocket;
+
+#[macro_use]
+extern crate diesel_migrations;
+
+use rocket::{fairing::AdHoc, Build, Rocket};
+use rocket_sync_db_pools::{database, diesel};
+
+embed_migrations!();
+
+#[database("postgres_hilde")]
+struct HildeDbConn(diesel::PgConnection);
+
+async fn run_db_migrations(rocket: Rocket<Build>) -> Result<Rocket<Build>, Rocket<Build>> {
+    let conn = HildeDbConn::get_one(&rocket)
+        .await
+        .expect("database connection");
+    conn.run(|c| match embedded_migrations::run(c) {
+        Ok(()) => Ok(rocket),
+        Err(_) => Err(rocket),
+    })
+    .await
+}
+
+#[launch]
+fn rocket() -> _ {
+    rocket::build()
+        .attach(HildeDbConn::fairing())
+        .attach(AdHoc::try_on_ignite(
+            "Run database migrations",
+            run_db_migrations,
+        ))
 }