Updated roadmap

.dockerignore

@@ -1,5 +0,0 @@
-*
-
-!Cargo.toml
-!Cargo.lock
-!src/

.editorconfig

@@ -1,13 +0,0 @@
-root = true
-
-[*]
-end_of_line = lf
-insert_final_newline = false
-indent_size = 4
-indent_style = space
-
-[*.{yml,yaml}]
-indent_size = 2
-
-[Makefile]
-indent_style = tab

.env

@@ -1,2 +0,0 @@
-# This file is solely used by Diesel
-DATABASE_URL=postgres://hilde:hilde@localhost/hilde

.woodpecker.yml

@@ -1,77 +0,0 @@
-pipeline:
-  # Download the cache from S3
-  restore-cache:
-    image: plugins/s3-cache
-    pull: true
-
-    endpoint: https://s3.roosens.me
-    root: build-cache/
-    restore: true
-
-    secrets: [ cache_s3_access_key, cache_s3_secret_key ]
-
-
-  # =====BUILDING=====
-  build-backend:
-    image: rust:1.53-alpine
-    pull: true
-    group: build
-    environment:
-      - CARGO_HOME=.cargo
-    commands:
-      - apk add musl-dev postgresql-dev
-      - cargo build
-
-
-  # =====TESTING=====
-  test-backend:
-    image: rust:1.53-alpine
-    environment:
-      - CARGO_HOME=.cargo
-    commands:
-      - apk add musl-dev postgresql-dev
-      - cargo test
-
-
-  # =====LINTING=====
-  lint-backend:
-    image: rust:1.53-alpine
-    group: lint
-    environment:
-      - CARGO_HOME=.cargo
-    commands:
-      - apk add musl-dev postgresql-dev
-      - rustup component add rustfmt clippy
-      - cargo fmt -- --check
-      # This is run here because it requires compilation
-      - cargo clippy --all-targets -- -D warnings
-  
-
-  # =====REBUILD & FLUSH CACHE=====
-  rebuild-cache:
-    image: plugins/s3-cache
-
-    endpoint: https://s3.roosens.me
-    root: build-cache/
-    rebuild: true
-    mount:
-      - target
-      - .cargo
-
-    secrets: [ cache_s3_access_key, cache_s3_secret_key ]
-    # Push the cache, even on failure
-    when:
-      status: [ success, failure ]
-
-  flush-cache:
-    image: plugins/s3-cache
-
-    endpoint: https://s3.roosens.me
-    root: build-cache/
-    flush: true
-    flush_age: 14
-
-    secrets: [ cache_s3_access_key, cache_s3_secret_key ]
-    # Push the cache, even on failure
-    when:
-      status: [ success, failure ]

Cargo.lock

@@ -84,12 +84,6 @@ version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
 
-[[package]]
-name = "byteorder"
-version = "1.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
-
 [[package]]
 name = "bytes"
 version = "1.0.1"
@@ -176,40 +170,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "diesel"
-version = "1.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d"
-dependencies = [
- "bitflags",
- "byteorder",
- "diesel_derives",
- "pq-sys",
- "r2d2",
-]
-
-[[package]]
-name = "diesel_derives"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "diesel_migrations"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c"
-dependencies = [
- "migrations_internals",
- "migrations_macros",
-]
-
 [[package]]
 name = "discard"
 version = "1.0.4"
@@ -233,9 +193,9 @@ dependencies = [
 
 [[package]]
 name = "figment"
-version = "0.10.6"
+version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "790b4292c72618abbab50f787a477014fe15634f96291de45672ce46afe122df"
+checksum = "0ca029e813a72b7526d28273d25f3e4a2f365d1b7a1018a6f93ec9053a119763"
 dependencies = [
  "atomic",
  "pear",
@@ -402,9 +362,9 @@ checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.19"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
 dependencies = [
  "libc",
 ]
@@ -413,10 +373,7 @@ dependencies = [
 name = "hilde"
 version = "0.1.0"
 dependencies = [
- "diesel",
- "diesel_migrations",
  "rocket",
- "rocket_sync_db_pools",
 ]
 
 [[package]]
@@ -558,27 +515,6 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
 
-[[package]]
-name = "migrations_internals"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860"
-dependencies = [
- "diesel",
-]
-
-[[package]]
-name = "migrations_macros"
-version = "1.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c"
-dependencies = [
- "migrations_internals",
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -709,9 +645,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.7"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443"
+checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905"
 
 [[package]]
 name = "pin-utils"
@@ -725,15 +661,6 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
 
-[[package]]
-name = "pq-sys"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda"
-dependencies = [
- "vcpkg",
-]
-
 [[package]]
 name = "proc-macro-hack"
 version = "0.5.19"
@@ -777,17 +704,6 @@ dependencies = [
  "proc-macro2",
 ]
 
-[[package]]
-name = "r2d2"
-version = "0.8.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f"
-dependencies = [
- "log",
- "parking_lot",
- "scheduled-thread-pool",
-]
-
 [[package]]
 name = "rand"
 version = "0.8.4"
@@ -830,9 +746,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.9"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
+checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
 dependencies = [
  "bitflags",
 ]
@@ -948,30 +864,6 @@ dependencies = [
  "uncased",
 ]
 
-[[package]]
-name = "rocket_sync_db_pools"
-version = "0.1.0-rc.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38cfdfebd552d075c368e641c88a5cd6ce1c58c5c710548aeb777abb48830f4b"
-dependencies = [
- "diesel",
- "r2d2",
- "rocket",
- "rocket_sync_db_pools_codegen",
- "serde",
- "tokio",
-]
-
-[[package]]
-name = "rocket_sync_db_pools_codegen"
-version = "0.1.0-rc.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267808c094db5366e1d8925aaf9f2ce05ff9b3bd92cb18c7040a1fe219c2e25"
-dependencies = [
- "devise",
- "quote",
-]
-
 [[package]]
 name = "rustc_version"
 version = "0.2.3"
@@ -993,15 +885,6 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
-[[package]]
-name = "scheduled-thread-pool"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7"
-dependencies = [
- "parking_lot",
-]
-
 [[package]]
 name = "scoped-tls"
 version = "1.0.0"
@@ -1099,9 +982,9 @@ dependencies = [
 
 [[package]]
 name = "spin"
-version = "0.9.1"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5fdd7196b4ae35a111c6dc97f9cc152ca3ea8ad744f7cb46a9f27b3ef8f2f54"
+checksum = "b87bbf98cb81332a56c1ee8929845836f85e8ddd693157c30d76660196014478"
 
 [[package]]
 name = "stable-pattern"
@@ -1123,9 +1006,9 @@ dependencies = [
 
 [[package]]
 name = "state"
-version = "0.5.2"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cf4f5369e6d3044b5e365c9690f451516ac8f0954084622b49ea3fde2f6de5"
+checksum = "0b54c22963194db84a59ee48e1fa9ed6c1fa9909ad5db92a700aa6fe956d632b"
 dependencies = [
  "loom",
 ]
@@ -1244,9 +1127,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.7.1"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fb2ed024293bb19f7a5dc54fe83bf86532a44c12a2bb8ba40d64a4509395ca2"
+checksum = "c79ba603c337335df6ba6dd6afc38c38a7d5e1b0c871678439ea973cd62a118e"
 dependencies = [
  "autocfg",
  "bytes",
@@ -1379,12 +1262,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
 
-[[package]]
-name = "vcpkg"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
-
 [[package]]
 name = "version_check"
 version = "0.9.3"

Cargo.toml

@@ -1,17 +1,10 @@
 [package]
 name = "hilde"
 version = "0.1.0"
-authors = ["Jef Roosens <roosensjef@gmail.com>"]
+authors = ["jjr <roosensjef@gmail.com>"]
 edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 rocket = "0.5.0-rc.1"
-diesel = { version = "1.4.8", features = ["postgres"] }
-diesel_migrations = "1.4.0"
-
-[dependencies.rocket_sync_db_pools]
-version = "0.1.0-rc.1"
-default_features = false
-features = ["diesel_postgres_pool"]

Dockerfile

@@ -1,22 +0,0 @@
-FROM rust:1.57-alpine3.13 AS builder
-
-WORKDIR /src
-
-# Install build dependencies
-RUN apk add --no-cache \
-    musl-dev \
-    postgresql-dev
-
-# Copy over source code
-COPY Cargo.toml Cargo.lock ./
-COPY src ./src/
-
-RUN cargo build \
-    --release
-
-
-FROM alpine:3.15
-
-COPY --from=builder /src/target/release/hilde /usr/local/bin/hilde
-
-ENTRYPOINT [ "/usr/local/bin/hilde" ]

Makefile

@@ -1,59 +0,0 @@
-# The main usecase for this Makefile is to simplify database management
-
-
-all: build
-.PHONY: all
-
-
-# =====CARGO STUFF=====
-build:
-	@ cargo build
-.PHONY: build
-
-run:
-	@ cargo run
-.PHONY: run
-
-test:
-	@ cargo test
-.PHONY: test
-
-lint:
-	@ cargo fmt -- --check
-	@ cargo clippy --all-targets -- -D warnings
-.PHONY: lint
-
-format:
-	@ cargo fmt
-.PHONY: format
-
-
-# =====DATABASE STUFF=====
-db:
-	@ docker run \
-		--rm \
-		-itd \
-		-v hilde_db-data:/var/lib/postgresql/data \
-		-e POSTGRES_USER=hilde \
-		-e POSTGRES_PASSWORD=hilde \
-		-e POSTGRES_DB=hilde \
-		--name hilde_db \
-		-p 5432:5432 \
-		postgres:13-alpine
-.PHONY: db
-
-psql:
-	@ docker exec \
-		-it hilde_db \
-		psql -U hilde
-.PHONY: psql
-
-stop-db:
-	@ docker stop hilde_db
-.PHONY: stop-db
-
-
-# =====DOCKER STUFF=====
-image:
-	@ docker build -t hilde:latest .
-.PHONY: image

README.md

@@ -1,19 +1,11 @@
 # hilde
 
-A server implementation of a Debian repository.
+## What?
 
-## Development
+A server implementation of a Debian repository, written in Rust.
 
-To develop this project, you'll need a few things:
+## Why?
 
-* Docker (for setting up the database)
+The original reason was that I wanted to create my own "AUR" for Debian,
-* Cargo & Rust stable (all hail Rocket v0.5)
+populating it using my CI pipeline. And of course, the only true valid reason,
-* The PostgreSQL dev libraries (to compile Diesel)
+it sounded fun :)
-
-That's the ones I can think of right now. If I think of any others, I'll be
-sure to add them to the list.
-
-For ease of development, there's a Makefile wrapper that can do everything you
-need to do to work on Hilde. I recommend reading through it (it's not long),
-but the important ones are `make` & `make run`, which are just Cargo aliases,
-and `make db`, which starts a local PostgreSQL database using Docker.

ROADMAP.md

@@ -0,0 +1,86 @@
+# Roadmap
+
+This file describes a broad roadmap of what's currently planned with the
+project.
+
+## Readable repository
+
+The first goal is to make the server a fully functional Debian package
+repository. This includes:
+
+### Setting up a database
+
+Hilde stores an index of the stored deb files in a PostgreSQL database. Any
+data that can be calculated beforehand and/or stored efficiently inside a
+database (e.g. hashes, package names & versions, descriptions etc.) will be
+stored there. Any request that isn't a deb file will be queried and/or
+generated from this database. For example, the `Packages` file can be
+generated on the fly instead of storing it on disk. The goal is for the data
+storage to only include deb files.
+
+### Indexing the directory on startup
+
+On startup, Hilde will scan the packages directory for deb files. These will
+then all be parsed, and synced with the database.
+
+### Functioning as a repository
+
+Using this database backend, the server should fully mimic a Debian repository
+server ([an example](http://ftp.debian.org/debian/)). As there's no real reason
+not to, we should implement the
+[repository spec](https://wiki.debian.org/DebianRepository) as complete as
+possible.
+
+## The ideas
+
+Now that we have a repository that we can use, it's time to add some extra
+functionality.
+
+### Uploading versions
+
+This is the initial reason I wanted to start this project. My goal is to add a
+way to send POST requests to the server containing new versions of a new or
+existing package. These packages can be from any distribution, architecture, or
+name, and the server will just dynamically create the directories n stuff when
+needed. This feature can then be used in CI pipelines to simplify uploading
+versions to a repository.
+
+### User management & JWT tokens
+
+Considering we can't just make an upload endpoint publicly available, we'll
+need a system to log in. There will be an admin user that can do everything,
+but I want to add a detailed user management system. This system will have
+fine-grained controls over what user can do what, and each user is able to
+create JWT tokens that further narrows down the functionality. For example, a
+user could have read & write access to the stable distribution's updates
+section. Then they can create a token that only allows to write a certain
+package, and use it in their CI pipeline for that particular package.
+
+### Federation
+
+I think the idea of being able to spread a service across multiple hosts is
+really cool, so I've thought about adding this to Hilde as well. I have never
+created anything like this before, but I think the idea is really cool. My
+current approach would be as follows:
+
+The nodes communicate using a common secret key. Each node only requires the
+location of a single other node. When a new node connects to the network, it
+asks the node it's been given about which nodes it knows. These nodes are then
+added to the node's list of known nodes. These new nodes are then also
+notified, and this process is repeated until all nodes are discovered. Each
+node also keeps an index of which packages can be found on which node. This
+makes it so that every node can be used as an entrypoint to the network, making
+it easy to load balance.
+
+Each node could maybe send a notification to all the other nodes when it
+discovers/loses a node, or receives a new package.
+
+When uploading a new package, we could query the system to see which node has
+the most space left, as to more efficiently distribute the packages, if
+possible.
+
+### Mirroring repositories
+
+It might be useful to add functionality to mirror an existing repo. This could
+for example make a home setup fully self-sufficient, by mirroring the official
+Debian repositories.

Rocket.toml

@@ -1,2 +0,0 @@
-[debug.databases]
-postgres_hilde = { url = "postgres://hilde:hilde@localhost/hilde" }

diesel.toml

@@ -1,5 +0,0 @@
-# For documentation on how to configure this file,
-# see diesel.rs/guides/configuring-diesel-cli
-
-[print_schema]
-file = "src/schema.rs"

migrations/00000000000000_diesel_initial_setup/down.sql

@@ -1,6 +0,0 @@
--- This file was automatically created by Diesel to setup helper functions
--- and other internal bookkeeping. This file is safe to edit, any future
--- changes will be added to existing projects as new migrations.
-
-DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
-DROP FUNCTION IF EXISTS diesel_set_updated_at();

migrations/00000000000000_diesel_initial_setup/up.sql

@@ -1,36 +0,0 @@
--- This file was automatically created by Diesel to setup helper functions
--- and other internal bookkeeping. This file is safe to edit, any future
--- changes will be added to existing projects as new migrations.
-
-
-
-
--- Sets up a trigger for the given table to automatically set a column called
--- `updated_at` whenever the row is modified (unless `updated_at` was included
--- in the modified columns)
---
--- # Example
---
--- ```sql
--- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
---
--- SELECT diesel_manage_updated_at('users');
--- ```
-CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
-BEGIN
-    EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
-                    FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
-BEGIN
-    IF (
-        NEW IS DISTINCT FROM OLD AND
-        NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
-    ) THEN
-        NEW.updated_at := current_timestamp;
-    END IF;
-    RETURN NEW;
-END;
-$$ LANGUAGE plpgsql;

src/main.rs

@@ -1,34 +1,3 @@
-#[macro_use]
+fn main() {
-extern crate rocket;
+    println!("Hello, world!");
-
-#[macro_use]
-extern crate diesel_migrations;
-
-use rocket::{fairing::AdHoc, Build, Rocket};
-use rocket_sync_db_pools::{database, diesel};
-
-embed_migrations!();
-
-#[database("postgres_hilde")]
-struct HildeDbConn(diesel::PgConnection);
-
-async fn run_db_migrations(rocket: Rocket<Build>) -> Result<Rocket<Build>, Rocket<Build>> {
-    let conn = HildeDbConn::get_one(&rocket)
-        .await
-        .expect("database connection");
-    conn.run(|c| match embedded_migrations::run(c) {
-        Ok(()) => Ok(rocket),
-        Err(_) => Err(rocket),
-    })
-    .await
-}
-
-#[launch]
-fn rocket() -> _ {
-    rocket::build()
-        .attach(HildeDbConn::fairing())
-        .attach(AdHoc::try_on_ignite(
-            "Run database migrations",
-            run_db_migrations,
-        ))
 }