Compare commits

..

2 Commits

Author SHA1 Message Date
Jef Roosens 406a2b971a
Updated roadmap 2021-06-19 12:45:25 +02:00
Jef Roosens c163028815
Added roadmap 2021-06-17 19:44:17 +02:00
17 changed files with 109 additions and 419 deletions

View File

@ -1,5 +0,0 @@
*
!Cargo.toml
!Cargo.lock
!src/

View File

@ -1,13 +0,0 @@
root = true
[*]
end_of_line = lf
insert_final_newline = false
indent_size = 4
indent_style = space
[*.{yml,yaml}]
indent_size = 2
[Makefile]
indent_style = tab

2
.env
View File

@ -1,2 +0,0 @@
# This file is solely used by Diesel
DATABASE_URL=postgres://hilde:hilde@localhost/hilde

View File

@ -1,77 +0,0 @@
pipeline:
# Download the cache from S3
restore-cache:
image: plugins/s3-cache
pull: true
endpoint: https://s3.roosens.me
root: build-cache/
restore: true
secrets: [ cache_s3_access_key, cache_s3_secret_key ]
# =====BUILDING=====
build-backend:
image: rust:1.53-alpine
pull: true
group: build
environment:
- CARGO_HOME=.cargo
commands:
- apk add musl-dev postgresql-dev
- cargo build
# =====TESTING=====
test-backend:
image: rust:1.53-alpine
environment:
- CARGO_HOME=.cargo
commands:
- apk add musl-dev postgresql-dev
- cargo test
# =====LINTING=====
lint-backend:
image: rust:1.53-alpine
group: lint
environment:
- CARGO_HOME=.cargo
commands:
- apk add musl-dev postgresql-dev
- rustup component add rustfmt clippy
- cargo fmt -- --check
# This is run here because it requires compilation
- cargo clippy --all-targets -- -D warnings
# =====REBUILD & FLUSH CACHE=====
rebuild-cache:
image: plugins/s3-cache
endpoint: https://s3.roosens.me
root: build-cache/
rebuild: true
mount:
- target
- .cargo
secrets: [ cache_s3_access_key, cache_s3_secret_key ]
# Push the cache, even on failure
when:
status: [ success, failure ]
flush-cache:
image: plugins/s3-cache
endpoint: https://s3.roosens.me
root: build-cache/
flush: true
flush_age: 14
secrets: [ cache_s3_access_key, cache_s3_secret_key ]
# Push the cache, even on failure
when:
status: [ success, failure ]

151
Cargo.lock generated
View File

@ -84,12 +84,6 @@ version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.0.1" version = "1.0.1"
@ -176,40 +170,6 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "diesel"
version = "1.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d"
dependencies = [
"bitflags",
"byteorder",
"diesel_derives",
"pq-sys",
"r2d2",
]
[[package]]
name = "diesel_derives"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "diesel_migrations"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c"
dependencies = [
"migrations_internals",
"migrations_macros",
]
[[package]] [[package]]
name = "discard" name = "discard"
version = "1.0.4" version = "1.0.4"
@ -233,9 +193,9 @@ dependencies = [
[[package]] [[package]]
name = "figment" name = "figment"
version = "0.10.6" version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "790b4292c72618abbab50f787a477014fe15634f96291de45672ce46afe122df" checksum = "0ca029e813a72b7526d28273d25f3e4a2f365d1b7a1018a6f93ec9053a119763"
dependencies = [ dependencies = [
"atomic", "atomic",
"pear", "pear",
@ -402,9 +362,9 @@ checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
[[package]] [[package]]
name = "hermit-abi" name = "hermit-abi"
version = "0.1.19" version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
dependencies = [ dependencies = [
"libc", "libc",
] ]
@ -413,10 +373,7 @@ dependencies = [
name = "hilde" name = "hilde"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"diesel",
"diesel_migrations",
"rocket", "rocket",
"rocket_sync_db_pools",
] ]
[[package]] [[package]]
@ -558,27 +515,6 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
[[package]]
name = "migrations_internals"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860"
dependencies = [
"diesel",
]
[[package]]
name = "migrations_macros"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c"
dependencies = [
"migrations_internals",
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "mime" name = "mime"
version = "0.3.16" version = "0.3.16"
@ -709,9 +645,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.7" version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905"
[[package]] [[package]]
name = "pin-utils" name = "pin-utils"
@ -725,15 +661,6 @@ version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "pq-sys"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda"
dependencies = [
"vcpkg",
]
[[package]] [[package]]
name = "proc-macro-hack" name = "proc-macro-hack"
version = "0.5.19" version = "0.5.19"
@ -777,17 +704,6 @@ dependencies = [
"proc-macro2", "proc-macro2",
] ]
[[package]]
name = "r2d2"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f"
dependencies = [
"log",
"parking_lot",
"scheduled-thread-pool",
]
[[package]] [[package]]
name = "rand" name = "rand"
version = "0.8.4" version = "0.8.4"
@ -830,9 +746,9 @@ dependencies = [
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.2.9" version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
dependencies = [ dependencies = [
"bitflags", "bitflags",
] ]
@ -948,30 +864,6 @@ dependencies = [
"uncased", "uncased",
] ]
[[package]]
name = "rocket_sync_db_pools"
version = "0.1.0-rc.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38cfdfebd552d075c368e641c88a5cd6ce1c58c5c710548aeb777abb48830f4b"
dependencies = [
"diesel",
"r2d2",
"rocket",
"rocket_sync_db_pools_codegen",
"serde",
"tokio",
]
[[package]]
name = "rocket_sync_db_pools_codegen"
version = "0.1.0-rc.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5267808c094db5366e1d8925aaf9f2ce05ff9b3bd92cb18c7040a1fe219c2e25"
dependencies = [
"devise",
"quote",
]
[[package]] [[package]]
name = "rustc_version" name = "rustc_version"
version = "0.2.3" version = "0.2.3"
@ -993,15 +885,6 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "scheduled-thread-pool"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7"
dependencies = [
"parking_lot",
]
[[package]] [[package]]
name = "scoped-tls" name = "scoped-tls"
version = "1.0.0" version = "1.0.0"
@ -1099,9 +982,9 @@ dependencies = [
[[package]] [[package]]
name = "spin" name = "spin"
version = "0.9.1" version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5fdd7196b4ae35a111c6dc97f9cc152ca3ea8ad744f7cb46a9f27b3ef8f2f54" checksum = "b87bbf98cb81332a56c1ee8929845836f85e8ddd693157c30d76660196014478"
[[package]] [[package]]
name = "stable-pattern" name = "stable-pattern"
@ -1123,9 +1006,9 @@ dependencies = [
[[package]] [[package]]
name = "state" name = "state"
version = "0.5.2" version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87cf4f5369e6d3044b5e365c9690f451516ac8f0954084622b49ea3fde2f6de5" checksum = "0b54c22963194db84a59ee48e1fa9ed6c1fa9909ad5db92a700aa6fe956d632b"
dependencies = [ dependencies = [
"loom", "loom",
] ]
@ -1244,9 +1127,9 @@ dependencies = [
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.7.1" version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fb2ed024293bb19f7a5dc54fe83bf86532a44c12a2bb8ba40d64a4509395ca2" checksum = "c79ba603c337335df6ba6dd6afc38c38a7d5e1b0c871678439ea973cd62a118e"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"bytes", "bytes",
@ -1379,12 +1262,6 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]] [[package]]
name = "version_check" name = "version_check"
version = "0.9.3" version = "0.9.3"

View File

@ -1,17 +1,10 @@
[package] [package]
name = "hilde" name = "hilde"
version = "0.1.0" version = "0.1.0"
authors = ["Jef Roosens <roosensjef@gmail.com>"] authors = ["jjr <roosensjef@gmail.com>"]
edition = "2018" edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
rocket = "0.5.0-rc.1" rocket = "0.5.0-rc.1"
diesel = { version = "1.4.8", features = ["postgres"] }
diesel_migrations = "1.4.0"
[dependencies.rocket_sync_db_pools]
version = "0.1.0-rc.1"
default_features = false
features = ["diesel_postgres_pool"]

View File

@ -1,22 +0,0 @@
FROM rust:1.57-alpine3.13 AS builder
WORKDIR /src
# Install build dependencies
RUN apk add --no-cache \
musl-dev \
postgresql-dev
# Copy over source code
COPY Cargo.toml Cargo.lock ./
COPY src ./src/
RUN cargo build \
--release
FROM alpine:3.15
COPY --from=builder /src/target/release/hilde /usr/local/bin/hilde
ENTRYPOINT [ "/usr/local/bin/hilde" ]

View File

View File

@ -1,59 +0,0 @@
# The main usecase for this Makefile is to simplify database management
all: build
.PHONY: all
# =====CARGO STUFF=====
build:
@ cargo build
.PHONY: build
run:
@ cargo run
.PHONY: run
test:
@ cargo test
.PHONY: test
lint:
@ cargo fmt -- --check
@ cargo clippy --all-targets -- -D warnings
.PHONY: lint
format:
@ cargo fmt
.PHONY: format
# =====DATABASE STUFF=====
db:
@ docker run \
--rm \
-itd \
-v hilde_db-data:/var/lib/postgresql/data \
-e POSTGRES_USER=hilde \
-e POSTGRES_PASSWORD=hilde \
-e POSTGRES_DB=hilde \
--name hilde_db \
-p 5432:5432 \
postgres:13-alpine
.PHONY: db
psql:
@ docker exec \
-it hilde_db \
psql -U hilde
.PHONY: psql
stop-db:
@ docker stop hilde_db
.PHONY: stop-db
# =====DOCKER STUFF=====
image:
@ docker build -t hilde:latest .
.PHONY: image

View File

@ -1,19 +1,11 @@
# hilde # hilde
A server implementation of a Debian repository. ## What?
## Development A server implementation of a Debian repository, written in Rust.
To develop this project, you'll need a few things: ## Why?
* Docker (for setting up the database) The original reason was that I wanted to create my own "AUR" for Debian,
* Cargo & Rust stable (all hail Rocket v0.5) populating it using my CI pipeline. And of course, the only true valid reason,
* The PostgreSQL dev libraries (to compile Diesel) it sounded fun :)
That's the ones I can think of right now. If I think of any others, I'll be
sure to add them to the list.
For ease of development, there's a Makefile wrapper that can do everything you
need to do to work on Hilde. I recommend reading through it (it's not long),
but the important ones are `make` & `make run`, which are just Cargo aliases,
and `make db`, which starts a local PostgreSQL database using Docker.

86
ROADMAP.md 100644
View File

@ -0,0 +1,86 @@
# Roadmap
This file describes a broad roadmap of what's currently planned with the
project.
## Readable repository
The first goal is to make the server a fully functional Debian package
repository. This includes:
### Setting up a database
Hilde stores an index of the stored deb files in a PostgreSQL database. Any
data that can be calculated beforehand and/or stored efficiently inside a
database (e.g. hashes, package names & versions, descriptions etc.) will be
stored there. Any request that isn't a deb file will be queried and/or
generated from this database. For example, the `Packages` file can be
generated on the fly instead of storing it on disk. The goal is for the data
storage to only include deb files.
### Indexing the directory on startup
On startup, Hilde will scan the packages directory for deb files. These will
then all be parsed, and synced with the database.
### Functioning as a repository
Using this database backend, the server should fully mimic a Debian repository
server ([an example](http://ftp.debian.org/debian/)). As there's no real reason
not to, we should implement the
[repository spec](https://wiki.debian.org/DebianRepository) as complete as
possible.
## The ideas
Now that we have a repository that we can use, it's time to add some extra
functionality.
### Uploading versions
This is the initial reason I wanted to start this project. My goal is to add a
way to send POST requests to the server containing new versions of a new or
existing package. These packages can be from any distribution, architecture, or
name, and the server will just dynamically create the directories n stuff when
needed. This feature can then be used in CI pipelines to simplify uploading
versions to a repository.
### User management & JWT tokens
Considering we can't just make an upload endpoint publicly available, we'll
need a system to log in. There will be an admin user that can do everything,
but I want to add a detailed user management system. This system will have
fine-grained controls over what user can do what, and each user is able to
create JWT tokens that further narrows down the functionality. For example, a
user could have read & write access to the stable distribution's updates
section. Then they can create a token that only allows to write a certain
package, and use it in their CI pipeline for that particular package.
### Federation
I think the idea of being able to spread a service across multiple hosts is
really cool, so I've thought about adding this to Hilde as well. I have never
created anything like this before, but I think the idea is really cool. My
current approach would be as follows:
The nodes communicate using a common secret key. Each node only requires the
location of a single other node. When a new node connects to the network, it
asks the node it's been given about which nodes it knows. These nodes are then
added to the node's list of known nodes. These new nodes are then also
notified, and this process is repeated until all nodes are discovered. Each
node also keeps an index of which packages can be found on which node. This
makes it so that every node can be used as an entrypoint to the network, making
it easy to load balance.
Each node could maybe send a notification to all the other nodes when it
discovers/loses a node, or receives a new package.
When uploading a new package, we could query the system to see which node has
the most space left, as to more efficiently distribute the packages, if
possible.
### Mirroring repositories
It might be useful to add functionality to mirror an existing repo. This could
for example make a home setup fully self-sufficient, by mirroring the official
Debian repositories.

View File

@ -1,2 +0,0 @@
[debug.databases]
postgres_hilde = { url = "postgres://hilde:hilde@localhost/hilde" }

View File

@ -1,5 +0,0 @@
# For documentation on how to configure this file,
# see diesel.rs/guides/configuring-diesel-cli
[print_schema]
file = "src/schema.rs"

View File

View File

@ -1,6 +0,0 @@
-- This file was automatically created by Diesel to setup helper functions
-- and other internal bookkeeping. This file is safe to edit, any future
-- changes will be added to existing projects as new migrations.
DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
DROP FUNCTION IF EXISTS diesel_set_updated_at();

View File

@ -1,36 +0,0 @@
-- This file was automatically created by Diesel to setup helper functions
-- and other internal bookkeeping. This file is safe to edit, any future
-- changes will be added to existing projects as new migrations.
-- Sets up a trigger for the given table to automatically set a column called
-- `updated_at` whenever the row is modified (unless `updated_at` was included
-- in the modified columns)
--
-- # Example
--
-- ```sql
-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
--
-- SELECT diesel_manage_updated_at('users');
-- ```
CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
BEGIN
EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
BEGIN
IF (
NEW IS DISTINCT FROM OLD AND
NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
) THEN
NEW.updated_at := current_timestamp;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;

View File

@ -1,34 +1,3 @@
#[macro_use] fn main() {
extern crate rocket; println!("Hello, world!");
#[macro_use]
extern crate diesel_migrations;
use rocket::{fairing::AdHoc, Build, Rocket};
use rocket_sync_db_pools::{database, diesel};
embed_migrations!();
#[database("postgres_hilde")]
struct HildeDbConn(diesel::PgConnection);
async fn run_db_migrations(rocket: Rocket<Build>) -> Result<Rocket<Build>, Rocket<Build>> {
let conn = HildeDbConn::get_one(&rocket)
.await
.expect("database connection");
conn.run(|c| match embedded_migrations::run(c) {
Ok(()) => Ok(rocket),
Err(_) => Err(rocket),
})
.await
}
#[launch]
fn rocket() -> _ {
rocket::build()
.attach(HildeDbConn::fairing())
.attach(AdHoc::try_on_ignite(
"Run database migrations",
run_db_migrations,
))
} }