diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 1917669..0000000 --- a/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -* - -!Cargo.toml -!Cargo.lock -!src/ diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 77e3fe3..0000000 --- a/.editorconfig +++ /dev/null @@ -1,13 +0,0 @@ -root = true - -[*] -end_of_line = lf -insert_final_newline = false -indent_size = 4 -indent_style = space - -[*.{yml,yaml}] -indent_size = 2 - -[Makefile] -indent_style = tab diff --git a/.env b/.env deleted file mode 100644 index d966102..0000000 --- a/.env +++ /dev/null @@ -1,2 +0,0 @@ -# This file is solely used by Diesel -DATABASE_URL=postgres://hilde:hilde@localhost/hilde diff --git a/.woodpecker.yml b/.woodpecker.yml deleted file mode 100644 index cb88ba5..0000000 --- a/.woodpecker.yml +++ /dev/null @@ -1,77 +0,0 @@ -pipeline: - # Download the cache from S3 - restore-cache: - image: plugins/s3-cache - pull: true - - endpoint: https://s3.roosens.me - root: build-cache/ - restore: true - - secrets: [ cache_s3_access_key, cache_s3_secret_key ] - - - # =====BUILDING===== - build-backend: - image: rust:1.53-alpine - pull: true - group: build - environment: - - CARGO_HOME=.cargo - commands: - - apk add musl-dev postgresql-dev - - cargo build - - - # =====TESTING===== - test-backend: - image: rust:1.53-alpine - environment: - - CARGO_HOME=.cargo - commands: - - apk add musl-dev postgresql-dev - - cargo test - - - # =====LINTING===== - lint-backend: - image: rust:1.53-alpine - group: lint - environment: - - CARGO_HOME=.cargo - commands: - - apk add musl-dev postgresql-dev - - rustup component add rustfmt clippy - - cargo fmt -- --check - # This is run here because it requires compilation - - cargo clippy --all-targets -- -D warnings - - - # =====REBUILD & FLUSH CACHE===== - rebuild-cache: - image: plugins/s3-cache - - endpoint: https://s3.roosens.me - root: build-cache/ - rebuild: true - mount: - - target - - .cargo - - secrets: [ cache_s3_access_key, cache_s3_secret_key ] - # Push the cache, even on failure - when: - status: [ success, failure ] - - flush-cache: - image: plugins/s3-cache - - endpoint: https://s3.roosens.me - root: build-cache/ - flush: true - flush_age: 14 - - secrets: [ cache_s3_access_key, cache_s3_secret_key ] - # Push the cache, even on failure - when: - status: [ success, failure ] diff --git a/Cargo.lock b/Cargo.lock index 1cf7bc9..905d09f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,12 +84,6 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - [[package]] name = "bytes" version = "1.0.1" @@ -176,40 +170,6 @@ dependencies = [ "syn", ] -[[package]] -name = "diesel" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d" -dependencies = [ - "bitflags", - "byteorder", - "diesel_derives", - "pq-sys", - "r2d2", -] - -[[package]] -name = "diesel_derives" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "diesel_migrations" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" -dependencies = [ - "migrations_internals", - "migrations_macros", -] - [[package]] name = "discard" version = "1.0.4" @@ -233,9 +193,9 @@ dependencies = [ [[package]] name = "figment" -version = "0.10.6" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790b4292c72618abbab50f787a477014fe15634f96291de45672ce46afe122df" +checksum = "0ca029e813a72b7526d28273d25f3e4a2f365d1b7a1018a6f93ec9053a119763" dependencies = [ "atomic", "pear", @@ -402,9 +362,9 @@ checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] @@ -413,10 +373,7 @@ dependencies = [ name = "hilde" version = "0.1.0" dependencies = [ - "diesel", - "diesel_migrations", "rocket", - "rocket_sync_db_pools", ] [[package]] @@ -558,27 +515,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" -[[package]] -name = "migrations_internals" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" -dependencies = [ - "diesel", -] - -[[package]] -name = "migrations_macros" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" -dependencies = [ - "migrations_internals", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "mime" version = "0.3.16" @@ -709,9 +645,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project-lite" -version = "0.2.7" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" [[package]] name = "pin-utils" @@ -725,15 +661,6 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" -[[package]] -name = "pq-sys" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda" -dependencies = [ - "vcpkg", -] - [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -777,17 +704,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r2d2" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" -dependencies = [ - "log", - "parking_lot", - "scheduled-thread-pool", -] - [[package]] name = "rand" version = "0.8.4" @@ -830,9 +746,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.9" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ "bitflags", ] @@ -948,30 +864,6 @@ dependencies = [ "uncased", ] -[[package]] -name = "rocket_sync_db_pools" -version = "0.1.0-rc.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cfdfebd552d075c368e641c88a5cd6ce1c58c5c710548aeb777abb48830f4b" -dependencies = [ - "diesel", - "r2d2", - "rocket", - "rocket_sync_db_pools_codegen", - "serde", - "tokio", -] - -[[package]] -name = "rocket_sync_db_pools_codegen" -version = "0.1.0-rc.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267808c094db5366e1d8925aaf9f2ce05ff9b3bd92cb18c7040a1fe219c2e25" -dependencies = [ - "devise", - "quote", -] - [[package]] name = "rustc_version" version = "0.2.3" @@ -993,15 +885,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" -[[package]] -name = "scheduled-thread-pool" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" -dependencies = [ - "parking_lot", -] - [[package]] name = "scoped-tls" version = "1.0.0" @@ -1099,9 +982,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fdd7196b4ae35a111c6dc97f9cc152ca3ea8ad744f7cb46a9f27b3ef8f2f54" +checksum = "b87bbf98cb81332a56c1ee8929845836f85e8ddd693157c30d76660196014478" [[package]] name = "stable-pattern" @@ -1123,9 +1006,9 @@ dependencies = [ [[package]] name = "state" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cf4f5369e6d3044b5e365c9690f451516ac8f0954084622b49ea3fde2f6de5" +checksum = "0b54c22963194db84a59ee48e1fa9ed6c1fa9909ad5db92a700aa6fe956d632b" dependencies = [ "loom", ] @@ -1244,9 +1127,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.7.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fb2ed024293bb19f7a5dc54fe83bf86532a44c12a2bb8ba40d64a4509395ca2" +checksum = "c79ba603c337335df6ba6dd6afc38c38a7d5e1b0c871678439ea973cd62a118e" dependencies = [ "autocfg", "bytes", @@ -1379,12 +1262,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.3" diff --git a/Cargo.toml b/Cargo.toml index 6375ee6..2e688d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,17 +1,10 @@ [package] name = "hilde" version = "0.1.0" -authors = ["Jef Roosens "] +authors = ["jjr "] edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] rocket = "0.5.0-rc.1" -diesel = { version = "1.4.8", features = ["postgres"] } -diesel_migrations = "1.4.0" - -[dependencies.rocket_sync_db_pools] -version = "0.1.0-rc.1" -default_features = false -features = ["diesel_postgres_pool"] diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 088caa0..0000000 --- a/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM rust:1.57-alpine3.13 AS builder - -WORKDIR /src - -# Install build dependencies -RUN apk add --no-cache \ - musl-dev \ - postgresql-dev - -# Copy over source code -COPY Cargo.toml Cargo.lock ./ -COPY src ./src/ - -RUN cargo build \ - --release - - -FROM alpine:3.15 - -COPY --from=builder /src/target/release/hilde /usr/local/bin/hilde - -ENTRYPOINT [ "/usr/local/bin/hilde" ] diff --git a/LICENSE b/LICENSE.md similarity index 100% rename from LICENSE rename to LICENSE.md diff --git a/Makefile b/Makefile deleted file mode 100644 index ffd9238..0000000 --- a/Makefile +++ /dev/null @@ -1,59 +0,0 @@ -# The main usecase for this Makefile is to simplify database management - - -all: build -.PHONY: all - - -# =====CARGO STUFF===== -build: - @ cargo build -.PHONY: build - -run: - @ cargo run -.PHONY: run - -test: - @ cargo test -.PHONY: test - -lint: - @ cargo fmt -- --check - @ cargo clippy --all-targets -- -D warnings -.PHONY: lint - -format: - @ cargo fmt -.PHONY: format - - -# =====DATABASE STUFF===== -db: - @ docker run \ - --rm \ - -itd \ - -v hilde_db-data:/var/lib/postgresql/data \ - -e POSTGRES_USER=hilde \ - -e POSTGRES_PASSWORD=hilde \ - -e POSTGRES_DB=hilde \ - --name hilde_db \ - -p 5432:5432 \ - postgres:13-alpine -.PHONY: db - -psql: - @ docker exec \ - -it hilde_db \ - psql -U hilde -.PHONY: psql - -stop-db: - @ docker stop hilde_db -.PHONY: stop-db - - -# =====DOCKER STUFF===== -image: - @ docker build -t hilde:latest . -.PHONY: image diff --git a/README.md b/README.md index 7491eaa..1bfa8e9 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,11 @@ # hilde -A server implementation of a Debian repository. +## What? -## Development +A server implementation of a Debian repository, written in Rust. -To develop this project, you'll need a few things: +## Why? -* Docker (for setting up the database) -* Cargo & Rust stable (all hail Rocket v0.5) -* The PostgreSQL dev libraries (to compile Diesel) - -That's the ones I can think of right now. If I think of any others, I'll be -sure to add them to the list. - -For ease of development, there's a Makefile wrapper that can do everything you -need to do to work on Hilde. I recommend reading through it (it's not long), -but the important ones are `make` & `make run`, which are just Cargo aliases, -and `make db`, which starts a local PostgreSQL database using Docker. \ No newline at end of file +The original reason was that I wanted to create my own "AUR" for Debian, +populating it using my CI pipeline. And of course, the only true valid reason, +it sounded fun :) diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..1a7cc9a --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,86 @@ +# Roadmap + +This file describes a broad roadmap of what's currently planned with the +project. + +## Readable repository + +The first goal is to make the server a fully functional Debian package +repository. This includes: + +### Setting up a database + +Hilde stores an index of the stored deb files in a PostgreSQL database. Any +data that can be calculated beforehand and/or stored efficiently inside a +database (e.g. hashes, package names & versions, descriptions etc.) will be +stored there. Any request that isn't a deb file will be queried and/or +generated from this database. For example, the `Packages` file can be +generated on the fly instead of storing it on disk. The goal is for the data +storage to only include deb files. + +### Indexing the directory on startup + +On startup, Hilde will scan the packages directory for deb files. These will +then all be parsed, and synced with the database. + +### Functioning as a repository + +Using this database backend, the server should fully mimic a Debian repository +server ([an example](http://ftp.debian.org/debian/)). As there's no real reason +not to, we should implement the +[repository spec](https://wiki.debian.org/DebianRepository) as complete as +possible. + +## The ideas + +Now that we have a repository that we can use, it's time to add some extra +functionality. + +### Uploading versions + +This is the initial reason I wanted to start this project. My goal is to add a +way to send POST requests to the server containing new versions of a new or +existing package. These packages can be from any distribution, architecture, or +name, and the server will just dynamically create the directories n stuff when +needed. This feature can then be used in CI pipelines to simplify uploading +versions to a repository. + +### User management & JWT tokens + +Considering we can't just make an upload endpoint publicly available, we'll +need a system to log in. There will be an admin user that can do everything, +but I want to add a detailed user management system. This system will have +fine-grained controls over what user can do what, and each user is able to +create JWT tokens that further narrows down the functionality. For example, a +user could have read & write access to the stable distribution's updates +section. Then they can create a token that only allows to write a certain +package, and use it in their CI pipeline for that particular package. + +### Federation + +I think the idea of being able to spread a service across multiple hosts is +really cool, so I've thought about adding this to Hilde as well. I have never +created anything like this before, but I think the idea is really cool. My +current approach would be as follows: + +The nodes communicate using a common secret key. Each node only requires the +location of a single other node. When a new node connects to the network, it +asks the node it's been given about which nodes it knows. These nodes are then +added to the node's list of known nodes. These new nodes are then also +notified, and this process is repeated until all nodes are discovered. Each +node also keeps an index of which packages can be found on which node. This +makes it so that every node can be used as an entrypoint to the network, making +it easy to load balance. + +Each node could maybe send a notification to all the other nodes when it +discovers/loses a node, or receives a new package. + +When uploading a new package, we could query the system to see which node has +the most space left, as to more efficiently distribute the packages, if +possible. + +### Mirroring repositories + +It might be useful to add functionality to mirror an existing repo. This could +for example make a home setup fully self-sufficient, by mirroring the official +Debian repositories. diff --git a/Rocket.toml b/Rocket.toml deleted file mode 100644 index 9390626..0000000 --- a/Rocket.toml +++ /dev/null @@ -1,2 +0,0 @@ -[debug.databases] -postgres_hilde = { url = "postgres://hilde:hilde@localhost/hilde" } diff --git a/diesel.toml b/diesel.toml deleted file mode 100644 index 92267c8..0000000 --- a/diesel.toml +++ /dev/null @@ -1,5 +0,0 @@ -# For documentation on how to configure this file, -# see diesel.rs/guides/configuring-diesel-cli - -[print_schema] -file = "src/schema.rs" diff --git a/migrations/.gitkeep b/migrations/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/migrations/00000000000000_diesel_initial_setup/down.sql b/migrations/00000000000000_diesel_initial_setup/down.sql deleted file mode 100644 index a9f5260..0000000 --- a/migrations/00000000000000_diesel_initial_setup/down.sql +++ /dev/null @@ -1,6 +0,0 @@ --- This file was automatically created by Diesel to setup helper functions --- and other internal bookkeeping. This file is safe to edit, any future --- changes will be added to existing projects as new migrations. - -DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); -DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/migrations/00000000000000_diesel_initial_setup/up.sql b/migrations/00000000000000_diesel_initial_setup/up.sql deleted file mode 100644 index d68895b..0000000 --- a/migrations/00000000000000_diesel_initial_setup/up.sql +++ /dev/null @@ -1,36 +0,0 @@ --- This file was automatically created by Diesel to setup helper functions --- and other internal bookkeeping. This file is safe to edit, any future --- changes will be added to existing projects as new migrations. - - - - --- Sets up a trigger for the given table to automatically set a column called --- `updated_at` whenever the row is modified (unless `updated_at` was included --- in the modified columns) --- --- # Example --- --- ```sql --- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); --- --- SELECT diesel_manage_updated_at('users'); --- ``` -CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ -BEGIN - EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s - FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ -BEGIN - IF ( - NEW IS DISTINCT FROM OLD AND - NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at - ) THEN - NEW.updated_at := current_timestamp; - END IF; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; diff --git a/src/main.rs b/src/main.rs index 5db0513..e7a11a9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,34 +1,3 @@ -#[macro_use] -extern crate rocket; - -#[macro_use] -extern crate diesel_migrations; - -use rocket::{fairing::AdHoc, Build, Rocket}; -use rocket_sync_db_pools::{database, diesel}; - -embed_migrations!(); - -#[database("postgres_hilde")] -struct HildeDbConn(diesel::PgConnection); - -async fn run_db_migrations(rocket: Rocket) -> Result, Rocket> { - let conn = HildeDbConn::get_one(&rocket) - .await - .expect("database connection"); - conn.run(|c| match embedded_migrations::run(c) { - Ok(()) => Ok(rocket), - Err(_) => Err(rocket), - }) - .await -} - -#[launch] -fn rocket() -> _ { - rocket::build() - .attach(HildeDbConn::fairing()) - .attach(AdHoc::try_on_ignite( - "Run database migrations", - run_db_migrations, - )) +fn main() { + println!("Hello, world!"); }