Experimental static Docker image [CI SKIP]

.dockerignore

@@ -3,3 +3,4 @@
 !Cargo.toml
 !Cargo.lock
 !src/
+!migrations/

Cargo.lock

@@ -252,6 +252,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "futures"
 version = "0.3.15"
@@ -416,6 +431,7 @@ version = "0.1.0"
 dependencies = [
  "diesel",
  "diesel_migrations",
+ "openssl",
  "rocket",
  "rocket_sync_db_pools",
  "uuid",
@@ -655,6 +671,33 @@ version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
 
+[[package]]
+name = "openssl"
+version = "0.10.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d"
+dependencies = [
+ "autocfg",
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.11.1"
@@ -721,6 +764,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.10"
@@ -895,6 +944,7 @@ dependencies = [
  "rocket_codegen",
  "rocket_http",
  "serde",
+ "serde_json",
  "state",
  "tempfile",
  "time",

Cargo.toml

@@ -21,9 +21,13 @@ doc = true
 doctest = true
 
 [dependencies]
-rocket = "0.5.0-rc.1"
+diesel = { version = "1.4.*", features = ["postgres", "uuidv07"] }
-diesel = { version = "1.4.7", features = ["postgres", "uuidv07"] }
+diesel_migrations = "1.4.*"
-diesel_migrations = "1.4.0"
+openssl = "*"
+
+[dependencies.rocket]
+version = "0.5.0-rc.1"
+features = ["json"]
 
 [dependencies.rocket_sync_db_pools]
 version = "0.1.0-rc.1"

Dockerfile

@@ -1,22 +1,120 @@
-FROM rust:1.53-alpine3.13 AS builder
+ARG DI_VER="1.2.5"
-
-WORKDIR /src
-
-# Install build dependencies
-RUN apk add --no-cache \
-    musl-dev \
-    postgresql-dev
-
-# Copy over source code
-COPY Cargo.toml Cargo.lock ./
-COPY src ./src/
-
-RUN cargo build \
-    --release
 
 
-FROM alpine:3.13
+FROM rust:1.53 AS builder
 
-COPY --from=builder /src/target/release/hilde /usr/local/bin/hilde
+# Here, we define the versions of the libraries we want to build
+ARG DI_VER
+ARG PQ_VER="11.12"
+ARG ZLIB_VER="1.2.11"
+ARG SSL_VER="1.0.2u"
 
-ENTRYPOINT [ "/usr/local/bin/hilde" ]
+WORKDIR /usr/src
+
+ENV PREFIX="/usr/src/musl" \
+    CC="musl-gcc -fPIC -pie -static" \
+    LD_LIBRARY_PATH="$PREFIX" \
+    PKG_CONFIG_PATH="/usr/local/lib/pkgconfig" \
+    PATH="/usr/local/bin:/root/.cargo/bin:$PATH"
+
+# Minimal build requirements for hilde's C dependencies
+RUN apt update && \
+    apt install -y --no-install-recommends \
+        musl-dev \
+        musl-tools \
+        libpq-dev \
+        libssl-dev && \
+    rustup target add x86_64-unknown-linux-musl && \
+    mkdir "$PREFIX" && \
+    echo "$PREFIX/lib" >> /etc/ld-musl-x86_64.path
+
+
+# =====BUILDING LIBRARIES=====
+# Because I think it's cool, I wanted to make a fully static binary for hilde. That's why I'm
+# manually compiling zlib, openssl & libpq using musl-gcc. This certainly wouldn't have been
+# possible without this amazing Dockerfile, so I'm very glad I found it.
+# https://github.com/clux/muslrust/blob/master/Dockerfile
+
+# Build zlib (as in, gzip)
+RUN curl -sSL "https://zlib.net/zlib-$ZLIB_VER.tar.gz" | tar xz && \
+    cd "zlib-$ZLIB_VER" && \
+    LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
+        --static \
+        --prefix "$PREFIX" && \
+    make -j$(nproc) && \
+    make install
+
+# Build openSSL (needed to build libpq, & send https requests using reqwest)
+RUN curl -sSL "https://www.openssl.org/source/old/1.0.2/openssl-$SSL_VER.tar.gz" | tar xz && \
+    cd "openssl-$SSL_VER" && \
+    ./Configure \
+        no-zlib \
+        no-shared \
+        -fPIC \
+        --prefix="$PREFIX" \
+        --openssldir="$PREFIX/ssl" \
+        linux-x86_64 && \
+    env C_INCLUDE_PATH="$PREFIX/include" make depend 2> /dev/null && \
+    make -j$(nproc) && \
+    make install
+
+# Build libpq
+RUN curl -sSL "https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz" | tar xz && \
+    cd "postgresql-$PQ_VER" && \
+    LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \
+        --without-readline \
+        --with-openssl \
+        --prefix="$PREFIX" \
+        --host=x86_64-unknown-linux-musl && \
+    cd src/interfaces/libpq && \
+    make -s -j$(nproc) all-static-lib && \
+    make -s install install-lib-static && \
+    cd ../../bin/pg_config && \
+    make -j $(nproc) && \
+    make install
+
+# As far as I know these env vars just config the libraries in the correct way
+# to properly compile them later
+ENV PATH=$PREFIX/bin:$PATH \
+    PKG_CONFIG_ALLOW_CROSS=true \
+    PKG_CONFIG_ALL_STATIC=true \
+    PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL=true \
+    PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \
+    PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \
+    OPENSSL_STATIC=true \
+    OPENSSL_DIR=$PREFIX \
+    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
+    SSL_CERT_DIR=/etc/ssl/certs \
+    LIBZ_SYS_STATIC=1
+
+
+# =====BUILD DUMB-INIT=====
+# dumb-init is a very basic init system designed to act as the PID 1 process inside a Docker container.
+# The Makefile already specifies static build flags, so we don't have to specify them here.
+RUN curl -sSL "https://github.com/Yelp/dumb-init/archive/refs/tags/v$DI_VER.tar.gz" | \
+    tar xzf - && \
+    cd "dumb-init-$DI_VER" && \
+    CC="musl-gcc" make build
+
+
+# =====BUILD HILDE=====
+# Now that we've compiled all our C dependencies, we can build the actual binary using Cargo.
+
+COPY . ./hilde/
+RUN cd hilde && \
+    cargo build \
+        --release \
+        --target x86_64-unknown-linux-musl
+
+
+# =====CREATE RELEASE IMAGE=====
+# Thanks to all the compiling done above, we can now create a working Docker image using just static binaries.
+# This way, we don't need a base OS, which drastically reduces the final image size.
+FROM scratch
+ARG DI_VER
+
+COPY --from=builder /usr/src/dumb-init-$DI_VER/dumb-init /bin/dumb-init
+COPY --from=builder /usr/src/hilde/target/x86_64-unknown-linux-musl/release/hilde /bin/hilde
+
+ENTRYPOINT [ "/bin/dumb-init", "--" ]
+CMD [ "/bin/hilde" ]

migrations/2021-06-28-154739_distribution-metadata/down.sql

@@ -1,2 +0,0 @@
--- This file should undo anything in `up.sql`
-DROP TABLE distributions;

migrations/2021-06-28-154739_distribution-metadata/up.sql

@@ -1,12 +0,0 @@
--- Your SQL goes here
-CREATE TABLE distributions (
-    id uuid DEFAULT gen_random_uuid() PRIMARY KEY,
-
-    name text UNIQUE NOT NULL,
-    description text,
-    origin text,
-    label text,
-    version text,
-    suite text,
-    codename text
-);

src/hilde/main.rs

@@ -1,3 +1,7 @@
+// This needs to be explicitely included before diesel is imported to make sure
+// compilation succeeds
+extern crate openssl;
+
 #[macro_use]
 extern crate rocket;
 
@@ -33,5 +37,6 @@ fn rocket() -> _ {
             "Run database migrations",
             run_db_migrations,
         ))
-        .mount("/", routes::routes())
+        .mount("/pkgs", routes::pkgs::routes())
+        // .attach(routes::all())
 }

src/hilde/routes/mod.rs

@@ -1,13 +1 @@
-use rocket::Route;
+pub mod pkgs;
-use libhilde::packages::generate_release_file;
-use crate::HildeDbConn;
-
-pub fn routes() -> Vec<Route> {
-    routes![get_dist_release]
-}
-
-// NOTE: for now, only dists without slashes are supported
-#[get("/dists/<dist>/Release")]
-async fn get_dist_release(conn: HildeDbConn, dist: String) -> String {
-    conn.run(|c| generate_release_file(dist, c)).await
-}

src/hilde/routes/pkgs/mod.rs

@@ -0,0 +1,27 @@
+use rocket::{
+    Route,
+    fs::TempFile,
+    serde::json::Json
+};
+
+pub fn routes() -> Vec<Route> {
+    routes![get_package_info, upload_package_version]
+}
+
+// #[get("/<pkg>")]
+// async fn get_package_versions(pkg: String) -> Json<Vec<String>> {
+//     Json(vec![])
+// }
+
+/// Returns the stored metadata for the given package version.
+#[get("/<pkg>/<version>")]
+fn get_package_info(pkg: String, version: String) {
+
+}
+
+/// Upload a package version to the server.
+#[put("/<pkg>/<version>", data="<file>")]
+async fn upload_package_version(mut file: TempFile<'_>, pkg: String, version: String) {
+    
+}
+

src/libhilde/lib.rs

@@ -1,5 +1,5 @@
-#[macro_use]
+// Not sure if this is needed here
-extern crate diesel;
+// extern crate openssl;
+#[macro_use] extern crate diesel;
 
-pub mod packages;
 pub mod schema;

src/libhilde/packages.rs

@@ -1,21 +0,0 @@
-use diesel::{PgConnection, Queryable, QueryDsl, ExpressionMethods, RunQueryDsl};
-use crate::schema::distributions::dsl::*;
-
-#[derive(Queryable)]
-struct Distribution {
-    id: uuid::Uuid,
-    name: String,
-    description: Option<String>,
-    origin: Option<String>,
-    label: Option<String>,
-    version: Option<String>,
-    suite: Option<String>,
-    codename: Option<String>
-}
-
-/// Generate a Release file for a given distribution
-pub fn generate_release_file(dist: String, conn: &PgConnection) -> String {
-    let results = distributions.filter(name.eq(dist)).load::<Distribution>(conn).unwrap();
-
-    format!("{}", results.len())
-}

src/libhilde/pkgs/deb.rs

@@ -0,0 +1,3 @@
+use std::fs::PathBuf;
+
+pub fn extract_control(path: PathBuf) -> 

src/libhilde/schema.rs

@@ -1,12 +0,0 @@
-table! {
-    distributions (id) {
-        id -> Uuid,
-        name -> Text,
-        description -> Nullable<Text>,
-        origin -> Nullable<Text>,
-        label -> Nullable<Text>,
-        version -> Nullable<Text>,
-        suite -> Nullable<Text>,
-        codename -> Nullable<Text>,
-    }
-}