From e08048d0f0136046fca2e2b43e50bcf8e58cd548 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 30 Jul 2023 15:52:11 +0200 Subject: [PATCH 01/22] feat(server): initialize database migrations --- Cargo.lock | 1660 ++++++++++++++++- server/Cargo.toml | 2 + server/src/cli.rs | 2 + .../m20230730_000001_create_repo_tables.rs | 47 + server/src/db/migrator/mod.rs | 12 + server/src/db/mod.rs | 16 + server/src/main.rs | 1 + 7 files changed, 1732 insertions(+), 8 deletions(-) create mode 100644 server/src/db/migrator/m20230730_000001_create_repo_tables.rs create mode 100644 server/src/db/migrator/mod.rs create mode 100644 server/src/db/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 19a7080..2abb492 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,29 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.0.2" @@ -26,6 +49,33 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.3.2" @@ -75,6 +125,34 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] + [[package]] name = "async-trait" version = "0.1.71" @@ -83,7 +161,16 @@ checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.25", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -156,6 +243,29 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -167,6 +277,21 @@ name = "bitflags" version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +dependencies = [ + "serde", +] + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] [[package]] name = "block-buffer" @@ -177,6 +302,85 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borsh" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" +dependencies = [ + "borsh-derive", + "hashbrown 0.12.3", +] + +[[package]] +name = "borsh-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" +dependencies = [ + "borsh-derive-internal", + "borsh-schema-derive-internal", + "proc-macro-crate", + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "borsh-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "borsh-schema-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "bytecheck" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.4.0" @@ -195,6 +399,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "winapi", +] + [[package]] name = "clap" version = "4.3.12" @@ -227,7 +444,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.25", ] [[package]] @@ -242,6 +459,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "const-oid" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "795bc6e66a8e340f075fcf6227e417a2dc976b92b91f3cdc778bb858778b6747" + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + [[package]] name = "cpufeatures" version = "0.2.9" @@ -251,6 +480,40 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" + +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -261,6 +524,28 @@ dependencies = [ "typenum", ] +[[package]] +name = "der" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ed52955ce76b1554f509074bb357d3fb8ac9b51288a65a3fd480d1dfba946" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "digest" version = "0.10.7" @@ -268,9 +553,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -292,6 +600,41 @@ dependencies = [ "libc", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "pin-project", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -307,6 +650,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.28" @@ -349,6 +698,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -363,7 +723,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.25", ] [[package]] @@ -423,6 +783,12 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "h2" version = "0.3.19" @@ -435,7 +801,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -447,12 +813,37 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.6", +] + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash 0.8.3", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" +dependencies = [ + "hashbrown 0.14.0", +] [[package]] name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -475,6 +866,33 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + [[package]] name = "http" version = "0.2.9" @@ -539,6 +957,39 @@ dependencies = [ "want", ] +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -546,7 +997,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + +[[package]] +name = "inherent" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce243b1bfa62ffc028f1cc3b6034ec63d649f3031bc8a4fbbb004e1ac17d1f68" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", ] [[package]] @@ -560,17 +1032,38 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "libarchive" @@ -594,6 +1087,23 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + +[[package]] +name = "libsqlite3-sys" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.3" @@ -631,6 +1141,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest", +] + [[package]] name = "memchr" version = "2.5.0" @@ -653,6 +1172,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -673,6 +1198,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -683,6 +1218,65 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -708,6 +1302,39 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "ordered-float" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc2dbde8f8a79f2102cc474ceb0ad68e3b80b85289ea62389b60e66777e4213" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ouroboros" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ba07320d39dfea882faa70554b4bd342a5f273ed59ba7c1c6b4c840492c954" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4c6225c69b4ca778c0aea097321a64c421cf4577b331c61b229267edabb6f8" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.25", +] + [[package]] name = "overload" version = "0.1.1" @@ -737,6 +1364,21 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.0" @@ -760,7 +1402,7 @@ checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.25", ] [[package]] @@ -775,12 +1417,72 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.64" @@ -790,6 +1492,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quote" version = "1.0.29" @@ -799,6 +1521,42 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -852,6 +1610,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +[[package]] +name = "rend" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab" +dependencies = [ + "bytecheck", +] + [[package]] name = "rieterd" version = "0.1.0" @@ -860,6 +1627,8 @@ dependencies = [ "clap", "futures", "libarchive", + "sea-orm", + "sea-orm-migration", "sha256", "tokio", "tokio-util", @@ -870,6 +1639,88 @@ dependencies = [ "uuid", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rkyv" +version = "0.7.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" +dependencies = [ + "bitvec", + "bytecheck", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "rsa" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" +dependencies = [ + "byteorder", + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust_decimal" +version = "1.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a2ab0025103a60ecaaf3abf24db1db240a4e1c15837090d2c32f625ac98abea" +dependencies = [ + "arrayvec", + "borsh", + "byteorder", + "bytes", + "num-traits", + "rand", + "rkyv", + "serde", + "serde_json", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -889,6 +1740,36 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustls" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" +dependencies = [ + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.13" @@ -907,11 +1788,200 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sea-bae" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd3534a9978d0aa7edd2808dc1f8f31c4d0ecd31ddf71d997b3c98e9f3c9114" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.25", +] + +[[package]] +name = "sea-orm" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b508f060b689cd91abd76973ba4f1b0f416bb09915e2177fd3c6f853be76bc" +dependencies = [ + "async-stream", + "async-trait", + "bigdecimal", + "chrono", + "futures", + "log", + "ouroboros", + "rust_decimal", + "sea-orm-macros", + "sea-query", + "sea-query-binder", + "serde", + "serde_json", + "sqlx", + "strum", + "thiserror", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sea-orm-cli" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3b389690bccc6565e6a6af367599957b196288c4f556a7b4157a8c5086eb8d" +dependencies = [ + "chrono", + "clap", + "dotenvy", + "glob", + "regex", + "sea-schema", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "sea-orm-macros" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d49560a5a1bbd57f82fa1a622a8deefa51e44ce3a0b27f012f50bb6092a914f" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "sea-bae", + "syn 2.0.25", + "unicode-ident", +] + +[[package]] +name = "sea-orm-migration" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdccbb31e93d1a426a0df84f5f36d0b84d302444c3156e3a4fedb0b8da64ae87" +dependencies = [ + "async-trait", + "clap", + "dotenvy", + "futures", + "sea-orm", + "sea-orm-cli", + "sea-schema", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "sea-query" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aeb899964df7038e7274306b742951b82a04f835bca8a4683a4c254a6bf35fa" +dependencies = [ + "bigdecimal", + "chrono", + "derivative", + "inherent", + "ordered-float", + "rust_decimal", + "sea-query-derive", + "serde_json", + "time", + "uuid", +] + +[[package]] +name = "sea-query-binder" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36bbb68df92e820e4d5aeb17b4acd5cc8b5d18b2c36a4dd6f4626aabfa7ab1b9" +dependencies = [ + "bigdecimal", + "chrono", + "rust_decimal", + "sea-query", + "serde_json", + "sqlx", + "time", + "uuid", +] + +[[package]] +name = "sea-query-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd78f2e0ee8e537e9195d1049b752e0433e2cac125426bccb7b5c3e508096117" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "sea-schema" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e09eb40c78cee8fef8dfbb648036a26b7ad1f618499203ad0e8b6f97593f7f" +dependencies = [ + "futures", + "sea-query", + "sea-schema-derive", +] + +[[package]] +name = "sea-schema-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f686050f76bffc4f635cda8aea6df5548666b830b52387e8bc7de11056d11e" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "serde" version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] [[package]] name = "serde_json" @@ -946,6 +2016,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.7" @@ -985,6 +2066,22 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "slab" version = "0.4.8" @@ -1010,12 +2107,304 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" +dependencies = [ + "ahash 0.8.3", + "atoi", + "bigdecimal", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "dotenvy", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap 2.0.0", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rust_decimal", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "time", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" +dependencies = [ + "atoi", + "base64", + "bigdecimal", + "bitflags 2.3.3", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "rust_decimal", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" +dependencies = [ + "atoi", + "base64", + "bigdecimal", + "bitflags 2.3.3", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "num-bigint", + "once_cell", + "rand", + "rust_decimal", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stringprep" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.25" @@ -1033,6 +2422,45 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] + [[package]] name = "thread_local" version = "1.1.7" @@ -1043,6 +2471,48 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" +dependencies = [ + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.29.1" @@ -1071,7 +2541,18 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.25", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", ] [[package]] @@ -1088,6 +2569,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "tower" version = "0.4.13" @@ -1162,7 +2652,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.25", ] [[package]] @@ -1225,12 +2715,56 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + [[package]] name = "unicode-ident" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.1" @@ -1244,6 +2778,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" dependencies = [ "getrandom", + "serde", ] [[package]] @@ -1252,6 +2787,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -1273,6 +2814,85 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.25", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" +dependencies = [ + "rustls-webpki", +] + +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" + [[package]] name = "winapi" version = "0.3.9" @@ -1295,6 +2915,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -1360,3 +2989,18 @@ name = "windows_x86_64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/server/Cargo.toml b/server/Cargo.toml index 4081a63..13520fc 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,6 +11,8 @@ axum = { version = "0.6.18", features = ["http2"] } clap = { version = "4.3.12", features = ["env", "derive"] } futures = "0.3.28" libarchive = { path = "../libarchive" } +sea-orm = { version = "0.12.1", features = ["sqlx-sqlite", "runtime-tokio-rustls", "macros"] } +sea-orm-migration = "0.12.1" sha256 = "1.1.4" tokio = { version = "1.29.1", features = ["full"] } tokio-util = { version = "0.7.8", features = ["io"] } diff --git a/server/src/cli.rs b/server/src/cli.rs index 687b45b..dfb8ba8 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -45,6 +45,8 @@ impl Cli { pub async fn run(&self) { self.init_tracing(); + let db = crate::db::init("sqlite://test.db?mode=rwc").await.unwrap(); + let config = Config { repo_dir: self.repo_dir.clone(), pkg_dir: self.pkg_dir.clone(), diff --git a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs new file mode 100644 index 0000000..4aa74e1 --- /dev/null +++ b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs @@ -0,0 +1,47 @@ +use sea_orm_migration::prelude::*; + +pub struct Migration; + +impl MigrationName for Migration { + fn name(&self) -> &str { + "m_20230730_000001_create_repo_tables" + } +} + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + // Define how to apply this migration: Create the Bakery table. + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Repo::Table) + .col( + ColumnDef::new(Repo::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(Repo::Name).string().not_null()) + .col(ColumnDef::new(Repo::Description).string()) + .to_owned(), + ) + .await + } + + // Define how to rollback this migration: Drop the Bakery table. + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(Repo::Table).to_owned()) + .await + } +} + +#[derive(Iden)] +pub enum Repo { + Table, + Id, + Name, + Description, +} diff --git a/server/src/db/migrator/mod.rs b/server/src/db/migrator/mod.rs new file mode 100644 index 0000000..d939276 --- /dev/null +++ b/server/src/db/migrator/mod.rs @@ -0,0 +1,12 @@ +use sea_orm_migration::prelude::*; + +pub struct Migrator; + +mod m20230730_000001_create_repo_tables; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![Box::new(m20230730_000001_create_repo_tables::Migration)] + } +} diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs new file mode 100644 index 0000000..ee85b4a --- /dev/null +++ b/server/src/db/mod.rs @@ -0,0 +1,16 @@ +mod migrator; + +use migrator::Migrator; +use sea_orm::ConnectOptions; +use sea_orm::Database; +use sea_orm_migration::MigratorTrait; + +pub async fn init>( + opt: C, +) -> Result { + let mut db = Database::connect(opt).await?; + + Migrator::refresh(&db).await?; + + Ok(db) +} diff --git a/server/src/main.rs b/server/src/main.rs index 2332b4b..3cfaa5c 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,4 +1,5 @@ mod cli; +mod db; mod error; mod repo; From 37218536c5bda33e27686875e3785acc1c7b38da Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 30 Jul 2023 16:55:44 +0200 Subject: [PATCH 02/22] feat(server): start api using CRUD operations --- Cargo.lock | 9 +++++---- server/Cargo.toml | 1 + server/src/api/mod.rs | 19 +++++++++++++++++++ server/src/cli.rs | 2 ++ server/src/db/entities/mod.rs | 5 +++++ server/src/db/entities/prelude.rs | 3 +++ server/src/db/entities/repo.rs | 18 ++++++++++++++++++ .../m20230730_000001_create_repo_tables.rs | 2 +- server/src/db/mod.rs | 3 ++- server/src/error.rs | 10 ++++++++++ server/src/main.rs | 3 +++ 11 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 server/src/api/mod.rs create mode 100644 server/src/db/entities/mod.rs create mode 100644 server/src/db/entities/prelude.rs create mode 100644 server/src/db/entities/repo.rs diff --git a/Cargo.lock b/Cargo.lock index 2abb492..7e82ecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1629,6 +1629,7 @@ dependencies = [ "libarchive", "sea-orm", "sea-orm-migration", + "serde", "sha256", "tokio", "tokio-util", @@ -1965,18 +1966,18 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "60363bdd39a7be0266a520dab25fdc9241d2f987b08a01e01f0ec6d06a981348" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "f28482318d6641454cb273da158647922d1be6b5a2fcc6165cd89ebdd7ed576b" dependencies = [ "proc-macro2", "quote", diff --git a/server/Cargo.toml b/server/Cargo.toml index 13520fc..68a7c6b 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -13,6 +13,7 @@ futures = "0.3.28" libarchive = { path = "../libarchive" } sea-orm = { version = "0.12.1", features = ["sqlx-sqlite", "runtime-tokio-rustls", "macros"] } sea-orm-migration = "0.12.1" +serde = { version = "1.0.178", features = ["derive"] } sha256 = "1.1.4" tokio = { version = "1.29.1", features = ["full"] } tokio-util = { version = "0.7.8", features = ["io"] } diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs new file mode 100644 index 0000000..7f1824b --- /dev/null +++ b/server/src/api/mod.rs @@ -0,0 +1,19 @@ +use axum::Router; +use axum::extract::State; +use axum::routing::get; +use sea_orm::entity::EntityTrait; +use sea_orm::query::QueryOrder; +use axum::Json; + +use crate::db::entities::repo; + +pub fn router() -> Router { + Router::new() + .route("/repos", get(get_repos)) +} + +async fn get_repos(State(global): State) -> crate::Result>> { + let repos = repo::Entity::find().order_by_asc(repo::Column::Id).all(&global.db).await?; + + Ok(Json(repos)) +} diff --git a/server/src/cli.rs b/server/src/cli.rs index dfb8ba8..b109e2b 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -56,10 +56,12 @@ impl Cli { let global = Global { config, repo_manager: Arc::new(RwLock::new(repo_manager)), + db }; // build our application with a single route let app = Router::new() + .nest("/api", crate::api::router()) .merge(crate::repo::router()) .with_state(global) .layer(TraceLayer::new_for_http()); diff --git a/server/src/db/entities/mod.rs b/server/src/db/entities/mod.rs new file mode 100644 index 0000000..f1e964a --- /dev/null +++ b/server/src/db/entities/mod.rs @@ -0,0 +1,5 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +pub mod prelude; + +pub mod repo; diff --git a/server/src/db/entities/prelude.rs b/server/src/db/entities/prelude.rs new file mode 100644 index 0000000..8b651f8 --- /dev/null +++ b/server/src/db/entities/prelude.rs @@ -0,0 +1,3 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +pub use super::repo::Entity as Repo; diff --git a/server/src/db/entities/repo.rs b/server/src/db/entities/repo.rs new file mode 100644 index 0000000..0676a76 --- /dev/null +++ b/server/src/db/entities/repo.rs @@ -0,0 +1,18 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::Serialize; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] +#[sea_orm(table_name = "repo")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub name: String, + pub description: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs index 4aa74e1..4c36ade 100644 --- a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs +++ b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs @@ -23,7 +23,7 @@ impl MigrationTrait for Migration { .auto_increment() .primary_key(), ) - .col(ColumnDef::new(Repo::Name).string().not_null()) + .col(ColumnDef::new(Repo::Name).string().not_null().unique_key()) .col(ColumnDef::new(Repo::Description).string()) .to_owned(), ) diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index ee85b4a..e1ad0b9 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -1,4 +1,5 @@ mod migrator; +pub mod entities; use migrator::Migrator; use sea_orm::ConnectOptions; @@ -8,7 +9,7 @@ use sea_orm_migration::MigratorTrait; pub async fn init>( opt: C, ) -> Result { - let mut db = Database::connect(opt).await?; + let db = Database::connect(opt).await?; Migrator::refresh(&db).await?; diff --git a/server/src/error.rs b/server/src/error.rs index 0a7772f..0dca03c 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -11,6 +11,7 @@ pub enum ServerError { IO(io::Error), Axum(axum::Error), Status(StatusCode), + Db(sea_orm::DbErr), } impl fmt::Display for ServerError { @@ -19,6 +20,7 @@ impl fmt::Display for ServerError { ServerError::IO(err) => write!(fmt, "{}", err), ServerError::Axum(err) => write!(fmt, "{}", err), ServerError::Status(status) => write!(fmt, "{}", status), + ServerError::Db(err) => write!(fmt, "{}", err), } } } @@ -33,6 +35,8 @@ impl IntoResponse for ServerError { ServerError::IO(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), ServerError::Axum(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), ServerError::Status(status) => status.into_response(), + ServerError::Db(sea_orm::DbErr::RecordNotFound(_)) => StatusCode::NOT_FOUND.into_response(), + ServerError::Db(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), } } } @@ -60,3 +64,9 @@ impl From for ServerError { Self::Status(status) } } + +impl From for ServerError { + fn from(err: sea_orm::DbErr) -> Self { + ServerError::Db(err) + } +} diff --git a/server/src/main.rs b/server/src/main.rs index 3cfaa5c..8884047 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -2,12 +2,14 @@ mod cli; mod db; mod error; mod repo; +mod api; use clap::Parser; pub use error::{Result, ServerError}; use repo::RepoGroupManager; use std::path::PathBuf; use std::sync::{Arc, RwLock}; +use sea_orm::DatabaseConnection; #[derive(Clone)] pub struct Config { @@ -19,6 +21,7 @@ pub struct Config { pub struct Global { config: Config, repo_manager: Arc>, + db: DatabaseConnection } #[tokio::main] From 25627e166e9b2cc5749ec7f35c4ca8be72a47f16 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 30 Jul 2023 18:21:07 +0200 Subject: [PATCH 03/22] feat(server): example of pagination --- server/src/api/mod.rs | 30 ++++++++++++++++++++++-------- server/src/cli.rs | 4 ++-- server/src/db/mod.rs | 4 ++-- server/src/error.rs | 4 +++- server/src/main.rs | 6 +++--- 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 7f1824b..982dc02 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -1,19 +1,33 @@ -use axum::Router; -use axum::extract::State; +use axum::extract::{Query, State}; use axum::routing::get; +use axum::Json; +use axum::Router; use sea_orm::entity::EntityTrait; use sea_orm::query::QueryOrder; -use axum::Json; +use sea_orm::PaginatorTrait; +use serde::Deserialize; use crate::db::entities::repo; -pub fn router() -> Router { - Router::new() - .route("/repos", get(get_repos)) +#[derive(Deserialize)] +pub struct Pagination { + page: Option, + per_page: Option, } -async fn get_repos(State(global): State) -> crate::Result>> { - let repos = repo::Entity::find().order_by_asc(repo::Column::Id).all(&global.db).await?; +pub fn router() -> Router { + Router::new().route("/repos", get(get_repos)) +} + +async fn get_repos( + State(global): State, + Query(params): Query, +) -> crate::Result>> { + let repos = repo::Entity::find() + .order_by_asc(repo::Column::Id) + .paginate(&global.db, params.per_page.unwrap_or(25)) + .fetch_page(params.page.unwrap_or(0)) + .await?; Ok(Json(repos)) } diff --git a/server/src/cli.rs b/server/src/cli.rs index b109e2b..90c368b 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -45,7 +45,7 @@ impl Cli { pub async fn run(&self) { self.init_tracing(); - let db = crate::db::init("sqlite://test.db?mode=rwc").await.unwrap(); + let db = crate::db::init("sqlite://test.db").await.unwrap(); let config = Config { repo_dir: self.repo_dir.clone(), @@ -56,7 +56,7 @@ impl Cli { let global = Global { config, repo_manager: Arc::new(RwLock::new(repo_manager)), - db + db, }; // build our application with a single route diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index e1ad0b9..4e0e804 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -1,5 +1,5 @@ -mod migrator; pub mod entities; +mod migrator; use migrator::Migrator; use sea_orm::ConnectOptions; @@ -11,7 +11,7 @@ pub async fn init>( ) -> Result { let db = Database::connect(opt).await?; - Migrator::refresh(&db).await?; + Migrator::up(&db, None).await?; Ok(db) } diff --git a/server/src/error.rs b/server/src/error.rs index 0dca03c..8ff467a 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -35,7 +35,9 @@ impl IntoResponse for ServerError { ServerError::IO(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), ServerError::Axum(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), ServerError::Status(status) => status.into_response(), - ServerError::Db(sea_orm::DbErr::RecordNotFound(_)) => StatusCode::NOT_FOUND.into_response(), + ServerError::Db(sea_orm::DbErr::RecordNotFound(_)) => { + StatusCode::NOT_FOUND.into_response() + } ServerError::Db(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), } } diff --git a/server/src/main.rs b/server/src/main.rs index 8884047..a6d41b5 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,15 +1,15 @@ +mod api; mod cli; mod db; mod error; mod repo; -mod api; use clap::Parser; pub use error::{Result, ServerError}; use repo::RepoGroupManager; +use sea_orm::DatabaseConnection; use std::path::PathBuf; use std::sync::{Arc, RwLock}; -use sea_orm::DatabaseConnection; #[derive(Clone)] pub struct Config { @@ -21,7 +21,7 @@ pub struct Config { pub struct Global { config: Config, repo_manager: Arc>, - db: DatabaseConnection + db: DatabaseConnection, } #[tokio::main] From e63d0b55652cd4ee12d453a277d03cfce487d906 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 31 Jul 2023 23:11:02 +0200 Subject: [PATCH 04/22] feat(server): pagination --- Cargo.toml | 6 ++++++ server/Cargo.toml | 6 ------ server/src/api/mod.rs | 39 +++++++++++++++++++++++------------- server/src/api/pagination.rs | 34 +++++++++++++++++++++++++++++++ server/src/error.rs | 9 +++++++++ 5 files changed, 74 insertions(+), 20 deletions(-) create mode 100644 server/src/api/pagination.rs diff --git a/Cargo.toml b/Cargo.toml index cdd0740..a2754f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,3 +4,9 @@ members = [ 'libarchive', 'libarchive3-sys' ] + +[profile.release] +lto = "fat" +codegen-units = 1 +panic = "abort" +strip = true diff --git a/server/Cargo.toml b/server/Cargo.toml index 68a7c6b..aeb0f48 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -22,9 +22,3 @@ tower-http = { version = "0.4.1", features = ["fs", "trace"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } uuid = { version = "1.4.0", features = ["v4"] } - -[profile.release] -lto = "fat" -codegen-units = 1 -panic = "abort" -strip = true diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 982dc02..91a1023 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -1,33 +1,44 @@ -use axum::extract::{Query, State}; +mod pagination; + +use axum::extract::{Path, Query, State}; use axum::routing::get; use axum::Json; use axum::Router; use sea_orm::entity::EntityTrait; use sea_orm::query::QueryOrder; use sea_orm::PaginatorTrait; -use serde::Deserialize; + +use pagination::PaginatedResponse; use crate::db::entities::repo; -#[derive(Deserialize)] -pub struct Pagination { - page: Option, - per_page: Option, -} - pub fn router() -> Router { - Router::new().route("/repos", get(get_repos)) + Router::new() + .route("/repos", get(get_repos)) + .route("/repos/:id", get(get_single_repo)) } async fn get_repos( State(global): State, - Query(params): Query, -) -> crate::Result>> { + Query(pagination): Query, +) -> crate::Result>> { let repos = repo::Entity::find() .order_by_asc(repo::Column::Id) - .paginate(&global.db, params.per_page.unwrap_or(25)) - .fetch_page(params.page.unwrap_or(0)) + .paginate(&global.db, pagination.per_page.unwrap_or(25)) + .fetch_page(pagination.page.unwrap_or(1) - 1) .await?; - Ok(Json(repos)) + Ok(Json(pagination.res(repos))) +} + +async fn get_single_repo( + State(global): State, + Path(id): Path, +) -> crate::Result> { + let repo = repo::Entity::find_by_id(id) + .one(&global.db) + .await? + .ok_or(axum::http::StatusCode::NOT_FOUND)?; + + Ok(Json(repo)) } diff --git a/server/src/api/pagination.rs b/server/src/api/pagination.rs new file mode 100644 index 0000000..ae3812d --- /dev/null +++ b/server/src/api/pagination.rs @@ -0,0 +1,34 @@ +use axum::response::{IntoResponse, Response}; +use axum::Json; +use serde::{Deserialize, Serialize}; + +pub const DEFAULT_PAGE: u64 = 0; +pub const DEFAULT_PER_PAGE: u64 = 25; + +#[derive(Deserialize)] +pub struct Query { + pub page: Option, + pub per_page: Option, +} + +#[derive(Serialize)] +pub struct PaginatedResponse +where + T: for<'de> Serialize, +{ + pub page: u64, + pub per_page: u64, + pub count: usize, + pub items: Vec, +} + +impl Query { + pub fn res Serialize>(self, items: Vec) -> PaginatedResponse { + PaginatedResponse { + page: self.page.unwrap_or(DEFAULT_PAGE), + per_page: self.page.unwrap_or(DEFAULT_PER_PAGE), + count: items.len(), + items, + } + } +} diff --git a/server/src/error.rs b/server/src/error.rs index 8ff467a..a359572 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -12,6 +12,7 @@ pub enum ServerError { Axum(axum::Error), Status(StatusCode), Db(sea_orm::DbErr), + Status(StatusCode), } impl fmt::Display for ServerError { @@ -21,6 +22,7 @@ impl fmt::Display for ServerError { ServerError::Axum(err) => write!(fmt, "{}", err), ServerError::Status(status) => write!(fmt, "{}", status), ServerError::Db(err) => write!(fmt, "{}", err), + ServerError::Status(status) => write!(fmt, "{}", status), } } } @@ -39,6 +41,7 @@ impl IntoResponse for ServerError { StatusCode::NOT_FOUND.into_response() } ServerError::Db(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + ServerError::Status(status) => status.into_response(), } } } @@ -72,3 +75,9 @@ impl From for ServerError { ServerError::Db(err) } } + +impl From for ServerError { + fn from(status: StatusCode) -> Self { + ServerError::Status(status) + } +} From a2d844c582afe185bc12d6574cb781520e8f546a Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 1 Aug 2023 13:29:20 +0200 Subject: [PATCH 05/22] feat(server): start of package database schema --- server/Cargo.toml | 11 ++- server/src/api/mod.rs | 15 +++ server/src/cli.rs | 5 +- server/src/db/entities/mod.rs | 2 + server/src/db/entities/package.rs | 53 ++++++++++ server/src/db/entities/package_license.rs | 33 +++++++ server/src/db/entities/prelude.rs | 2 + server/src/db/entities/repo.rs | 16 ++- .../m20230730_000001_create_repo_tables.rs | 98 ++++++++++++++++++- 9 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 server/src/db/entities/package.rs create mode 100644 server/src/db/entities/package_license.rs diff --git a/server/Cargo.toml b/server/Cargo.toml index aeb0f48..400b621 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,7 +11,6 @@ axum = { version = "0.6.18", features = ["http2"] } clap = { version = "4.3.12", features = ["env", "derive"] } futures = "0.3.28" libarchive = { path = "../libarchive" } -sea-orm = { version = "0.12.1", features = ["sqlx-sqlite", "runtime-tokio-rustls", "macros"] } sea-orm-migration = "0.12.1" serde = { version = "1.0.178", features = ["derive"] } sha256 = "1.1.4" @@ -22,3 +21,13 @@ tower-http = { version = "0.4.1", features = ["fs", "trace"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } uuid = { version = "1.4.0", features = ["v4"] } + +[dependencies.sea-orm] +version = "0.12.1" +features = [ + "sqlx-sqlite", + "sqlx-postgres", + "runtime-tokio-rustls", + "macros", + "with-chrono" +] diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 91a1023..011f008 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -10,12 +10,14 @@ use sea_orm::PaginatorTrait; use pagination::PaginatedResponse; +use crate::db::entities::package; use crate::db::entities::repo; pub fn router() -> Router { Router::new() .route("/repos", get(get_repos)) .route("/repos/:id", get(get_single_repo)) + .route("/packages", get(get_packages)) } async fn get_repos( @@ -42,3 +44,16 @@ async fn get_single_repo( Ok(Json(repo)) } + +async fn get_packages( + State(global): State, + Query(pagination): Query, +) -> crate::Result>> { + let pkgs = package::Entity::find() + .order_by_asc(package::Column::Id) + .paginate(&global.db, pagination.per_page.unwrap_or(25)) + .fetch_page(pagination.page.unwrap_or(1) - 1) + .await?; + + Ok(Json(pagination.res(pkgs))) +} diff --git a/server/src/cli.rs b/server/src/cli.rs index 90c368b..bdad1fb 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -45,7 +45,10 @@ impl Cli { pub async fn run(&self) { self.init_tracing(); - let db = crate::db::init("sqlite://test.db").await.unwrap(); + // let db = crate::db::init("sqlite://test.db").await.unwrap(); + let db = crate::db::init("postgres://rieter:rieter@localhost:5432/rieter") + .await + .unwrap(); let config = Config { repo_dir: self.repo_dir.clone(), diff --git a/server/src/db/entities/mod.rs b/server/src/db/entities/mod.rs index f1e964a..e5ad810 100644 --- a/server/src/db/entities/mod.rs +++ b/server/src/db/entities/mod.rs @@ -2,4 +2,6 @@ pub mod prelude; +pub mod package; +pub mod package_license; pub mod repo; diff --git a/server/src/db/entities/package.rs b/server/src/db/entities/package.rs new file mode 100644 index 0000000..7920aac --- /dev/null +++ b/server/src/db/entities/package.rs @@ -0,0 +1,53 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "package")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub repo_id: i32, + pub base: String, + pub name: String, + pub version: String, + pub arch: String, + pub size: i64, + pub c_size: i64, + pub description: Option, + pub url: Option, + pub build_date: Option, + pub packager: Option, + pub pgp_sig: Option, + pub pgp_sig_size: Option, + pub sha256_sum: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::package_license::Entity")] + PackageLicense, + #[sea_orm( + belongs_to = "super::repo::Entity", + from = "Column::RepoId", + to = "super::repo::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Repo, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::PackageLicense.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Repo.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/entities/package_license.rs b/server/src/db/entities/package_license.rs new file mode 100644 index 0000000..2920d3f --- /dev/null +++ b/server/src/db/entities/package_license.rs @@ -0,0 +1,33 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "package_license")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub package_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub value: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::package::Entity", + from = "Column::PackageId", + to = "super::package::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Package, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Package.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/entities/prelude.rs b/server/src/db/entities/prelude.rs index 8b651f8..1f8176c 100644 --- a/server/src/db/entities/prelude.rs +++ b/server/src/db/entities/prelude.rs @@ -1,3 +1,5 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 +pub use super::package::Entity as Package; +pub use super::package_license::Entity as PackageLicense; pub use super::repo::Entity as Repo; diff --git a/server/src/db/entities/repo.rs b/server/src/db/entities/repo.rs index 0676a76..b7a1af1 100644 --- a/server/src/db/entities/repo.rs +++ b/server/src/db/entities/repo.rs @@ -1,18 +1,28 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 use sea_orm::entity::prelude::*; -use serde::Serialize; +use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] #[sea_orm(table_name = "repo")] pub struct Model { #[sea_orm(primary_key)] pub id: i32, + #[sea_orm(unique)] pub name: String, pub description: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} +pub enum Relation { + #[sea_orm(has_many = "super::package::Entity")] + Package, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Package.def() + } +} impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs index 4c36ade..5e6420a 100644 --- a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs +++ b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs @@ -10,7 +10,6 @@ impl MigrationName for Migration { #[async_trait::async_trait] impl MigrationTrait for Migration { - // Define how to apply this migration: Create the Bakery table. async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { manager .create_table( @@ -27,11 +26,81 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Repo::Description).string()) .to_owned(), ) + .await?; + manager + .create_table( + Table::create() + .table(Package::Table) + .col( + ColumnDef::new(Package::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(Package::RepoId).integer().not_null()) + .col(ColumnDef::new(Package::Base).string_len(255).not_null()) + .col(ColumnDef::new(Package::Name).string_len(255).not_null()) + .col(ColumnDef::new(Package::Version).string_len(255).not_null()) + .col(ColumnDef::new(Package::Arch).string_len(255).not_null()) + .col(ColumnDef::new(Package::Size).big_integer().not_null()) + .col(ColumnDef::new(Package::CSize).big_integer().not_null()) + .col(ColumnDef::new(Package::Description).string()) + .col(ColumnDef::new(Package::Url).string_len(255)) + .col(ColumnDef::new(Package::BuildDate).date_time()) + .col(ColumnDef::new(Package::Packager).string_len(255)) + .col(ColumnDef::new(Package::PgpSig).string_len(255)) + .col(ColumnDef::new(Package::PgpSigSize).big_integer()) + .col(ColumnDef::new(Package::Sha256Sum).char_len(64).not_null()) + .foreign_key( + ForeignKey::create() + .name("fk-package-repo_id") + .from(Package::Table, Package::RepoId) + .to(Repo::Table, Repo::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(PackageLicense::Table) + .col( + ColumnDef::new(PackageLicense::PackageId) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(PackageLicense::Value) + .string_len(255) + .not_null(), + ) + .primary_key( + Index::create() + .col(PackageLicense::PackageId) + .col(PackageLicense::Value), + ) + .foreign_key( + ForeignKey::create() + .name("fk-package_license-package_id") + .from(PackageLicense::Table, PackageLicense::PackageId) + .to(Package::Table, Package::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) .await } // Define how to rollback this migration: Drop the Bakery table. async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(PackageLicense::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(Package::Table).to_owned()) + .await?; manager .drop_table(Table::drop().table(Repo::Table).to_owned()) .await @@ -45,3 +114,30 @@ pub enum Repo { Name, Description, } + +#[derive(Iden)] +pub enum Package { + Table, + Id, + RepoId, + Name, + Base, + Version, + Description, + Size, + CSize, + Url, + Arch, + BuildDate, + Packager, + PgpSig, + PgpSigSize, + Sha256Sum, +} + +#[derive(Iden)] +pub enum PackageLicense { + Table, + PackageId, + Value, +} From f706b72b7c0b722ecdc3715ac444aa867c7e0df4 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 1 Aug 2023 13:54:26 +0200 Subject: [PATCH 06/22] feat(server): improve package parse semantics --- Cargo.lock | 39 +++++++++--- server/Cargo.toml | 1 + .../m20230730_000001_create_repo_tables.rs | 2 +- server/src/repo/package.rs | 59 ++++++++++++------- 4 files changed, 69 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e82ecd..d9c6efe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -407,8 +407,11 @@ checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "time 0.1.45", + "wasm-bindgen", "winapi", ] @@ -774,7 +777,7 @@ checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -1194,7 +1197,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] @@ -1624,6 +1627,7 @@ name = "rieterd" version = "0.1.0" dependencies = [ "axum", + "chrono", "clap", "futures", "libarchive", @@ -1834,7 +1838,7 @@ dependencies = [ "sqlx", "strum", "thiserror", - "time", + "time 0.3.23", "tracing", "url", "uuid", @@ -1902,7 +1906,7 @@ dependencies = [ "rust_decimal", "sea-query-derive", "serde_json", - "time", + "time 0.3.23", "uuid", ] @@ -1918,7 +1922,7 @@ dependencies = [ "sea-query", "serde_json", "sqlx", - "time", + "time 0.3.23", "uuid", ] @@ -2196,7 +2200,7 @@ dependencies = [ "smallvec", "sqlformat", "thiserror", - "time", + "time 0.3.23", "tokio", "tokio-stream", "tracing", @@ -2285,7 +2289,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", - "time", + "time 0.3.23", "tracing", "uuid", "whoami", @@ -2330,7 +2334,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", - "time", + "time 0.3.23", "tracing", "uuid", "whoami", @@ -2355,7 +2359,7 @@ dependencies = [ "percent-encoding", "serde", "sqlx-core", - "time", + "time 0.3.23", "tracing", "url", "uuid", @@ -2472,6 +2476,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + [[package]] name = "time" version = "0.3.23" @@ -2809,6 +2824,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/server/Cargo.toml b/server/Cargo.toml index 400b621..c147eae 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -8,6 +8,7 @@ authors = ["Jef Roosens"] [dependencies] axum = { version = "0.6.18", features = ["http2"] } +chrono = { version = "0.4.26", features = ["serde"] } clap = { version = "4.3.12", features = ["env", "derive"] } futures = "0.3.28" libarchive = { path = "../libarchive" } diff --git a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs index 5e6420a..df56657 100644 --- a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs +++ b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs @@ -47,7 +47,7 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(Package::CSize).big_integer().not_null()) .col(ColumnDef::new(Package::Description).string()) .col(ColumnDef::new(Package::Url).string_len(255)) - .col(ColumnDef::new(Package::BuildDate).date_time()) + .col(ColumnDef::new(Package::BuildDate).date_time().not_null()) .col(ColumnDef::new(Package::Packager).string_len(255)) .col(ColumnDef::new(Package::PgpSig).string_len(255)) .col(ColumnDef::new(Package::PgpSigSize).big_integer()) diff --git a/server/src/repo/package.rs b/server/src/repo/package.rs index 4ff9433..6d8070b 100644 --- a/server/src/repo/package.rs +++ b/server/src/repo/package.rs @@ -1,3 +1,4 @@ +use chrono::NaiveDateTime; use libarchive::read::{Archive, Builder}; use libarchive::{Entry, ReadFilter}; use std::fmt; @@ -17,18 +18,18 @@ pub struct Package { #[derive(Debug, Default)] pub struct PkgInfo { - pub name: String, pub base: String, + pub name: String, pub version: String, - pub description: String, - pub size: u64, - pub csize: u64, - pub url: String, pub arch: String, - pub build_date: i64, - pub packager: String, - pub pgpsig: String, - pub pgpsigsize: i64, + pub description: Option, + pub size: i64, + pub csize: i64, + pub url: Option, + pub build_date: NaiveDateTime, + pub packager: Option, + pub pgpsig: Option, + pub pgpsigsize: Option, pub groups: Vec, pub licenses: Vec, pub replaces: Vec, @@ -70,23 +71,27 @@ impl PkgInfo { "pkgname" => self.name = value.to_string(), "pkgbase" => self.base = value.to_string(), "pkgver" => self.version = value.to_string(), - "pkgdesc" => self.description = value.to_string(), + "pkgdesc" => self.description = Some(value.to_string()), "size" => { self.size = value.parse().map_err(|_| ParsePkgInfoError::InvalidSize)? } - "url" => self.url = value.to_string(), + "url" => self.url = Some(value.to_string()), "arch" => self.arch = value.to_string(), "builddate" => { - self.build_date = value + let seconds: i64 = value .parse() - .map_err(|_| ParsePkgInfoError::InvalidBuildDate)? + .map_err(|_| ParsePkgInfoError::InvalidBuildDate)?; + self.build_date = NaiveDateTime::from_timestamp_millis(seconds * 1000) + .ok_or(ParsePkgInfoError::InvalidBuildDate)? } - "packager" => self.packager = value.to_string(), - "pgpsig" => self.pgpsig = value.to_string(), + "packager" => self.packager = Some(value.to_string()), + "pgpsig" => self.pgpsig = Some(value.to_string()), "pgpsigsize" => { - self.pgpsigsize = value - .parse() - .map_err(|_| ParsePkgInfoError::InvalidPgpSigSize)? + self.pgpsigsize = Some( + value + .parse() + .map_err(|_| ParsePkgInfoError::InvalidPgpSigSize)?, + ) } "group" => self.groups.push(value.to_string()), "license" => self.licenses.push(value.to_string()), @@ -156,7 +161,8 @@ impl Package { } if let Some(mut info) = info { - info.csize = fs::metadata(path.as_ref())?.len(); + // I'll take my chances on a file size fitting in an i64 + info.csize = fs::metadata(path.as_ref())?.len().try_into().unwrap(); Ok(Package { path: path.as_ref().to_path_buf(), @@ -216,7 +222,10 @@ impl Package { write("NAME", &info.name)?; write("BASE", &info.base)?; write("VERSION", &info.version)?; - write("DESC", &info.description)?; + + if let Some(ref description) = info.description { + write("DESC", description)?; + } write("GROUPS", &info.groups.join("\n"))?; write("CSIZE", &info.csize.to_string())?; write("ISIZE", &info.size.to_string())?; @@ -225,11 +234,17 @@ impl Package { write("SHA256SUM", checksum)?; } - write("URL", &info.url)?; + if let Some(ref url) = info.url { + write("URL", url)?; + } + write("LICENSE", &info.licenses.join("\n"))?; write("ARCH", &info.arch)?; write("BUILDDATE", &info.build_date.to_string())?; - write("PACKAGER", &info.packager)?; + + if let Some(ref packager) = info.packager { + write("PACKAGER", packager)?; + } write("REPLACES", &info.replaces.join("\n"))?; write("CONFLICTS", &info.conflicts.join("\n"))?; From 7c6f485ea60fdebfcfe7fb8c8d3c555aa12f9bec Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 1 Aug 2023 14:38:50 +0200 Subject: [PATCH 07/22] feat(server): update database when publishing packages --- server/src/cli.rs | 8 +++--- server/src/db/entities/package.rs | 2 +- server/src/error.rs | 7 ------ server/src/main.rs | 2 +- server/src/repo/manager.rs | 13 +++++++--- server/src/repo/mod.rs | 34 ++++++++++++++++++++++++- server/src/repo/package.rs | 41 ++++++++++++++++++++++--------- 7 files changed, 78 insertions(+), 29 deletions(-) diff --git a/server/src/cli.rs b/server/src/cli.rs index bdad1fb..144e56e 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -45,10 +45,10 @@ impl Cli { pub async fn run(&self) { self.init_tracing(); - // let db = crate::db::init("sqlite://test.db").await.unwrap(); - let db = crate::db::init("postgres://rieter:rieter@localhost:5432/rieter") - .await - .unwrap(); + let db = crate::db::init("sqlite://test.db").await.unwrap(); + // let db = crate::db::init("postgres://rieter:rieter@localhost:5432/rieter") + // .await + // .unwrap(); let config = Config { repo_dir: self.repo_dir.clone(), diff --git a/server/src/db/entities/package.rs b/server/src/db/entities/package.rs index 7920aac..86606bf 100644 --- a/server/src/db/entities/package.rs +++ b/server/src/db/entities/package.rs @@ -17,7 +17,7 @@ pub struct Model { pub c_size: i64, pub description: Option, pub url: Option, - pub build_date: Option, + pub build_date: DateTime, pub packager: Option, pub pgp_sig: Option, pub pgp_sig_size: Option, diff --git a/server/src/error.rs b/server/src/error.rs index a359572..1299d2f 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -10,7 +10,6 @@ pub type Result = std::result::Result; pub enum ServerError { IO(io::Error), Axum(axum::Error), - Status(StatusCode), Db(sea_orm::DbErr), Status(StatusCode), } @@ -75,9 +74,3 @@ impl From for ServerError { ServerError::Db(err) } } - -impl From for ServerError { - fn from(status: StatusCode) -> Self { - ServerError::Status(status) - } -} diff --git a/server/src/main.rs b/server/src/main.rs index a6d41b5..8217909 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,6 +1,6 @@ mod api; mod cli; -mod db; +pub mod db; mod error; mod repo; diff --git a/server/src/repo/manager.rs b/server/src/repo/manager.rs index 6b11641..f5aad8c 100644 --- a/server/src/repo/manager.rs +++ b/server/src/repo/manager.rs @@ -104,9 +104,12 @@ impl RepoGroupManager { Ok(()) } - pub fn add_pkg_from_path>(&mut self, repo: &str, path: P) -> io::Result<()> { - let mut pkg = Package::open(&path)?; - pkg.calculate_checksum()?; + pub fn add_pkg_from_path>( + &mut self, + repo: &str, + path: P, + ) -> io::Result { + let pkg = Package::open(&path)?; self.add_pkg(repo, &pkg)?; @@ -118,7 +121,9 @@ impl RepoGroupManager { .join(pkg.file_name()); fs::create_dir_all(dest_pkg_path.parent().unwrap())?; - fs::rename(&path, dest_pkg_path) + fs::rename(&path, dest_pkg_path)?; + + Ok(pkg) } /// Add a package to the given repo, returning to what architectures the package was added. diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index c5cb9f0..de3e74d 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -4,6 +4,7 @@ mod package; pub use manager::RepoGroupManager; use axum::body::Body; +use crate::db::entities::{package as db_package, repo as db_repo}; use axum::extract::{BodyStream, Path, State}; use axum::http::Request; use axum::http::StatusCode; @@ -11,6 +12,7 @@ use axum::response::IntoResponse; use axum::routing::{delete, post}; use axum::Router; use futures::StreamExt; +use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter}; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; use tower::util::ServiceExt; @@ -51,9 +53,39 @@ async fn post_package_archive( // Remove the downloaded file if the adding failed if res.is_err() { let _ = tokio::fs::remove_file(path).await; + + return res; } - Ok(res?) + let pkg = res.unwrap(); + + // Query the repo for its ID, or create it if it does not already exist + let repo_entity = db_repo::Entity::find() + .filter(db_repo::Column::Name.eq(&repo)) + .one(&global.db) + .await?; + + let repo_id = if let Some(repo_entity) = repo_entity { + repo_entity.id + } else { + let model = db_repo::ActiveModel { + name: sea_orm::Set(repo.clone()), + ..Default::default() + }; + + db_repo::Entity::insert(model) + .exec(&global.db) + .await? + .last_insert_id + }; + + // Insert the package's data into the database + let mut model: db_package::ActiveModel = pkg.into(); + model.repo_id = sea_orm::Set(repo_id); + + model.insert(&global.db).await?; + + Ok(()) } /// Serve the package archive files and database archives. If files are requested for an diff --git a/server/src/repo/package.rs b/server/src/repo/package.rs index 6d8070b..ddafbd9 100644 --- a/server/src/repo/package.rs +++ b/server/src/repo/package.rs @@ -1,11 +1,14 @@ use chrono::NaiveDateTime; use libarchive::read::{Archive, Builder}; use libarchive::{Entry, ReadFilter}; +use sea_orm::ActiveValue::Set; use std::fmt; use std::fs; use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; +use crate::db::entities::package; + const IGNORED_FILES: [&str; 5] = [".BUILDINFO", ".INSTALL", ".MTREE", ".PKGINFO", ".CHANGELOG"]; #[derive(Debug)] @@ -39,7 +42,7 @@ pub struct PkgInfo { pub optdepends: Vec, pub makedepends: Vec, pub checkdepends: Vec, - pub sha256sum: Option, + pub sha256sum: String, } #[derive(Debug, PartialEq, Eq)] @@ -163,6 +166,7 @@ impl Package { if let Some(mut info) = info { // I'll take my chances on a file size fitting in an i64 info.csize = fs::metadata(path.as_ref())?.len().try_into().unwrap(); + info.sha256sum = sha256::try_digest(path.as_ref())?; Ok(Package { path: path.as_ref().to_path_buf(), @@ -178,12 +182,6 @@ impl Package { } } - pub fn calculate_checksum(&mut self) -> io::Result<()> { - self.info.sha256sum = Some(sha256::try_digest(self.path.as_ref())?); - - Ok(()) - } - pub fn full_name(&self) -> String { format!( "{}-{}-{}", @@ -230,9 +228,7 @@ impl Package { write("CSIZE", &info.csize.to_string())?; write("ISIZE", &info.size.to_string())?; - if let Some(checksum) = &info.sha256sum { - write("SHA256SUM", checksum)?; - } + write("SHA256SUM", &info.sha256sum)?; if let Some(ref url) = info.url { write("URL", url)?; @@ -240,7 +236,7 @@ impl Package { write("LICENSE", &info.licenses.join("\n"))?; write("ARCH", &info.arch)?; - write("BUILDDATE", &info.build_date.to_string())?; + write("BUILDDATE", &info.build_date.timestamp().to_string())?; if let Some(ref packager) = info.packager { write("PACKAGER", packager)?; @@ -271,3 +267,26 @@ impl Package { Ok(()) } } + +impl From for package::ActiveModel { + fn from(pkg: Package) -> Self { + let info = pkg.info; + + package::ActiveModel { + base: Set(info.base), + name: Set(info.name), + version: Set(info.version), + arch: Set(info.arch), + size: Set(info.size), + c_size: Set(info.csize), + description: Set(info.description), + url: Set(info.url), + build_date: Set(info.build_date), + packager: Set(info.packager), + pgp_sig: Set(info.pgpsig), + pgp_sig_size: Set(info.pgpsigsize), + sha256_sum: Set(info.sha256sum), + ..Default::default() + } + } +} From afe73d5314a20973928cef69e5402e236de697d0 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 1 Aug 2023 15:30:38 +0200 Subject: [PATCH 08/22] feat(server): log errors; configurable database --- server/src/cli.rs | 49 ++++++++++++++++++++++++++++++------------ server/src/main.rs | 5 +++-- server/src/repo/mod.rs | 7 +++--- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/server/src/cli.rs b/server/src/cli.rs index 144e56e..0725e64 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -4,14 +4,25 @@ use crate::{Config, Global}; use axum::extract::FromRef; use axum::Router; use clap::Parser; +use std::io; use std::path::PathBuf; use std::sync::{Arc, RwLock}; use tower_http::trace::TraceLayer; +use tracing::debug; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; #[derive(Parser)] #[command(author, version, about, long_about = None)] pub struct Cli { + /// Directory where package archives will be stored + pub pkg_dir: PathBuf, + /// Directory where repository metadata & SQLite database is stored + pub data_dir: PathBuf, + + /// Database connection URL; either sqlite:// or postgres://. Defaults to rieter.sqlite in the + /// data directory + #[arg(short, long)] + pub database_url: Option, /// Port the server will listen on #[arg(short, long, value_name = "PORT", default_value_t = 8000)] pub port: u16, @@ -22,10 +33,6 @@ pub struct Cli { default_value = "tower_http=debug,rieterd=debug" )] pub log: String, - /// Directory where package archives will be stored - pub pkg_dir: PathBuf, - /// Directory where repository metadata is stored - pub repo_dir: PathBuf, } impl FromRef for Arc> { @@ -42,19 +49,31 @@ impl Cli { .init(); } - pub async fn run(&self) { + pub async fn run(&self) -> crate::Result<()> { self.init_tracing(); - let db = crate::db::init("sqlite://test.db").await.unwrap(); + let db_url = if let Some(url) = &self.database_url { + url.clone() + } else { + format!( + "sqlite://{}", + self.data_dir.join("rieter.sqlite").to_string_lossy() + ) + }; + + debug!("Connecting to database with URL {}", db_url); + + let db = crate::db::init(db_url).await?; // let db = crate::db::init("postgres://rieter:rieter@localhost:5432/rieter") - // .await - // .unwrap(); + // .await + // .unwrap(); let config = Config { - repo_dir: self.repo_dir.clone(), + data_dir: self.data_dir.clone(), + repo_dir: self.data_dir.join("repos"), pkg_dir: self.pkg_dir.clone(), }; - let repo_manager = RepoGroupManager::new(&self.repo_dir, &self.pkg_dir); + let repo_manager = RepoGroupManager::new(&config.repo_dir, &self.pkg_dir); let global = Global { config, @@ -70,9 +89,11 @@ impl Cli { .layer(TraceLayer::new_for_http()); // run it with hyper on localhost:3000 - axum::Server::bind(&format!("0.0.0.0:{}", self.port).parse().unwrap()) - .serve(app.into_make_service()) - .await - .unwrap(); + Ok( + axum::Server::bind(&format!("0.0.0.0:{}", self.port).parse().unwrap()) + .serve(app.into_make_service()) + .await + .map_err(|err| io::Error::new(io::ErrorKind::Other, err))?, + ) } } diff --git a/server/src/main.rs b/server/src/main.rs index 8217909..e3dbf36 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -13,6 +13,7 @@ use std::sync::{Arc, RwLock}; #[derive(Clone)] pub struct Config { + data_dir: PathBuf, repo_dir: PathBuf, pkg_dir: PathBuf, } @@ -25,7 +26,7 @@ pub struct Global { } #[tokio::main] -async fn main() { +async fn main() -> crate::Result<()> { let cli = cli::Cli::parse(); - cli.run().await; + cli.run().await } diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index de3e74d..b2942a4 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -45,16 +45,17 @@ async fn post_package_archive( let clone = Arc::clone(&global.repo_manager); let path_clone = path.clone(); + let repo_clone = repo.clone(); let res = tokio::task::spawn_blocking(move || { - clone.write().unwrap().add_pkg_from_path(&repo, &path_clone) + clone.write().unwrap().add_pkg_from_path(&repo_clone, &path_clone) }) .await?; // Remove the downloaded file if the adding failed - if res.is_err() { + if let Err(err) = res { let _ = tokio::fs::remove_file(path).await; - return res; + return Err(err.into()); } let pkg = res.unwrap(); From bc191587478ad3dcab53c1de46676a63de6194e8 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 2 Aug 2023 22:41:23 +0200 Subject: [PATCH 09/22] refactor(server): clean up some stuff --- server/src/api/pagination.rs | 2 - server/src/error.rs | 2 - server/src/repo/mod.rs | 77 +++++++++++++++++++----------------- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/server/src/api/pagination.rs b/server/src/api/pagination.rs index ae3812d..3904ea4 100644 --- a/server/src/api/pagination.rs +++ b/server/src/api/pagination.rs @@ -1,5 +1,3 @@ -use axum::response::{IntoResponse, Response}; -use axum::Json; use serde::{Deserialize, Serialize}; pub const DEFAULT_PAGE: u64 = 0; diff --git a/server/src/error.rs b/server/src/error.rs index 1299d2f..4fbb7c4 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -21,7 +21,6 @@ impl fmt::Display for ServerError { ServerError::Axum(err) => write!(fmt, "{}", err), ServerError::Status(status) => write!(fmt, "{}", status), ServerError::Db(err) => write!(fmt, "{}", err), - ServerError::Status(status) => write!(fmt, "{}", status), } } } @@ -40,7 +39,6 @@ impl IntoResponse for ServerError { StatusCode::NOT_FOUND.into_response() } ServerError::Db(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), - ServerError::Status(status) => status.into_response(), } } } diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index b2942a4..0a28969 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -3,8 +3,8 @@ mod package; pub use manager::RepoGroupManager; -use axum::body::Body; use crate::db::entities::{package as db_package, repo as db_repo}; +use axum::body::Body; use axum::extract::{BodyStream, Path, State}; use axum::http::Request; use axum::http::StatusCode; @@ -47,46 +47,51 @@ async fn post_package_archive( let path_clone = path.clone(); let repo_clone = repo.clone(); let res = tokio::task::spawn_blocking(move || { - clone.write().unwrap().add_pkg_from_path(&repo_clone, &path_clone) + clone + .write() + .unwrap() + .add_pkg_from_path(&repo_clone, &path_clone) }) .await?; - // Remove the downloaded file if the adding failed - if let Err(err) = res { - let _ = tokio::fs::remove_file(path).await; + match res { + // Insert the newly added package into the database + Ok(pkg) => { + // Query the repo for its ID, or create it if it does not already exist + let repo_entity = db_repo::Entity::find() + .filter(db_repo::Column::Name.eq(&repo)) + .one(&global.db) + .await?; - return Err(err.into()); + let repo_id = if let Some(repo_entity) = repo_entity { + repo_entity.id + } else { + let model = db_repo::ActiveModel { + name: sea_orm::Set(repo.clone()), + ..Default::default() + }; + + db_repo::Entity::insert(model) + .exec(&global.db) + .await? + .last_insert_id + }; + + // Insert the package's data into the database + let mut model: db_package::ActiveModel = pkg.into(); + model.repo_id = sea_orm::Set(repo_id); + + model.insert(&global.db).await?; + + Ok(()) + } + // Remove the uploaded file and return the error + Err(err) => { + tokio::fs::remove_file(path).await?; + + Err(err.into()) + } } - - let pkg = res.unwrap(); - - // Query the repo for its ID, or create it if it does not already exist - let repo_entity = db_repo::Entity::find() - .filter(db_repo::Column::Name.eq(&repo)) - .one(&global.db) - .await?; - - let repo_id = if let Some(repo_entity) = repo_entity { - repo_entity.id - } else { - let model = db_repo::ActiveModel { - name: sea_orm::Set(repo.clone()), - ..Default::default() - }; - - db_repo::Entity::insert(model) - .exec(&global.db) - .await? - .last_insert_id - }; - - // Insert the package's data into the database - let mut model: db_package::ActiveModel = pkg.into(); - model.repo_id = sea_orm::Set(repo_id); - - model.insert(&global.db).await?; - - Ok(()) } /// Serve the package archive files and database archives. If files are requested for an From a7e0c03b58216ac0baacaee0684fdb0bf8e2a8bc Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 3 Aug 2023 09:34:33 +0200 Subject: [PATCH 10/22] feat(server): authorized requests --- Cargo.lock | 13 ++++++++++--- server/Cargo.toml | 2 +- server/src/repo/mod.rs | 21 ++++++++++++++++++--- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9c6efe..804a3d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -243,6 +243,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "base64" version = "0.21.2" @@ -1762,7 +1768,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64", + "base64 0.21.2", ] [[package]] @@ -2255,7 +2261,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" dependencies = [ "atoi", - "base64", + "base64 0.21.2", "bigdecimal", "bitflags 2.3.3", "byteorder", @@ -2302,7 +2308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" dependencies = [ "atoi", - "base64", + "base64 0.21.2", "bigdecimal", "bitflags 2.3.3", "byteorder", @@ -2616,6 +2622,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c" dependencies = [ + "base64 0.20.0", "bitflags 2.3.3", "bytes", "futures-core", diff --git a/server/Cargo.toml b/server/Cargo.toml index c147eae..bf4fd7f 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -18,7 +18,7 @@ sha256 = "1.1.4" tokio = { version = "1.29.1", features = ["full"] } tokio-util = { version = "0.7.8", features = ["io"] } tower = { version = "0.4.13", features = ["make"] } -tower-http = { version = "0.4.1", features = ["fs", "trace"] } +tower-http = { version = "0.4.1", features = ["fs", "trace", "auth"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } uuid = { version = "1.4.0", features = ["v4"] } diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index 0a28969..aec846b 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -17,15 +17,28 @@ use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; use tower::util::ServiceExt; use tower_http::services::{ServeDir, ServeFile}; +use tower_http::validate_request::ValidateRequestHeaderLayer; use uuid::Uuid; pub fn router() -> Router { Router::new() - .route("/:repo", post(post_package_archive).delete(delete_repo)) - .route("/:repo/:arch", delete(delete_arch_repo)) + .route( + "/:repo", + post(post_package_archive) + .delete(delete_repo) + .layer(ValidateRequestHeaderLayer::bearer("test")), + ) + .route( + "/:repo/:arch", + delete(delete_arch_repo).layer(ValidateRequestHeaderLayer::bearer("test")), + ) + // Routes added after the layer do not get that layer applied, so the GET requests will not + // be authorized .route( "/:repo/:arch/:filename", - delete(delete_package).get(get_file), + delete(delete_package) + .layer(ValidateRequestHeaderLayer::bearer("test")) + .get(get_file), ) } @@ -57,6 +70,8 @@ async fn post_package_archive( match res { // Insert the newly added package into the database Ok(pkg) => { + tracing::info!("Added '{}' to repository '{}'", pkg.file_name(), repo); + // Query the repo for its ID, or create it if it does not already exist let repo_entity = db_repo::Entity::find() .filter(db_repo::Column::Name.eq(&repo)) From 33c8477b09a7af14edfa15a864ae7999b1855748 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 3 Aug 2023 11:03:01 +0200 Subject: [PATCH 11/22] feat(server): log repository updates --- CHANGELOG.md | 1 + server/src/repo/mod.rs | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f715cfe..7cdad22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,3 +16,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 requests * Packages of architecture "any" are part of every architecture's database + * Bearer authentication for private routes diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index aec846b..ad99740 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -26,18 +26,18 @@ pub fn router() -> Router { "/:repo", post(post_package_archive) .delete(delete_repo) - .layer(ValidateRequestHeaderLayer::bearer("test")), + .route_layer(ValidateRequestHeaderLayer::bearer("test")), ) .route( "/:repo/:arch", - delete(delete_arch_repo).layer(ValidateRequestHeaderLayer::bearer("test")), + delete(delete_arch_repo).route_layer(ValidateRequestHeaderLayer::bearer("test")), ) // Routes added after the layer do not get that layer applied, so the GET requests will not // be authorized .route( "/:repo/:arch/:filename", delete(delete_package) - .layer(ValidateRequestHeaderLayer::bearer("test")) + .route_layer(ValidateRequestHeaderLayer::bearer("test")) .get(get_file), ) } @@ -165,10 +165,13 @@ async fn delete_repo( ) -> crate::Result { let clone = Arc::clone(&global.repo_manager); + let repo_clone = repo.clone(); let repo_removed = - tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo(&repo)).await??; + tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo(&repo_clone)).await??; if repo_removed { + tracing::info!("Removed repo '{}'", repo); + Ok(StatusCode::OK) } else { Ok(StatusCode::NOT_FOUND) @@ -181,11 +184,14 @@ async fn delete_arch_repo( ) -> crate::Result { let clone = Arc::clone(&global.repo_manager); + let log = format!("Removed architecture '{}' from repo '{}'", arch, repo); let repo_removed = tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo_arch(&repo, &arch)) .await??; if repo_removed { + tracing::info!(log); + Ok(StatusCode::OK) } else { Ok(StatusCode::NOT_FOUND) @@ -205,6 +211,7 @@ async fn delete_package( } let name = name_parts[..name_parts.len() - 3].join("-"); + let log = format!("Removed '{}' from repository '{}'", file_name, repo); let clone = Arc::clone(&global.repo_manager); @@ -214,6 +221,8 @@ async fn delete_package( .await??; if pkg_removed { + tracing::info!(log); + Ok(StatusCode::OK) } else { Ok(StatusCode::NOT_FOUND) From fd1c2d3647156caac9b6ed15af37cf0713cdaad4 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 3 Aug 2023 11:08:38 +0200 Subject: [PATCH 12/22] feat(server): configurable api key --- server/src/cli.rs | 5 ++++- server/src/main.rs | 1 + server/src/repo/mod.rs | 15 ++++++++------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/server/src/cli.rs b/server/src/cli.rs index 0725e64..8b160d5 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -18,6 +18,8 @@ pub struct Cli { pub pkg_dir: PathBuf, /// Directory where repository metadata & SQLite database is stored pub data_dir: PathBuf, + /// API key to authenticate private routes with + pub api_key: String, /// Database connection URL; either sqlite:// or postgres://. Defaults to rieter.sqlite in the /// data directory @@ -72,6 +74,7 @@ impl Cli { data_dir: self.data_dir.clone(), repo_dir: self.data_dir.join("repos"), pkg_dir: self.pkg_dir.clone(), + api_key: self.api_key.clone(), }; let repo_manager = RepoGroupManager::new(&config.repo_dir, &self.pkg_dir); @@ -84,7 +87,7 @@ impl Cli { // build our application with a single route let app = Router::new() .nest("/api", crate::api::router()) - .merge(crate::repo::router()) + .merge(crate::repo::router(&self.api_key)) .with_state(global) .layer(TraceLayer::new_for_http()); diff --git a/server/src/main.rs b/server/src/main.rs index e3dbf36..c2b0eaf 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -16,6 +16,7 @@ pub struct Config { data_dir: PathBuf, repo_dir: PathBuf, pkg_dir: PathBuf, + api_key: String, } #[derive(Clone)] diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index ad99740..2bad3d5 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -20,24 +20,24 @@ use tower_http::services::{ServeDir, ServeFile}; use tower_http::validate_request::ValidateRequestHeaderLayer; use uuid::Uuid; -pub fn router() -> Router { +pub fn router(api_key: &str) -> Router { Router::new() .route( "/:repo", post(post_package_archive) .delete(delete_repo) - .route_layer(ValidateRequestHeaderLayer::bearer("test")), + .route_layer(ValidateRequestHeaderLayer::bearer(api_key)), ) .route( "/:repo/:arch", - delete(delete_arch_repo).route_layer(ValidateRequestHeaderLayer::bearer("test")), + delete(delete_arch_repo).route_layer(ValidateRequestHeaderLayer::bearer(api_key)), ) // Routes added after the layer do not get that layer applied, so the GET requests will not // be authorized .route( "/:repo/:arch/:filename", delete(delete_package) - .route_layer(ValidateRequestHeaderLayer::bearer("test")) + .route_layer(ValidateRequestHeaderLayer::bearer(api_key)) .get(get_file), ) } @@ -167,10 +167,11 @@ async fn delete_repo( let repo_clone = repo.clone(); let repo_removed = - tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo(&repo_clone)).await??; + tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo(&repo_clone)) + .await??; if repo_removed { - tracing::info!("Removed repo '{}'", repo); + tracing::info!("Removed repository '{}'", repo); Ok(StatusCode::OK) } else { @@ -184,7 +185,7 @@ async fn delete_arch_repo( ) -> crate::Result { let clone = Arc::clone(&global.repo_manager); - let log = format!("Removed architecture '{}' from repo '{}'", arch, repo); + let log = format!("Removed architecture '{}' from repository '{}'", arch, repo); let repo_removed = tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo_arch(&repo, &arch)) .await??; From b8d53f43b6050096f3d6edb83111db11bbe44d47 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 3 Aug 2023 14:31:12 +0200 Subject: [PATCH 13/22] fix(server): actually serve desc & files in db archives --- libarchive/src/archive.rs | 6 ++++++ libarchive/src/write/file.rs | 34 ++++++++++++++++++++++------------ server/src/repo/manager.rs | 8 ++++++++ 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/libarchive/src/archive.rs b/libarchive/src/archive.rs index faaacc9..d8f7086 100644 --- a/libarchive/src/archive.rs +++ b/libarchive/src/archive.rs @@ -314,6 +314,12 @@ pub trait Entry { ffi::archive_entry_set_mode(self.entry_mut(), mode); } } + + fn set_size(&mut self, size: i64) { + unsafe { + ffi::archive_entry_set_size(self.entry_mut(), size); + } + } } #[derive(Debug, PartialEq, Eq)] diff --git a/libarchive/src/write/file.rs b/libarchive/src/write/file.rs index 08f2b45..9c7abe2 100644 --- a/libarchive/src/write/file.rs +++ b/libarchive/src/write/file.rs @@ -39,22 +39,32 @@ impl FileWriter { } } - let mut buf = [0; 8192]; + let mut buf = [0; 4096]; loop { match r.read(&mut buf) { Ok(0) => return Ok(()), - Ok(written) => unsafe { - match ffi::archive_write_data( - self.handle_mut(), - buf.as_ptr() as *const _, - written, - ) as i32 - { - ffi::ARCHIVE_OK => (), - _ => return Err(ArchiveError::from(self as &dyn Handle).into()), - }; - }, + // Write entire buffer + Ok(buf_len) => { + let mut written: usize = 0; + + while written < buf_len { + let res = unsafe { + ffi::archive_write_data( + self.handle_mut(), + &buf[written] as *const u8 as *const _, + buf_len - written, + ) + } as isize; + + // Negative values signal errors + if res < 0 { + return Err(ArchiveError::from(self as &dyn Handle).into()); + } + + written += usize::try_from(res).unwrap(); + } + } Err(err) => match err.kind() { io::ErrorKind::Interrupted => (), _ => return Err(err.into()), diff --git a/server/src/repo/manager.rs b/server/src/repo/manager.rs index f5aad8c..411cf98 100644 --- a/server/src/repo/manager.rs +++ b/server/src/repo/manager.rs @@ -65,9 +65,13 @@ impl RepoGroupManager { // The desc file needs to be added to both archives let path_in_tar = PathBuf::from(entry.file_name()).join("desc"); let src_path = entry.path().join("desc"); + let metadata = src_path.metadata()?; let mut ar_entry = WriteEntry::new(); ar_entry.set_pathname(&path_in_tar); + // These small text files will definitely fit inside an i64 + ar_entry.set_size(metadata.len().try_into().unwrap()); + ar_entry.set_filetype(libarchive::archive::FileType::RegularFile); ar_entry.set_mode(0o100644); ar_db.append_path(&mut ar_entry, &src_path)?; @@ -76,10 +80,14 @@ impl RepoGroupManager { // The files file is only required in the files database let path_in_tar = PathBuf::from(entry.file_name()).join("files"); let src_path = entry.path().join("files"); + let metadata = src_path.metadata()?; let mut ar_entry = WriteEntry::new(); + ar_entry.set_filetype(libarchive::archive::FileType::RegularFile); ar_entry.set_pathname(&path_in_tar); ar_entry.set_mode(0o100644); + // These small text files will definitely fit inside an i64 + ar_entry.set_size(metadata.len().try_into().unwrap()); ar_files.append_path(&mut ar_entry, src_path)?; } From b85f57b11274f07285f62330f2d16431970cf06b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 3 Aug 2023 15:08:34 +0200 Subject: [PATCH 14/22] fix(server): also properly serve file database --- server/src/repo/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index 2bad3d5..0c2b327 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -20,6 +20,8 @@ use tower_http::services::{ServeDir, ServeFile}; use tower_http::validate_request::ValidateRequestHeaderLayer; use uuid::Uuid; +const DB_FILE_EXTS: [&str; 4] = [".db", ".files", ".db.tar.gz", ".files.tar.gz"]; + pub fn router(api_key: &str) -> Router { Router::new() .route( @@ -120,9 +122,9 @@ async fn get_file( let repo_dir = global.config.repo_dir.join(&repo).join(&arch); let repo_exists = tokio::fs::try_exists(&repo_dir).await?; - let res = if file_name.ends_with(".db") || file_name.ends_with(".db.tar.gz") { + let res = if DB_FILE_EXTS.iter().any(|ext| file_name.ends_with(ext)) { // Append tar extension to ensure we find the file - if file_name.ends_with(".db") { + if !file_name.ends_with(".tar.gz") { file_name.push_str(".tar.gz"); }; From 731ad37a2a64d2f20f0ad2e06f4996054f9913ec Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 3 Aug 2023 21:25:42 +0200 Subject: [PATCH 15/22] feat(server): properly sync database with repo operations --- server/src/api/pagination.rs | 2 +- server/src/repo/manager.rs | 45 +++++++ server/src/repo/mod.rs | 227 +++++++++++++++++++++-------------- 3 files changed, 185 insertions(+), 89 deletions(-) diff --git a/server/src/api/pagination.rs b/server/src/api/pagination.rs index 3904ea4..376e06c 100644 --- a/server/src/api/pagination.rs +++ b/server/src/api/pagination.rs @@ -24,7 +24,7 @@ impl Query { pub fn res Serialize>(self, items: Vec) -> PaginatedResponse { PaginatedResponse { page: self.page.unwrap_or(DEFAULT_PAGE), - per_page: self.page.unwrap_or(DEFAULT_PER_PAGE), + per_page: self.per_page.unwrap_or(DEFAULT_PER_PAGE), count: items.len(), items, } diff --git a/server/src/repo/manager.rs b/server/src/repo/manager.rs index 411cf98..5846b5d 100644 --- a/server/src/repo/manager.rs +++ b/server/src/repo/manager.rs @@ -259,4 +259,49 @@ impl RepoGroupManager { Ok(false) } + + /// Wrapper around `remove_pkg` that accepts a path relative to the package directory to a + /// package archive. + pub fn remove_pkg_from_path>( + &mut self, + path: P, + sync: bool, + ) -> io::Result> { + let path = path.as_ref(); + let components: Vec<_> = path.iter().collect(); + + if let [repo, _arch, file_name] = components[..] { + let full_path = self.pkg_dir.join(path); + + if full_path.try_exists()? { + let file_name = file_name.to_string_lossy(); + let (name, version, release, arch) = parse_pkg_filename(&file_name); + + let metadata_dir_name = format!("{}-{}-{}", name, version, release); + + // Remove package archive and entry in database + fs::remove_file(full_path)?; + fs::remove_dir_all(self.repo_dir.join(repo).join(arch).join(metadata_dir_name))?; + + if sync { + if arch == ANY_ARCH { + self.sync_all(&repo.to_string_lossy())?; + } else { + self.sync(&repo.to_string_lossy(), arch)?; + } + } + + Ok(Some(( + name, + version.to_string(), + release.to_string(), + arch.to_string(), + ))) + } else { + Ok(None) + } + } else { + Ok(None) + } + } } diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index 0c2b327..d300ffe 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -3,6 +3,8 @@ mod package; pub use manager::RepoGroupManager; +use std::path::PathBuf; + use crate::db::entities::{package as db_package, repo as db_repo}; use axum::body::Body; use axum::extract::{BodyStream, Path, State}; @@ -12,7 +14,7 @@ use axum::response::IntoResponse; use axum::routing::{delete, post}; use axum::Router; use futures::StreamExt; -use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter}; +use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, ModelTrait, QueryFilter}; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; use tower::util::ServiceExt; @@ -44,73 +46,6 @@ pub fn router(api_key: &str) -> Router { ) } -async fn post_package_archive( - State(global): State, - Path(repo): Path, - mut body: BodyStream, -) -> crate::Result<()> { - // We first stream the uploaded file to disk - let uuid: uuid::fmt::Simple = Uuid::new_v4().into(); - let path = global.config.pkg_dir.join(uuid.to_string()); - let mut f = fs::File::create(&path).await?; - - while let Some(chunk) = body.next().await { - f.write_all(&chunk?).await?; - } - - let clone = Arc::clone(&global.repo_manager); - let path_clone = path.clone(); - let repo_clone = repo.clone(); - let res = tokio::task::spawn_blocking(move || { - clone - .write() - .unwrap() - .add_pkg_from_path(&repo_clone, &path_clone) - }) - .await?; - - match res { - // Insert the newly added package into the database - Ok(pkg) => { - tracing::info!("Added '{}' to repository '{}'", pkg.file_name(), repo); - - // Query the repo for its ID, or create it if it does not already exist - let repo_entity = db_repo::Entity::find() - .filter(db_repo::Column::Name.eq(&repo)) - .one(&global.db) - .await?; - - let repo_id = if let Some(repo_entity) = repo_entity { - repo_entity.id - } else { - let model = db_repo::ActiveModel { - name: sea_orm::Set(repo.clone()), - ..Default::default() - }; - - db_repo::Entity::insert(model) - .exec(&global.db) - .await? - .last_insert_id - }; - - // Insert the package's data into the database - let mut model: db_package::ActiveModel = pkg.into(); - model.repo_id = sea_orm::Set(repo_id); - - model.insert(&global.db).await?; - - Ok(()) - } - // Remove the uploaded file and return the error - Err(err) => { - tokio::fs::remove_file(path).await?; - - Err(err.into()) - } - } -} - /// Serve the package archive files and database archives. If files are requested for an /// architecture that does not have any explicit packages, a repository containing only "any" files /// is returned. @@ -161,6 +96,85 @@ async fn get_file( Ok(res) } +async fn post_package_archive( + State(global): State, + Path(repo): Path, + mut body: BodyStream, +) -> crate::Result<()> { + // We first stream the uploaded file to disk + let uuid: uuid::fmt::Simple = Uuid::new_v4().into(); + let path = global.config.pkg_dir.join(uuid.to_string()); + let mut f = fs::File::create(&path).await?; + + while let Some(chunk) = body.next().await { + f.write_all(&chunk?).await?; + } + + let clone = Arc::clone(&global.repo_manager); + let path_clone = path.clone(); + let repo_clone = repo.clone(); + let res = tokio::task::spawn_blocking(move || { + clone + .write() + .unwrap() + .add_pkg_from_path(&repo_clone, &path_clone) + }) + .await?; + + match res { + // Insert the newly added package into the database + Ok(pkg) => { + tracing::info!("Added '{}' to repository '{}'", pkg.file_name(), repo); + + // Query the repo for its ID, or create it if it does not already exist + let res = db_repo::Entity::find() + .filter(db_repo::Column::Name.eq(&repo)) + .one(&global.db) + .await?; + + let repo_id = if let Some(repo_entity) = res { + repo_entity.id + } else { + let model = db_repo::ActiveModel { + name: sea_orm::Set(repo.clone()), + ..Default::default() + }; + + db_repo::Entity::insert(model) + .exec(&global.db) + .await? + .last_insert_id + }; + + // If the package already exists in the database, we remove it first + let res = db_package::Entity::find() + .filter(db_package::Column::RepoId.eq(repo_id)) + .filter(db_package::Column::Name.eq(&pkg.info.name)) + .filter(db_package::Column::Arch.eq(&pkg.info.arch)) + .one(&global.db) + .await?; + + if let Some(entry) = res { + entry.delete(&global.db).await?; + } + + // Insert the package's data into the database + let mut model: db_package::ActiveModel = pkg.into(); + model.repo_id = sea_orm::Set(repo_id); + + model.insert(&global.db).await?; + + Ok(()) + } + // Remove the uploaded file and return the error + Err(err) => { + tokio::fs::remove_file(path).await?; + + Err(err.into()) + } + } +} + async fn delete_repo( State(global): State, Path(repo): Path, @@ -173,6 +187,15 @@ async fn delete_repo( .await??; if repo_removed { + let res = db_repo::Entity::find() + .filter(db_repo::Column::Name.eq(&repo)) + .one(&global.db) + .await?; + + if let Some(repo_entry) = res { + repo_entry.delete(&global.db).await?; + } + tracing::info!("Removed repository '{}'", repo); Ok(StatusCode::OK) @@ -187,13 +210,31 @@ async fn delete_arch_repo( ) -> crate::Result { let clone = Arc::clone(&global.repo_manager); - let log = format!("Removed architecture '{}' from repository '{}'", arch, repo); - let repo_removed = - tokio::task::spawn_blocking(move || clone.write().unwrap().remove_repo_arch(&repo, &arch)) - .await??; + let arch_clone = arch.clone(); + let repo_clone = repo.clone(); + let repo_removed = tokio::task::spawn_blocking(move || { + clone + .write() + .unwrap() + .remove_repo_arch(&repo_clone, &arch_clone) + }) + .await??; if repo_removed { - tracing::info!(log); + let res = db_repo::Entity::find() + .filter(db_repo::Column::Name.eq(&repo)) + .one(&global.db) + .await?; + + if let Some(repo_entry) = res { + // Also remove all packages for that architecture from database + db_package::Entity::delete_many() + .filter(db_package::Column::RepoId.eq(repo_entry.id)) + .filter(db_package::Column::Arch.eq(&arch)) + .exec(&global.db) + .await?; + } + tracing::info!("Removed architecture '{}' from repository '{}'", arch, repo); Ok(StatusCode::OK) } else { @@ -205,26 +246,36 @@ async fn delete_package( State(global): State, Path((repo, arch, file_name)): Path<(String, String, String)>, ) -> crate::Result { - let name_parts = file_name.split('-').collect::>(); - - // Package archive files use the naming scheme pkgname-pkgver-pkgrel-arch, so a valid - // name contains at least 4 dash-separated sections - if name_parts.len() < 4 { - return Ok(StatusCode::NOT_FOUND); - } - - let name = name_parts[..name_parts.len() - 3].join("-"); - let log = format!("Removed '{}' from repository '{}'", file_name, repo); - let clone = Arc::clone(&global.repo_manager); + let path = PathBuf::from(&repo).join(arch).join(&file_name); - let pkg_removed = tokio::task::spawn_blocking(move || { - clone.write().unwrap().remove_pkg(&repo, &arch, &name, true) + let res = tokio::task::spawn_blocking(move || { + clone.write().unwrap().remove_pkg_from_path(path, true) }) .await??; - if pkg_removed { - tracing::info!(log); + if let Some((name, version, release, arch)) = res { + let res = db_repo::Entity::find() + .filter(db_repo::Column::Name.eq(&repo)) + .one(&global.db) + .await?; + + if let Some(repo_entry) = res { + // Also remove entry from database + let res = db_package::Entity::find() + .filter(db_package::Column::RepoId.eq(repo_entry.id)) + .filter(db_package::Column::Name.eq(name)) + .filter(db_package::Column::Version.eq(format!("{}-{}", version, release))) + .filter(db_package::Column::Arch.eq(arch)) + .one(&global.db) + .await?; + + if let Some(entry) = res { + entry.delete(&global.db).await?; + } + } + + tracing::info!("Removed '{}' from repository '{}'", file_name, repo); Ok(StatusCode::OK) } else { From aef2c823e5c0dba4bdbc6d85a3483de614792be5 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 4 Aug 2023 15:23:38 +0200 Subject: [PATCH 16/22] feat(server): add more package metadata tables --- libarchive/src/archive.rs | 2 +- libarchive/src/write/file.rs | 2 +- server/src/api/mod.rs | 31 +++ server/src/db/entities/mod.rs | 5 + server/src/db/entities/package.rs | 40 ++++ server/src/db/entities/package_conflicts.rs | 33 +++ server/src/db/entities/package_depends.rs | 35 ++++ server/src/db/entities/package_group.rs | 33 +++ server/src/db/entities/package_provides.rs | 33 +++ server/src/db/entities/package_replaces.rs | 33 +++ server/src/db/entities/prelude.rs | 5 + .../m20230730_000001_create_repo_tables.rs | 198 +++++++++++++++++- server/src/repo/manager.rs | 4 + server/src/repo/mod.rs | 16 +- server/src/repo/package.rs | 4 +- 15 files changed, 466 insertions(+), 8 deletions(-) create mode 100644 server/src/db/entities/package_conflicts.rs create mode 100644 server/src/db/entities/package_depends.rs create mode 100644 server/src/db/entities/package_group.rs create mode 100644 server/src/db/entities/package_provides.rs create mode 100644 server/src/db/entities/package_replaces.rs diff --git a/libarchive/src/archive.rs b/libarchive/src/archive.rs index d8f7086..97c9d0a 100644 --- a/libarchive/src/archive.rs +++ b/libarchive/src/archive.rs @@ -39,7 +39,7 @@ pub enum ReadFormat { Zip, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub enum ReadFilter { All, Bzip2, diff --git a/libarchive/src/write/file.rs b/libarchive/src/write/file.rs index 9c7abe2..5d932f1 100644 --- a/libarchive/src/write/file.rs +++ b/libarchive/src/write/file.rs @@ -54,7 +54,7 @@ impl FileWriter { self.handle_mut(), &buf[written] as *const u8 as *const _, buf_len - written, - ) + ) } as isize; // Negative values signal errors diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 011f008..abb67c3 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -6,11 +6,13 @@ use axum::Json; use axum::Router; use sea_orm::entity::EntityTrait; use sea_orm::query::QueryOrder; +use sea_orm::ModelTrait; use sea_orm::PaginatorTrait; use pagination::PaginatedResponse; use crate::db::entities::package; +use crate::db::entities::package_license; use crate::db::entities::repo; pub fn router() -> Router { @@ -18,6 +20,7 @@ pub fn router() -> Router { .route("/repos", get(get_repos)) .route("/repos/:id", get(get_single_repo)) .route("/packages", get(get_packages)) + .route("/packages/:id", get(get_single_package)) } async fn get_repos( @@ -57,3 +60,31 @@ async fn get_packages( Ok(Json(pagination.res(pkgs))) } + +#[derive(serde::Serialize)] +pub struct PackageRes { + #[serde(flatten)] + entry: package::Model, + licenses: Vec, +} + +async fn get_single_package( + State(global): State, + Path(id): Path, +) -> crate::Result> { + let entry = package::Entity::find_by_id(id) + .one(&global.db) + .await? + .ok_or(axum::http::StatusCode::NOT_FOUND)?; + let licenses = entry + .find_related(package_license::Entity) + .all(&global.db) + .await? + .iter() + .map(|e| e.value.clone()) + .collect(); + + let res = PackageRes { entry, licenses }; + + Ok(Json(res)) +} diff --git a/server/src/db/entities/mod.rs b/server/src/db/entities/mod.rs index e5ad810..828fec2 100644 --- a/server/src/db/entities/mod.rs +++ b/server/src/db/entities/mod.rs @@ -3,5 +3,10 @@ pub mod prelude; pub mod package; +pub mod package_conflicts; +pub mod package_depends; +pub mod package_group; pub mod package_license; +pub mod package_provides; +pub mod package_replaces; pub mod repo; diff --git a/server/src/db/entities/package.rs b/server/src/db/entities/package.rs index 86606bf..c34abb1 100644 --- a/server/src/db/entities/package.rs +++ b/server/src/db/entities/package.rs @@ -26,8 +26,18 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { + #[sea_orm(has_many = "super::package_conflicts::Entity")] + PackageConflicts, + #[sea_orm(has_many = "super::package_depends::Entity")] + PackageDepends, + #[sea_orm(has_many = "super::package_group::Entity")] + PackageGroup, #[sea_orm(has_many = "super::package_license::Entity")] PackageLicense, + #[sea_orm(has_many = "super::package_provides::Entity")] + PackageProvides, + #[sea_orm(has_many = "super::package_replaces::Entity")] + PackageReplaces, #[sea_orm( belongs_to = "super::repo::Entity", from = "Column::RepoId", @@ -38,12 +48,42 @@ pub enum Relation { Repo, } +impl Related for Entity { + fn to() -> RelationDef { + Relation::PackageConflicts.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::PackageDepends.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::PackageGroup.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::PackageLicense.def() } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::PackageProvides.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::PackageReplaces.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::Repo.def() diff --git a/server/src/db/entities/package_conflicts.rs b/server/src/db/entities/package_conflicts.rs new file mode 100644 index 0000000..e9e8ba8 --- /dev/null +++ b/server/src/db/entities/package_conflicts.rs @@ -0,0 +1,33 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "package_conflicts")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub package_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub value: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::package::Entity", + from = "Column::PackageId", + to = "super::package::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Package, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Package.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/entities/package_depends.rs b/server/src/db/entities/package_depends.rs new file mode 100644 index 0000000..b22f717 --- /dev/null +++ b/server/src/db/entities/package_depends.rs @@ -0,0 +1,35 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "package_depends")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub package_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub r#type: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub value: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::package::Entity", + from = "Column::PackageId", + to = "super::package::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Package, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Package.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/entities/package_group.rs b/server/src/db/entities/package_group.rs new file mode 100644 index 0000000..61e69f2 --- /dev/null +++ b/server/src/db/entities/package_group.rs @@ -0,0 +1,33 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "package_group")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub package_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub value: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::package::Entity", + from = "Column::PackageId", + to = "super::package::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Package, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Package.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/entities/package_provides.rs b/server/src/db/entities/package_provides.rs new file mode 100644 index 0000000..7fca6ee --- /dev/null +++ b/server/src/db/entities/package_provides.rs @@ -0,0 +1,33 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "package_provides")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub package_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub value: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::package::Entity", + from = "Column::PackageId", + to = "super::package::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Package, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Package.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/entities/package_replaces.rs b/server/src/db/entities/package_replaces.rs new file mode 100644 index 0000000..0946b2d --- /dev/null +++ b/server/src/db/entities/package_replaces.rs @@ -0,0 +1,33 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "package_replaces")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub package_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub value: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::package::Entity", + from = "Column::PackageId", + to = "super::package::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Package, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Package.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/entities/prelude.rs b/server/src/db/entities/prelude.rs index 1f8176c..9eae171 100644 --- a/server/src/db/entities/prelude.rs +++ b/server/src/db/entities/prelude.rs @@ -1,5 +1,10 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 pub use super::package::Entity as Package; +pub use super::package_conflicts::Entity as PackageConflicts; +pub use super::package_depends::Entity as PackageDepends; +pub use super::package_group::Entity as PackageGroup; pub use super::package_license::Entity as PackageLicense; +pub use super::package_provides::Entity as PackageProvides; +pub use super::package_replaces::Entity as PackageReplaces; pub use super::repo::Entity as Repo; diff --git a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs index df56657..aceae78 100644 --- a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs +++ b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs @@ -90,7 +90,152 @@ impl MigrationTrait for Migration { ) .to_owned(), ) - .await + .await?; + manager + .create_table( + Table::create() + .table(PackageGroup::Table) + .col(ColumnDef::new(PackageGroup::PackageId).integer().not_null()) + .col( + ColumnDef::new(PackageGroup::Value) + .string_len(255) + .not_null(), + ) + .primary_key( + Index::create() + .col(PackageGroup::PackageId) + .col(PackageGroup::Value), + ) + .foreign_key( + ForeignKey::create() + .name("fk-package_group-package_id") + .from(PackageGroup::Table, PackageGroup::PackageId) + .to(Package::Table, Package::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(PackageReplaces::Table) + .col( + ColumnDef::new(PackageReplaces::PackageId) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(PackageReplaces::Value) + .string_len(255) + .not_null(), + ) + .primary_key( + Index::create() + .col(PackageReplaces::PackageId) + .col(PackageReplaces::Value), + ) + .foreign_key( + ForeignKey::create() + .name("fk-package_replaces-package_id") + .from(PackageReplaces::Table, PackageReplaces::PackageId) + .to(Package::Table, Package::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(PackageConflicts::Table) + .col( + ColumnDef::new(PackageConflicts::PackageId) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(PackageConflicts::Value) + .string_len(255) + .not_null(), + ) + .primary_key( + Index::create() + .col(PackageConflicts::PackageId) + .col(PackageConflicts::Value), + ) + .foreign_key( + ForeignKey::create() + .name("fk-package_conflicts-package_id") + .from(PackageConflicts::Table, PackageConflicts::PackageId) + .to(Package::Table, Package::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(PackageProvides::Table) + .col( + ColumnDef::new(PackageProvides::PackageId) + .integer() + .not_null(), + ) + .col( + ColumnDef::new(PackageProvides::Value) + .string_len(255) + .not_null(), + ) + .primary_key( + Index::create() + .col(PackageProvides::PackageId) + .col(PackageProvides::Value), + ) + .foreign_key( + ForeignKey::create() + .name("fk-package_provides-package_id") + .from(PackageProvides::Table, PackageProvides::PackageId) + .to(Package::Table, Package::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(PackageDepends::Table) + .col( + ColumnDef::new(PackageDepends::PackageId) + .integer() + .not_null(), + ) + .col(ColumnDef::new(PackageDepends::Type).integer().not_null()) + .col( + ColumnDef::new(PackageDepends::Value) + .string_len(255) + .not_null(), + ) + .primary_key( + Index::create() + .col(PackageDepends::PackageId) + .col(PackageDepends::Type) + .col(PackageDepends::Value), + ) + .foreign_key( + ForeignKey::create() + .name("fk-package_depends-package_id") + .from(PackageDepends::Table, PackageDepends::PackageId) + .to(Package::Table, Package::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + + Ok(()) } // Define how to rollback this migration: Drop the Bakery table. @@ -98,6 +243,21 @@ impl MigrationTrait for Migration { manager .drop_table(Table::drop().table(PackageLicense::Table).to_owned()) .await?; + manager + .drop_table(Table::drop().table(PackageGroup::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(PackageReplaces::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(PackageConflicts::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(PackageProvides::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(PackageDepends::Table).to_owned()) + .await?; manager .drop_table(Table::drop().table(Package::Table).to_owned()) .await?; @@ -141,3 +301,39 @@ pub enum PackageLicense { PackageId, Value, } + +#[derive(Iden)] +pub enum PackageGroup { + Table, + PackageId, + Value, +} + +#[derive(Iden)] +pub enum PackageReplaces { + Table, + PackageId, + Value, +} + +#[derive(Iden)] +pub enum PackageConflicts { + Table, + PackageId, + Value, +} + +#[derive(Iden)] +pub enum PackageProvides { + Table, + PackageId, + Value, +} + +#[derive(Iden)] +pub enum PackageDepends { + Table, + PackageId, + Type, + Value, +} diff --git a/server/src/repo/manager.rs b/server/src/repo/manager.rs index 5846b5d..c288f30 100644 --- a/server/src/repo/manager.rs +++ b/server/src/repo/manager.rs @@ -136,6 +136,10 @@ impl RepoGroupManager { /// Add a package to the given repo, returning to what architectures the package was added. pub fn add_pkg(&mut self, repo: &str, pkg: &Package) -> io::Result<()> { + // TODO + // * if arch is "any", check if package doesn't already exist for other architecture + // * if arch isn't "any", check if package doesn't already exist for "any" architecture + // We first remove any existing version of the package self.remove_pkg(repo, &pkg.info.arch, &pkg.info.name, false)?; diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index d300ffe..b2c2fec 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -5,7 +5,9 @@ pub use manager::RepoGroupManager; use std::path::PathBuf; -use crate::db::entities::{package as db_package, repo as db_repo}; +use crate::db::entities::{ + package as db_package, package_license as db_package_license, repo as db_repo, +}; use axum::body::Body; use axum::extract::{BodyStream, Path, State}; use axum::http::Request; @@ -159,10 +161,18 @@ async fn post_package_archive( } // Insert the package's data into the database - let mut model: db_package::ActiveModel = pkg.into(); + let mut model: db_package::ActiveModel = pkg.clone().into(); model.repo_id = sea_orm::Set(repo_id); - model.insert(&global.db).await?; + let pkg_entry = model.insert(&global.db).await?; + db_package_license::Entity::insert_many(pkg.info.licenses.iter().map(|s| { + db_package_license::ActiveModel { + package_id: sea_orm::Set(pkg_entry.id), + value: sea_orm::Set(s.to_string()), + } + })) + .exec(&global.db) + .await?; Ok(()) } diff --git a/server/src/repo/package.rs b/server/src/repo/package.rs index ddafbd9..18c69c3 100644 --- a/server/src/repo/package.rs +++ b/server/src/repo/package.rs @@ -11,7 +11,7 @@ use crate::db::entities::package; const IGNORED_FILES: [&str; 5] = [".BUILDINFO", ".INSTALL", ".MTREE", ".PKGINFO", ".CHANGELOG"]; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Package { pub path: PathBuf, pub info: PkgInfo, @@ -19,7 +19,7 @@ pub struct Package { pub compression: ReadFilter, } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct PkgInfo { pub base: String, pub name: String, From 428bf6ec4ccfc89952699bb560481d72e558f57c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 4 Aug 2023 16:52:51 +0200 Subject: [PATCH 17/22] feat(server): start db query abstraction --- server/src/api/mod.rs | 52 ++++++++++++++--------------- server/src/api/pagination.rs | 8 ++++- server/src/cli.rs | 2 +- server/src/db/conn.rs | 61 ++++++++++++++++++++++++++++++++++ server/src/db/mod.rs | 64 ++++++++++++++++++++++++++++++++---- server/src/main.rs | 3 +- server/src/repo/mod.rs | 58 +++++++++++++------------------- 7 files changed, 176 insertions(+), 72 deletions(-) create mode 100644 server/src/db/conn.rs diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index abb67c3..184c1a4 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -4,16 +4,11 @@ use axum::extract::{Path, Query, State}; use axum::routing::get; use axum::Json; use axum::Router; -use sea_orm::entity::EntityTrait; -use sea_orm::query::QueryOrder; use sea_orm::ModelTrait; -use sea_orm::PaginatorTrait; use pagination::PaginatedResponse; -use crate::db::entities::package; -use crate::db::entities::package_license; -use crate::db::entities::repo; +use crate::db; pub fn router() -> Router { Router::new() @@ -26,22 +21,24 @@ pub fn router() -> Router { async fn get_repos( State(global): State, Query(pagination): Query, -) -> crate::Result>> { - let repos = repo::Entity::find() - .order_by_asc(repo::Column::Id) - .paginate(&global.db, pagination.per_page.unwrap_or(25)) - .fetch_page(pagination.page.unwrap_or(1) - 1) +) -> crate::Result>> { + let (total_pages, repos) = global + .db + .repos( + pagination.per_page.unwrap_or(25), + pagination.page.unwrap_or(1) - 1, + ) .await?; - - Ok(Json(pagination.res(repos))) + Ok(Json(pagination.res(total_pages, repos))) } async fn get_single_repo( State(global): State, Path(id): Path, -) -> crate::Result> { - let repo = repo::Entity::find_by_id(id) - .one(&global.db) +) -> crate::Result> { + let repo = global + .db + .repo(id) .await? .ok_or(axum::http::StatusCode::NOT_FOUND)?; @@ -51,20 +48,22 @@ async fn get_single_repo( async fn get_packages( State(global): State, Query(pagination): Query, -) -> crate::Result>> { - let pkgs = package::Entity::find() - .order_by_asc(package::Column::Id) - .paginate(&global.db, pagination.per_page.unwrap_or(25)) - .fetch_page(pagination.page.unwrap_or(1) - 1) +) -> crate::Result>> { + let (total_pages, pkgs) = global + .db + .packages( + pagination.per_page.unwrap_or(25), + pagination.page.unwrap_or(1) - 1, + ) .await?; - Ok(Json(pagination.res(pkgs))) + Ok(Json(pagination.res(total_pages, pkgs))) } #[derive(serde::Serialize)] pub struct PackageRes { #[serde(flatten)] - entry: package::Model, + entry: db::package::Model, licenses: Vec, } @@ -72,12 +71,13 @@ async fn get_single_package( State(global): State, Path(id): Path, ) -> crate::Result> { - let entry = package::Entity::find_by_id(id) - .one(&global.db) + let entry = global + .db + .package(id) .await? .ok_or(axum::http::StatusCode::NOT_FOUND)?; let licenses = entry - .find_related(package_license::Entity) + .find_related(db::PackageLicense) .all(&global.db) .await? .iter() diff --git a/server/src/api/pagination.rs b/server/src/api/pagination.rs index 376e06c..aa1e5cb 100644 --- a/server/src/api/pagination.rs +++ b/server/src/api/pagination.rs @@ -16,15 +16,21 @@ where { pub page: u64, pub per_page: u64, + pub total_pages: u64, pub count: usize, pub items: Vec, } impl Query { - pub fn res Serialize>(self, items: Vec) -> PaginatedResponse { + pub fn res Serialize>( + self, + total_pages: u64, + items: Vec, + ) -> PaginatedResponse { PaginatedResponse { page: self.page.unwrap_or(DEFAULT_PAGE), per_page: self.per_page.unwrap_or(DEFAULT_PER_PAGE), + total_pages, count: items.len(), items, } diff --git a/server/src/cli.rs b/server/src/cli.rs index 8b160d5..d1647dd 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -65,7 +65,7 @@ impl Cli { debug!("Connecting to database with URL {}", db_url); - let db = crate::db::init(db_url).await?; + let db = crate::db::RieterDb::connect(db_url).await?; // let db = crate::db::init("postgres://rieter:rieter@localhost:5432/rieter") // .await // .unwrap(); diff --git a/server/src/db/conn.rs b/server/src/db/conn.rs new file mode 100644 index 0000000..2756236 --- /dev/null +++ b/server/src/db/conn.rs @@ -0,0 +1,61 @@ +use super::RieterDb; +use sea_orm::{DbBackend, DbErr, ExecResult, QueryResult, Statement}; +use std::{future::Future, pin::Pin}; + +// Allows RieterDb objects to be passed to ORM functions +impl sea_orm::ConnectionTrait for RieterDb { + fn get_database_backend(&self) -> DbBackend { + self.conn.get_database_backend() + } + fn execute<'life0, 'async_trait>( + &'life0 self, + stmt: Statement, + ) -> Pin> + Send + 'async_trait>> + where + Self: 'async_trait, + 'life0: 'async_trait, + { + self.conn.execute(stmt) + } + fn execute_unprepared<'life0, 'life1, 'async_trait>( + &'life0 self, + sql: &'life1 str, + ) -> Pin> + Send + 'async_trait>> + where + Self: 'async_trait, + 'life0: 'async_trait, + 'life1: 'async_trait, + { + self.conn.execute_unprepared(sql) + } + fn query_one<'life0, 'async_trait>( + &'life0 self, + stmt: Statement, + ) -> Pin< + Box< + dyn Future, DbErr>> + + Send + + 'async_trait, + >, + > + where + Self: 'async_trait, + 'life0: 'async_trait, + { + self.conn.query_one(stmt) + } + fn query_all<'life0, 'async_trait>( + &'life0 self, + stmt: Statement, + ) -> Pin< + Box< + dyn Future, DbErr>> + Send + 'async_trait, + >, + > + where + Self: 'async_trait, + 'life0: 'async_trait, + { + self.conn.query_all(stmt) + } +} diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index 4e0e804..02c4284 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -1,17 +1,69 @@ +mod conn; pub mod entities; mod migrator; use migrator::Migrator; +use sea_orm::ColumnTrait; use sea_orm::ConnectOptions; use sea_orm::Database; +use sea_orm::DatabaseConnection; +use sea_orm::EntityTrait; +use sea_orm::PaginatorTrait; +use sea_orm::QueryFilter; +use sea_orm::QueryOrder; use sea_orm_migration::MigratorTrait; -pub async fn init>( - opt: C, -) -> Result { - let db = Database::connect(opt).await?; +pub use entities::prelude::*; +pub use entities::*; - Migrator::up(&db, None).await?; +type Result = std::result::Result; - Ok(db) +#[derive(Clone, Debug)] +pub struct RieterDb { + pub conn: DatabaseConnection, +} + +impl RieterDb { + pub async fn connect>(opt: C) -> Result { + let db = Database::connect(opt).await?; + + Migrator::up(&db, None).await?; + + Ok(Self { conn: db }) + } + + pub async fn repos(&self, per_page: u64, page: u64) -> Result<(u64, Vec)> { + let paginator = Repo::find() + .order_by_asc(repo::Column::Id) + .paginate(&self.conn, per_page); + let repos = paginator.fetch_page(page).await?; + let total_pages = paginator.num_pages().await?; + + Ok((total_pages, repos)) + } + + pub async fn repo(&self, id: i32) -> Result> { + repo::Entity::find_by_id(id).one(&self.conn).await + } + + pub async fn repo_by_name(&self, name: &str) -> Result> { + Repo::find() + .filter(repo::Column::Name.eq(name)) + .one(&self.conn) + .await + } + + pub async fn packages(&self, per_page: u64, page: u64) -> Result<(u64, Vec)> { + let paginator = Package::find() + .order_by_asc(package::Column::Id) + .paginate(&self.conn, per_page); + let packages = paginator.fetch_page(page).await?; + let total_pages = paginator.num_pages().await?; + + Ok((total_pages, packages)) + } + + pub async fn package(&self, id: i32) -> Result> { + package::Entity::find_by_id(id).one(&self.conn).await + } } diff --git a/server/src/main.rs b/server/src/main.rs index c2b0eaf..fc5c110 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -7,7 +7,6 @@ mod repo; use clap::Parser; pub use error::{Result, ServerError}; use repo::RepoGroupManager; -use sea_orm::DatabaseConnection; use std::path::PathBuf; use std::sync::{Arc, RwLock}; @@ -23,7 +22,7 @@ pub struct Config { pub struct Global { config: Config, repo_manager: Arc>, - db: DatabaseConnection, + db: db::RieterDb, } #[tokio::main] diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index b2c2fec..8ab2b89 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -5,9 +5,7 @@ pub use manager::RepoGroupManager; use std::path::PathBuf; -use crate::db::entities::{ - package as db_package, package_license as db_package_license, repo as db_repo, -}; +use crate::db; use axum::body::Body; use axum::extract::{BodyStream, Path, State}; use axum::http::Request; @@ -129,30 +127,27 @@ async fn post_package_archive( tracing::info!("Added '{}' to repository '{}'", pkg.file_name(), repo); // Query the repo for its ID, or create it if it does not already exist - let res = db_repo::Entity::find() - .filter(db_repo::Column::Name.eq(&repo)) - .one(&global.db) - .await?; + let res = global.db.repo_by_name(&repo).await?; let repo_id = if let Some(repo_entity) = res { repo_entity.id } else { - let model = db_repo::ActiveModel { + let model = db::repo::ActiveModel { name: sea_orm::Set(repo.clone()), ..Default::default() }; - db_repo::Entity::insert(model) + db::Repo::insert(model) .exec(&global.db) .await? .last_insert_id }; // If the package already exists in the database, we remove it first - let res = db_package::Entity::find() - .filter(db_package::Column::RepoId.eq(repo_id)) - .filter(db_package::Column::Name.eq(&pkg.info.name)) - .filter(db_package::Column::Arch.eq(&pkg.info.arch)) + let res = db::Package::find() + .filter(db::package::Column::RepoId.eq(repo_id)) + .filter(db::package::Column::Name.eq(&pkg.info.name)) + .filter(db::package::Column::Arch.eq(&pkg.info.arch)) .one(&global.db) .await?; @@ -161,12 +156,12 @@ async fn post_package_archive( } // Insert the package's data into the database - let mut model: db_package::ActiveModel = pkg.clone().into(); + let mut model: db::package::ActiveModel = pkg.clone().into(); model.repo_id = sea_orm::Set(repo_id); let pkg_entry = model.insert(&global.db).await?; - db_package_license::Entity::insert_many(pkg.info.licenses.iter().map(|s| { - db_package_license::ActiveModel { + db::PackageLicense::insert_many(pkg.info.licenses.iter().map(|s| { + db::package_license::ActiveModel { package_id: sea_orm::Set(pkg_entry.id), value: sea_orm::Set(s.to_string()), } @@ -197,10 +192,7 @@ async fn delete_repo( .await??; if repo_removed { - let res = db_repo::Entity::find() - .filter(db_repo::Column::Name.eq(&repo)) - .one(&global.db) - .await?; + let res = global.db.repo_by_name(&repo).await?; if let Some(repo_entry) = res { repo_entry.delete(&global.db).await?; @@ -231,16 +223,13 @@ async fn delete_arch_repo( .await??; if repo_removed { - let res = db_repo::Entity::find() - .filter(db_repo::Column::Name.eq(&repo)) - .one(&global.db) - .await?; + let res = global.db.repo_by_name(&repo).await?; if let Some(repo_entry) = res { // Also remove all packages for that architecture from database - db_package::Entity::delete_many() - .filter(db_package::Column::RepoId.eq(repo_entry.id)) - .filter(db_package::Column::Arch.eq(&arch)) + db::Package::delete_many() + .filter(db::package::Column::RepoId.eq(repo_entry.id)) + .filter(db::package::Column::Arch.eq(&arch)) .exec(&global.db) .await?; } @@ -265,18 +254,15 @@ async fn delete_package( .await??; if let Some((name, version, release, arch)) = res { - let res = db_repo::Entity::find() - .filter(db_repo::Column::Name.eq(&repo)) - .one(&global.db) - .await?; + let res = global.db.repo_by_name(&repo).await?; if let Some(repo_entry) = res { // Also remove entry from database - let res = db_package::Entity::find() - .filter(db_package::Column::RepoId.eq(repo_entry.id)) - .filter(db_package::Column::Name.eq(name)) - .filter(db_package::Column::Version.eq(format!("{}-{}", version, release))) - .filter(db_package::Column::Arch.eq(arch)) + let res = db::Package::find() + .filter(db::package::Column::RepoId.eq(repo_entry.id)) + .filter(db::package::Column::Name.eq(name)) + .filter(db::package::Column::Version.eq(format!("{}-{}", version, release))) + .filter(db::package::Column::Arch.eq(arch)) .one(&global.db) .await?; From 0ff225dddb9fe802b6211f7434337e6bd0ddd155 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 4 Aug 2023 17:33:03 +0200 Subject: [PATCH 18/22] refactor(server): further abstract db --- server/src/db/mod.rs | 62 ++++++++++++++++++++++++++++++++++-------- server/src/repo/mod.rs | 43 +++++++++++------------------ 2 files changed, 67 insertions(+), 38 deletions(-) diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index 02c4284..1dbaba5 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -2,19 +2,14 @@ mod conn; pub mod entities; mod migrator; -use migrator::Migrator; -use sea_orm::ColumnTrait; -use sea_orm::ConnectOptions; -use sea_orm::Database; -use sea_orm::DatabaseConnection; -use sea_orm::EntityTrait; -use sea_orm::PaginatorTrait; -use sea_orm::QueryFilter; -use sea_orm::QueryOrder; +use sea_orm::{ + ColumnTrait, ConnectOptions, Database, DatabaseConnection, DeleteResult, EntityTrait, + InsertResult, NotSet, PaginatorTrait, QueryFilter, QueryOrder, Set, +}; use sea_orm_migration::MigratorTrait; -pub use entities::prelude::*; -pub use entities::*; +pub use entities::{prelude::*, *}; +use migrator::Migrator; type Result = std::result::Result; @@ -53,6 +48,20 @@ impl RieterDb { .await } + pub async fn insert_repo( + &self, + name: &str, + description: Option<&str>, + ) -> Result> { + let model = repo::ActiveModel { + id: NotSet, + name: Set(String::from(name)), + description: Set(description.map(String::from)), + }; + + Repo::insert(model).exec(&self.conn).await + } + pub async fn packages(&self, per_page: u64, page: u64) -> Result<(u64, Vec)> { let paginator = Package::find() .order_by_asc(package::Column::Id) @@ -66,4 +75,35 @@ impl RieterDb { pub async fn package(&self, id: i32) -> Result> { package::Entity::find_by_id(id).one(&self.conn).await } + + pub async fn package_by_fields( + &self, + repo_id: i32, + name: &str, + version: Option<&str>, + arch: &str, + ) -> Result> { + let mut query = Package::find() + .filter(package::Column::RepoId.eq(repo_id)) + .filter(package::Column::Name.eq(name)) + .filter(package::Column::Arch.eq(arch)); + + if let Some(version) = version { + query = query.filter(package::Column::Version.eq(version)); + } + + query.one(&self.conn).await + } + + pub async fn delete_packages_with_arch( + &self, + repo_id: i32, + arch: &str, + ) -> Result { + Package::delete_many() + .filter(package::Column::RepoId.eq(repo_id)) + .filter(package::Column::Arch.eq(arch)) + .exec(&self.conn) + .await + } } diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index 8ab2b89..f87d572 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -14,7 +14,7 @@ use axum::response::IntoResponse; use axum::routing::{delete, post}; use axum::Router; use futures::StreamExt; -use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, ModelTrait, QueryFilter}; +use sea_orm::{ActiveModelTrait, EntityTrait, ModelTrait}; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; use tower::util::ServiceExt; @@ -132,23 +132,13 @@ async fn post_package_archive( let repo_id = if let Some(repo_entity) = res { repo_entity.id } else { - let model = db::repo::ActiveModel { - name: sea_orm::Set(repo.clone()), - ..Default::default() - }; - - db::Repo::insert(model) - .exec(&global.db) - .await? - .last_insert_id + global.db.insert_repo(&repo, None).await?.last_insert_id }; // If the package already exists in the database, we remove it first - let res = db::Package::find() - .filter(db::package::Column::RepoId.eq(repo_id)) - .filter(db::package::Column::Name.eq(&pkg.info.name)) - .filter(db::package::Column::Arch.eq(&pkg.info.arch)) - .one(&global.db) + let res = global + .db + .package_by_fields(repo_id, &pkg.info.name, None, &pkg.info.arch) .await?; if let Some(entry) = res { @@ -226,11 +216,9 @@ async fn delete_arch_repo( let res = global.db.repo_by_name(&repo).await?; if let Some(repo_entry) = res { - // Also remove all packages for that architecture from database - db::Package::delete_many() - .filter(db::package::Column::RepoId.eq(repo_entry.id)) - .filter(db::package::Column::Arch.eq(&arch)) - .exec(&global.db) + global + .db + .delete_packages_with_arch(repo_entry.id, &arch) .await?; } tracing::info!("Removed architecture '{}' from repository '{}'", arch, repo); @@ -257,13 +245,14 @@ async fn delete_package( let res = global.db.repo_by_name(&repo).await?; if let Some(repo_entry) = res { - // Also remove entry from database - let res = db::Package::find() - .filter(db::package::Column::RepoId.eq(repo_entry.id)) - .filter(db::package::Column::Name.eq(name)) - .filter(db::package::Column::Version.eq(format!("{}-{}", version, release))) - .filter(db::package::Column::Arch.eq(arch)) - .one(&global.db) + let res = global + .db + .package_by_fields( + repo_entry.id, + &name, + Some(&format!("{}-{}", version, release)), + &arch, + ) .await?; if let Some(entry) = res { From 2df52320d1b3934b2165aa627bf300b9cc3e2cb5 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 4 Aug 2023 18:40:17 +0200 Subject: [PATCH 19/22] feat(server): store all package info in database --- server/src/db/entities/mod.rs | 1 + server/src/db/entities/package.rs | 8 ++ server/src/db/entities/package_depends.rs | 2 +- server/src/db/entities/package_file.rs | 33 +++++ server/src/db/entities/prelude.rs | 1 + .../m20230730_000001_create_repo_tables.rs | 41 +++++- server/src/db/mod.rs | 118 +++++++++++++++++- server/src/repo/mod.rs | 19 +-- 8 files changed, 203 insertions(+), 20 deletions(-) create mode 100644 server/src/db/entities/package_file.rs diff --git a/server/src/db/entities/mod.rs b/server/src/db/entities/mod.rs index 828fec2..1111e7a 100644 --- a/server/src/db/entities/mod.rs +++ b/server/src/db/entities/mod.rs @@ -5,6 +5,7 @@ pub mod prelude; pub mod package; pub mod package_conflicts; pub mod package_depends; +pub mod package_file; pub mod package_group; pub mod package_license; pub mod package_provides; diff --git a/server/src/db/entities/package.rs b/server/src/db/entities/package.rs index c34abb1..b2e2b0b 100644 --- a/server/src/db/entities/package.rs +++ b/server/src/db/entities/package.rs @@ -30,6 +30,8 @@ pub enum Relation { PackageConflicts, #[sea_orm(has_many = "super::package_depends::Entity")] PackageDepends, + #[sea_orm(has_many = "super::package_file::Entity")] + PackageFile, #[sea_orm(has_many = "super::package_group::Entity")] PackageGroup, #[sea_orm(has_many = "super::package_license::Entity")] @@ -60,6 +62,12 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::PackageFile.def() + } +} + impl Related for Entity { fn to() -> RelationDef { Relation::PackageGroup.def() diff --git a/server/src/db/entities/package_depends.rs b/server/src/db/entities/package_depends.rs index b22f717..7e94374 100644 --- a/server/src/db/entities/package_depends.rs +++ b/server/src/db/entities/package_depends.rs @@ -9,7 +9,7 @@ pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub package_id: i32, #[sea_orm(primary_key, auto_increment = false)] - pub r#type: i32, + pub r#type: crate::db::PackageDepend, #[sea_orm(primary_key, auto_increment = false)] pub value: String, } diff --git a/server/src/db/entities/package_file.rs b/server/src/db/entities/package_file.rs new file mode 100644 index 0000000..6e994e0 --- /dev/null +++ b/server/src/db/entities/package_file.rs @@ -0,0 +1,33 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "package_file")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub package_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub value: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::package::Entity", + from = "Column::PackageId", + to = "super::package::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Package, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Package.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/db/entities/prelude.rs b/server/src/db/entities/prelude.rs index 9eae171..bee503c 100644 --- a/server/src/db/entities/prelude.rs +++ b/server/src/db/entities/prelude.rs @@ -3,6 +3,7 @@ pub use super::package::Entity as Package; pub use super::package_conflicts::Entity as PackageConflicts; pub use super::package_depends::Entity as PackageDepends; +pub use super::package_file::Entity as PackageFile; pub use super::package_group::Entity as PackageGroup; pub use super::package_license::Entity as PackageLicense; pub use super::package_provides::Entity as PackageProvides; diff --git a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs index aceae78..adefe56 100644 --- a/server/src/db/migrator/m20230730_000001_create_repo_tables.rs +++ b/server/src/db/migrator/m20230730_000001_create_repo_tables.rs @@ -212,7 +212,11 @@ impl MigrationTrait for Migration { .integer() .not_null(), ) - .col(ColumnDef::new(PackageDepends::Type).integer().not_null()) + .col( + ColumnDef::new(PackageDepends::Type) + .string_len(6) + .not_null(), + ) .col( ColumnDef::new(PackageDepends::Value) .string_len(255) @@ -234,6 +238,31 @@ impl MigrationTrait for Migration { .to_owned(), ) .await?; + manager + .create_table( + Table::create() + .table(PackageFile::Table) + .col(ColumnDef::new(PackageFile::PackageId).integer().not_null()) + .col( + ColumnDef::new(PackageFile::Value) + .string_len(255) + .not_null(), + ) + .primary_key( + Index::create() + .col(PackageFile::PackageId) + .col(PackageFile::Value), + ) + .foreign_key( + ForeignKey::create() + .name("fk-package_file-package_id") + .from(PackageFile::Table, PackageFile::PackageId) + .to(Package::Table, Package::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; Ok(()) } @@ -258,6 +287,9 @@ impl MigrationTrait for Migration { manager .drop_table(Table::drop().table(PackageDepends::Table).to_owned()) .await?; + manager + .drop_table(Table::drop().table(PackageFile::Table).to_owned()) + .await?; manager .drop_table(Table::drop().table(Package::Table).to_owned()) .await?; @@ -337,3 +369,10 @@ pub enum PackageDepends { Type, Value, } + +#[derive(Iden)] +pub enum PackageFile { + Table, + PackageId, + Value, +} diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index 1dbaba5..bc5204c 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -3,16 +3,31 @@ pub mod entities; mod migrator; use sea_orm::{ - ColumnTrait, ConnectOptions, Database, DatabaseConnection, DeleteResult, EntityTrait, - InsertResult, NotSet, PaginatorTrait, QueryFilter, QueryOrder, Set, + ActiveModelTrait, ColumnTrait, ConnectOptions, Database, DatabaseConnection, DeleteResult, + DeriveActiveEnum, EntityTrait, EnumIter, InsertResult, NotSet, PaginatorTrait, QueryFilter, + QueryOrder, Set, }; use sea_orm_migration::MigratorTrait; +use serde::{Deserialize, Serialize}; pub use entities::{prelude::*, *}; use migrator::Migrator; type Result = std::result::Result; +#[derive(EnumIter, DeriveActiveEnum, Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] +#[sea_orm(rs_type = "String", db_type = "String(Some(6))")] +pub enum PackageDepend { + #[sea_orm(string_value = "depend")] + Depend, + #[sea_orm(string_value = "make")] + Make, + #[sea_orm(string_value = "check")] + Check, + #[sea_orm(string_value = "opt")] + Opt, +} + #[derive(Clone, Debug)] pub struct RieterDb { pub conn: DatabaseConnection, @@ -106,4 +121,103 @@ impl RieterDb { .exec(&self.conn) .await } + + pub async fn insert_package( + &self, + repo_id: i32, + pkg: crate::repo::package::Package, + ) -> Result<()> { + let info = pkg.info; + + let model = package::ActiveModel { + id: NotSet, + repo_id: Set(repo_id), + base: Set(info.base), + name: Set(info.name), + version: Set(info.version), + arch: Set(info.arch), + size: Set(info.size), + c_size: Set(info.csize), + description: Set(info.description), + url: Set(info.url), + build_date: Set(info.build_date), + packager: Set(info.packager), + pgp_sig: Set(info.pgpsig), + pgp_sig_size: Set(info.pgpsigsize), + sha256_sum: Set(info.sha256sum), + }; + + let pkg_entry = model.insert(&self.conn).await?; + + // Insert all the related tables + PackageLicense::insert_many(info.licenses.iter().map(|s| package_license::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageGroup::insert_many(info.groups.iter().map(|s| package_group::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageReplaces::insert_many(info.replaces.iter().map(|s| package_replaces::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageConflicts::insert_many(info.conflicts.iter().map(|s| { + package_conflicts::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + } + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageProvides::insert_many(info.provides.iter().map(|s| package_provides::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + PackageFile::insert_many(pkg.files.iter().map(|s| package_file::ActiveModel { + package_id: Set(pkg_entry.id), + value: Set(s.display().to_string()), + })) + .on_empty_do_nothing() + .exec(self) + .await?; + + let deps = info + .depends + .iter() + .map(|d| (PackageDepend::Depend, d)) + .chain(info.makedepends.iter().map(|d| (PackageDepend::Make, d))) + .chain(info.checkdepends.iter().map(|d| (PackageDepend::Check, d))) + .chain(info.optdepends.iter().map(|d| (PackageDepend::Opt, d))) + .map(|(t, s)| package_depends::ActiveModel { + package_id: Set(pkg_entry.id), + r#type: Set(t), + value: Set(s.to_string()), + }); + + PackageDepends::insert_many(deps) + .on_empty_do_nothing() + .exec(self) + .await?; + + Ok(()) + } } diff --git a/server/src/repo/mod.rs b/server/src/repo/mod.rs index f87d572..ba8fbf7 100644 --- a/server/src/repo/mod.rs +++ b/server/src/repo/mod.rs @@ -1,11 +1,10 @@ mod manager; -mod package; +pub mod package; pub use manager::RepoGroupManager; use std::path::PathBuf; -use crate::db; use axum::body::Body; use axum::extract::{BodyStream, Path, State}; use axum::http::Request; @@ -14,7 +13,7 @@ use axum::response::IntoResponse; use axum::routing::{delete, post}; use axum::Router; use futures::StreamExt; -use sea_orm::{ActiveModelTrait, EntityTrait, ModelTrait}; +use sea_orm::ModelTrait; use std::sync::Arc; use tokio::{fs, io::AsyncWriteExt}; use tower::util::ServiceExt; @@ -145,19 +144,7 @@ async fn post_package_archive( entry.delete(&global.db).await?; } - // Insert the package's data into the database - let mut model: db::package::ActiveModel = pkg.clone().into(); - model.repo_id = sea_orm::Set(repo_id); - - let pkg_entry = model.insert(&global.db).await?; - db::PackageLicense::insert_many(pkg.info.licenses.iter().map(|s| { - db::package_license::ActiveModel { - package_id: sea_orm::Set(pkg_entry.id), - value: sea_orm::Set(s.to_string()), - } - })) - .exec(&global.db) - .await?; + global.db.insert_package(repo_id, pkg).await?; Ok(()) } From 5c8b7ac3e0276d8f28bab5a1542cb42f9b38d0b2 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 4 Aug 2023 19:08:55 +0200 Subject: [PATCH 20/22] feat(server): serve full package info from api --- server/src/api/mod.rs | 23 ++----------- server/src/db/mod.rs | 75 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 22 deletions(-) diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 184c1a4..cf6f94c 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -4,7 +4,6 @@ use axum::extract::{Path, Query, State}; use axum::routing::get; use axum::Json; use axum::Router; -use sea_orm::ModelTrait; use pagination::PaginatedResponse; @@ -60,31 +59,15 @@ async fn get_packages( Ok(Json(pagination.res(total_pages, pkgs))) } -#[derive(serde::Serialize)] -pub struct PackageRes { - #[serde(flatten)] - entry: db::package::Model, - licenses: Vec, -} - async fn get_single_package( State(global): State, Path(id): Path, -) -> crate::Result> { +) -> crate::Result> { let entry = global .db - .package(id) + .full_package(id) .await? .ok_or(axum::http::StatusCode::NOT_FOUND)?; - let licenses = entry - .find_related(db::PackageLicense) - .all(&global.db) - .await? - .iter() - .map(|e| e.value.clone()) - .collect(); - let res = PackageRes { entry, licenses }; - - Ok(Json(res)) + Ok(Json(entry)) } diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index bc5204c..1da82dd 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -4,8 +4,8 @@ mod migrator; use sea_orm::{ ActiveModelTrait, ColumnTrait, ConnectOptions, Database, DatabaseConnection, DeleteResult, - DeriveActiveEnum, EntityTrait, EnumIter, InsertResult, NotSet, PaginatorTrait, QueryFilter, - QueryOrder, Set, + DeriveActiveEnum, EntityTrait, EnumIter, InsertResult, ModelTrait, NotSet, PaginatorTrait, + QueryFilter, QueryOrder, Set, }; use sea_orm_migration::MigratorTrait; use serde::{Deserialize, Serialize}; @@ -28,6 +28,18 @@ pub enum PackageDepend { Opt, } +#[derive(Serialize)] +pub struct FullPackage { + #[serde(flatten)] + entry: package::Model, + licenses: Vec, + groups: Vec, + replaces: Vec, + provides: Vec, + depends: Vec<(PackageDepend, String)>, + files: Vec, +} + #[derive(Clone, Debug)] pub struct RieterDb { pub conn: DatabaseConnection, @@ -220,4 +232,63 @@ impl RieterDb { Ok(()) } + + pub async fn full_package(&self, id: i32) -> Result> { + if let Some(entry) = self.package(id).await? { + let licenses = entry + .find_related(PackageLicense) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + let groups = entry + .find_related(PackageGroup) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + let replaces = entry + .find_related(PackageReplaces) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + let provides = entry + .find_related(PackageProvides) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + let depends = entry + .find_related(PackageDepends) + .all(self) + .await? + .into_iter() + .map(|e| (e.r#type, e.value)) + .collect(); + let files = entry + .find_related(PackageFile) + .all(self) + .await? + .into_iter() + .map(|e| e.value) + .collect(); + + Ok(Some(FullPackage { + entry, + licenses, + groups, + replaces, + provides, + depends, + files, + })) + } else { + Ok(None) + } + } } From e5be4391783b2e700ee2c7a17ec62a409a4bb446 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 4 Aug 2023 21:35:45 +0200 Subject: [PATCH 21/22] chore: update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cdad22..d9f4871 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,3 +17,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Packages of architecture "any" are part of every architecture's database * Bearer authentication for private routes + * REST API + * Repository & package information available using JSON REST API From 86cd52a0813d034d3d5ab5fbc36e72a75d7fc4c0 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 5 Aug 2023 12:23:15 +0200 Subject: [PATCH 22/22] chore: add Dockerfile & ci for publishing dev images --- .dockerignore | 2 ++ .woodpecker/docker.yml | 20 ++++++++++++++++++++ Dockerfile | 33 ++++++++++++--------------------- docker-compose.yml | 17 +++++++++++++++++ server/src/cli.rs | 16 +++++++++++++--- 5 files changed, 64 insertions(+), 24 deletions(-) create mode 100644 .woodpecker/docker.yml create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore index e3f6cab..94ca2bf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,5 @@ target/ .git/ server/data/ server/test.db/ +Dockerfile +docker-compose.yml diff --git a/.woodpecker/docker.yml b/.woodpecker/docker.yml new file mode 100644 index 0000000..b26625b --- /dev/null +++ b/.woodpecker/docker.yml @@ -0,0 +1,20 @@ +branches: [dev] +platform: 'linux/amd64' +depends_on: + - build + +pipeline: + dev: + image: 'woodpeckerci/plugin-docker-buildx' + secrets: + - 'docker_username' + - 'docker_password' + settings: + registry: 'git.rustybever.be' + repo: 'git.rustybever.be/chewing_bever/rieter' + tags: + - 'dev' + platforms: [ 'linux/amd64' ] + when: + event: push + branch: dev diff --git a/Dockerfile b/Dockerfile index df91d82..88b51a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,21 +4,6 @@ ARG DI_VER=1.2.5 WORKDIR /app -# RUN apk add --no-cache \ -# build-base \ -# curl \ -# make \ -# unzip \ -# pkgconf \ -# openssl openssl-libs-static openssl-dev \ -# libarchive-static libarchive-dev \ -# zlib-static zlib-dev \ -# bzip2-static bzip2-dev \ -# xz-static xz-dev \ -# expat-static expat-dev \ -# zstd-static zstd-dev \ -# lz4-static lz4-dev \ -# acl-static acl-dev RUN apk add --no-cache \ build-base \ curl \ @@ -42,23 +27,29 @@ COPY . . # LIBARCHIVE_LDFLAGS='-lssl -lcrypto -L/lib -lz -lbz2 -llzma -lexpat -lzstd -llz4' # LIBARCHIVE_LDFLAGS='-L/usr/lib -lz -lbz2 -llzma -lexpat -lzstd -llz4 -lsqlite3' +# https://users.rust-lang.org/t/sigsegv-with-program-linked-against-openssl-in-an-alpine-container/52172 +ENV RUSTFLAGS='-C target-feature=-crt-static' + RUN cargo build --release && \ du -h target/release/rieterd && \ readelf -d target/release/rieterd && \ chmod +x target/release/rieterd - # [ "$(readelf -d target/debug/rieterd | grep NEEDED | wc -l)" = 0 ] && \ - # chmod +x target/debug/rieterd FROM alpine:3.18 -WORKDIR /app - RUN apk add --no-cache \ - libarchive + libgcc \ + libarchive \ + openssl COPY --from=builder /app/dumb-init /bin/dumb-init -COPY --from=builder /app/target/debug/rieterd /bin/rieterd +COPY --from=builder /app/target/release/rieterd /bin/rieterd + +ENV RIETER_PKG_DIR=/data/pkgs \ + RIETER_DATA_DIR=/data + +WORKDIR /data ENTRYPOINT ["/bin/dumb-init", "--"] CMD ["/bin/rieterd"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d05d6d0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3.4' + +services: + app: + image: 'rieterd:latest' + ports: + - '8000:8000' + environment: + - 'RIETER_API_KEY=test' + - 'RIETER_DATABASE_URL=postgres://rieter:rieter@db:5432/rieter' + + db: + image: 'postgres:15.3-alpine' + environment: + - 'POSTGRES_DB=rieter' + - 'POSTGRES_USER=rieter' + - 'POSTGRES_PASSWORD=rieter' diff --git a/server/src/cli.rs b/server/src/cli.rs index d1647dd..540f457 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -15,24 +15,34 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; #[command(author, version, about, long_about = None)] pub struct Cli { /// Directory where package archives will be stored + #[arg(env = "RIETER_PKG_DIR")] pub pkg_dir: PathBuf, /// Directory where repository metadata & SQLite database is stored + #[arg(env = "RIETER_DATA_DIR")] pub data_dir: PathBuf, /// API key to authenticate private routes with + #[arg(env = "RIETER_API_KEY")] pub api_key: String, /// Database connection URL; either sqlite:// or postgres://. Defaults to rieter.sqlite in the /// data directory - #[arg(short, long)] + #[arg(short, long, env = "RIETER_DATABASE_URL")] pub database_url: Option, /// Port the server will listen on - #[arg(short, long, value_name = "PORT", default_value_t = 8000)] + #[arg( + short, + long, + value_name = "PORT", + default_value_t = 8000, + env = "RIETER_PORT" + )] pub port: u16, /// Log levels for the tracing #[arg( long, value_name = "LOG_LEVEL", - default_value = "tower_http=debug,rieterd=debug" + default_value = "tower_http=debug,rieterd=debug", + env = "RIETER_LOG" )] pub log: String, }