From bb5643bb036eff0d8ece2a94ad05d14b42dcf901 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 13:45:21 +0200 Subject: [PATCH 1/4] feat: added ability to specify kind of target --- src/client/targets.v | 21 +++++----- src/console/targets/targets.v | 40 ++++++++++++++----- src/db/db.v | 8 +++- .../migrations/003-target-url-type/down.sql | 4 ++ src/db/migrations/003-target-url-type/up.sql | 1 + src/models/targets.v | 9 +++-- 6 files changed, 58 insertions(+), 25 deletions(-) create mode 100644 src/db/migrations/003-target-url-type/down.sql create mode 100644 src/db/migrations/003-target-url-type/up.sql diff --git a/src/client/targets.v b/src/client/targets.v index 86d8c2a..d7cc416 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -40,18 +40,17 @@ pub fn (c &Client) get_target(id int) ?Target { return data.data } +pub struct NewTarget { + kind string + url string + branch string + repo string + arch []string +} + // add_target adds a new target to the server. -pub fn (c &Client) add_target(url string, branch string, repo string, arch []string) ?Response { - mut params := { - 'url': url - 'branch': branch - 'repo': repo - } - - if arch.len > 0 { - params['arch'] = arch.join(',') - } - +pub fn (c &Client) add_target(t NewTarget) ?Response { + params := models.params_from(t) data := c.send_request(Method.post, '/api/v1/targets', params)? return data diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 530184d..71a5c92 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -3,7 +3,7 @@ module targets import cli import vieter.vconf import cron.expression { parse_expression } -import client +import client { NewTarget } import console import models { TargetFilter } @@ -65,14 +65,36 @@ pub fn cmd() cli.Command { }, cli.Command{ name: 'add' - required_args: 3 - usage: 'url branch repo' - description: 'Add a new Git repository target.' + required_args: 2 + usage: 'url repo' + description: "Add a new target with the given URL & target repo." + flags: [ + cli.Flag{ + name: 'kind' + description: "Kind of target to add. Defaults to 'git' if not specified. One of 'git', 'url'." + flag: cli.FlagType.string + default_value: ['git'] + }, + cli.Flag{ + name: 'branch' + description: "Which branch to clone; only applies to kind 'git'." + flag: cli.FlagType.string + } + ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - add(conf, cmd.args[0], cmd.args[1], cmd.args[2])? + kind := cmd.flags.get_string('kind')? + + t := NewTarget{ + kind: cmd.flags.get_string('kind')? + url: cmd.args[0] + repo: cmd.args[1] + branch: cmd.flags.get_string('branch') or { '' } + } + + add(conf, t)? } }, cli.Command{ @@ -171,15 +193,15 @@ pub fn cmd() cli.Command { fn list(conf Config, filter TargetFilter) ? { c := client.new(conf.address, conf.api_key) repos := c.get_targets(filter)? - data := repos.map([it.id.str(), it.url, it.branch, it.repo]) + data := repos.map([it.id.str(), it.kind, it.url, it.repo]) - println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?) + println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) } // add adds a new repository to the server's list. -fn add(conf Config, url string, branch string, repo string) ? { +fn add(conf Config, t &NewTarget) ? { c := client.new(conf.address, conf.api_key) - res := c.add_target(url, branch, repo, [])? + res := c.add_target(t)? println(res.message) } diff --git a/src/db/db.v b/src/db/db.v index 64a57d2..eaf71ab 100644 --- a/src/db/db.v +++ b/src/db/db.v @@ -16,9 +16,13 @@ const ( migrations_up = [ $embed_file('migrations/001-initial/up.sql'), $embed_file('migrations/002-rename-to-targets/up.sql'), + $embed_file('migrations/003-target-url-type/up.sql'), + ] + migrations_down = [ + $embed_file('migrations/001-initial/down.sql'), + $embed_file('migrations/002-rename-to-targets/down.sql'), + $embed_file('migrations/003-target-url-type/down.sql'), ] - migrations_down = [$embed_file('migrations/001-initial/down.sql'), - $embed_file('migrations/002-rename-to-targets/down.sql')] ) // init initializes a database & adds the correct tables. diff --git a/src/db/migrations/003-target-url-type/down.sql b/src/db/migrations/003-target-url-type/down.sql new file mode 100644 index 0000000..9d9b45c --- /dev/null +++ b/src/db/migrations/003-target-url-type/down.sql @@ -0,0 +1,4 @@ +-- I'm not sure whether I should remove any non-git targets here. Keeping them +-- will result in invalid targets, but removing them means losing data. +ALTER TABLE Target DROP COLUMN kind; + diff --git a/src/db/migrations/003-target-url-type/up.sql b/src/db/migrations/003-target-url-type/up.sql new file mode 100644 index 0000000..f6be4f4 --- /dev/null +++ b/src/db/migrations/003-target-url-type/up.sql @@ -0,0 +1 @@ +ALTER TABLE Target ADD COLUMN kind TEXT NOT NULL DEFAULT 'git'; diff --git a/src/models/targets.v b/src/models/targets.v index 537bbc4..b8fc88a 100644 --- a/src/models/targets.v +++ b/src/models/targets.v @@ -15,10 +15,13 @@ pub fn (gra &TargetArch) str() string { pub struct Target { pub mut: id int [primary; sql: serial] - // URL of the Git repository + kind string [nonull] + // If kind is git: URL of the Git repository + // If kind is url: URL to PKGBUILD file url string [nonull] - // Branch of the Git repository to use - branch string [nonull] + // Branch of the Git repository to use; only applicable when kind is git. + // If not provided, the repository is cloned with the default branch. + branch string // Which repo the builder should publish packages to repo string [nonull] // Cron schedule describing how frequently to build the repo. From bd079645090280eecd5e827ddd03220c8f9f53dd Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 13:56:38 +0200 Subject: [PATCH 2/4] feat(api): prevent invalid kind values --- src/client/targets.v | 8 ++++---- src/console/targets/targets.v | 13 ++++++++----- src/models/targets.v | 5 ++++- src/server/api_targets.v | 5 +++++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/client/targets.v b/src/client/targets.v index d7cc416..82c7878 100644 --- a/src/client/targets.v +++ b/src/client/targets.v @@ -41,11 +41,11 @@ pub fn (c &Client) get_target(id int) ?Target { } pub struct NewTarget { - kind string - url string + kind string + url string branch string - repo string - arch []string + repo string + arch []string } // add_target adds a new target to the server. diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index 71a5c92..a9934f1 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -67,7 +67,7 @@ pub fn cmd() cli.Command { name: 'add' required_args: 2 usage: 'url repo' - description: "Add a new target with the given URL & target repo." + description: 'Add a new target with the given URL & target repo.' flags: [ cli.Flag{ name: 'kind' @@ -79,14 +79,12 @@ pub fn cmd() cli.Command { name: 'branch' description: "Which branch to clone; only applies to kind 'git'." flag: cli.FlagType.string - } + }, ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? conf := vconf.load(prefix: 'VIETER_', default_path: config_file)? - kind := cmd.flags.get_string('kind')? - t := NewTarget{ kind: cmd.flags.get_string('kind')? url: cmd.args[0] @@ -152,6 +150,11 @@ pub fn cmd() cli.Command { description: 'Cron schedule for repository.' flag: cli.FlagType.string }, + cli.Flag{ + name: 'kind' + description: 'Kind of target.' + flag: cli.FlagType.string + }, ] execute: fn (cmd cli.Command) ? { config_file := cmd.flags.get_string('config-file')? @@ -195,7 +198,7 @@ fn list(conf Config, filter TargetFilter) ? { repos := c.get_targets(filter)? data := repos.map([it.id.str(), it.kind, it.url, it.repo]) - println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) + println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) } // add adds a new repository to the server's list. diff --git a/src/models/targets.v b/src/models/targets.v index b8fc88a..c8aa535 100644 --- a/src/models/targets.v +++ b/src/models/targets.v @@ -1,5 +1,7 @@ module models +pub const valid_kinds = ['git', 'url'] + pub struct TargetArch { pub: id int [primary; sql: serial] @@ -14,7 +16,7 @@ pub fn (gra &TargetArch) str() string { pub struct Target { pub mut: - id int [primary; sql: serial] + id int [primary; sql: serial] kind string [nonull] // If kind is git: URL of the Git repository // If kind is url: URL to PKGBUILD file @@ -35,6 +37,7 @@ pub mut: pub fn (gr &Target) str() string { mut parts := [ 'id: $gr.id', + 'kind: $gr.kind', 'url: $gr.url', 'branch: $gr.branch', 'repo: $gr.repo', diff --git a/src/server/api_targets.v b/src/server/api_targets.v index f807fe9..3867c94 100644 --- a/src/server/api_targets.v +++ b/src/server/api_targets.v @@ -52,6 +52,11 @@ fn (mut app App) v1_post_target() web.Result { return app.json(http.Status.bad_request, new_response(err.msg())) } + // Ensure someone doesn't submit an invalid kind + if new_repo.kind !in models.valid_kinds { + return app.json(http.Status.bad_request, new_response('Invalid kind.')) + } + app.db.add_target(new_repo) return app.json(http.Status.ok, new_response('Repo added successfully.')) From 8f91c1fde52fd7d3a6a11e4c3fa54e0cc414afd5 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 14:31:34 +0200 Subject: [PATCH 3/4] feat(build): added support for 'url' kind --- .../{build_script.sh => build_script_git.sh} | 4 +- src/build/build_script_git_branch.sh | 20 ++++++++++ src/build/build_script_url.sh | 22 +++++++++++ src/build/shell.v | 39 +++++++++++++++---- src/build/shell_test.v | 31 ++++++++++++++- src/console/targets/targets.v | 5 +-- 6 files changed, 107 insertions(+), 14 deletions(-) rename src/build/{build_script.sh => build_script_git.sh} (87%) create mode 100644 src/build/build_script_git_branch.sh create mode 100644 src/build/build_script_url.sh diff --git a/src/build/build_script.sh b/src/build/build_script_git.sh similarity index 87% rename from src/build/build_script.sh rename to src/build/build_script_git.sh index 29f163e..73e0965 100644 --- a/src/build/build_script.sh +++ b/src/build/build_script_git.sh @@ -4,8 +4,8 @@ echo -e '+ pacman -Syu --needed --noconfirm' pacman -Syu --needed --noconfirm echo -e '+ su builder' su builder -echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo' -git clone --single-branch --depth 1 --branch main https://examplerepo.com repo +echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo' +git clone --single-branch --depth 1 'https://examplerepo.com' repo echo -e '+ cd repo' cd repo echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' diff --git a/src/build/build_script_git_branch.sh b/src/build/build_script_git_branch.sh new file mode 100644 index 0000000..be1ff4f --- /dev/null +++ b/src/build/build_script_git_branch.sh @@ -0,0 +1,20 @@ +echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' +echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf +echo -e '+ pacman -Syu --needed --noconfirm' +pacman -Syu --needed --noconfirm +echo -e '+ su builder' +su builder +echo -e '+ git clone --single-branch --depth 1 --branch main '\''https://examplerepo.com'\'' repo' +git clone --single-branch --depth 1 --branch main 'https://examplerepo.com' repo +echo -e '+ cd repo' +cd repo +echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' +makepkg --nobuild --syncdeps --needed --noconfirm +echo -e '+ source PKGBUILD' +source PKGBUILD +echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' +curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 +echo -e '+ [ "$(id -u)" == 0 ] && exit 0' +[ "$(id -u)" == 0 ] && exit 0 +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/build_script_url.sh b/src/build/build_script_url.sh new file mode 100644 index 0000000..3bc97e1 --- /dev/null +++ b/src/build/build_script_url.sh @@ -0,0 +1,22 @@ +echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' +echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf +echo -e '+ pacman -Syu --needed --noconfirm' +pacman -Syu --needed --noconfirm +echo -e '+ su builder' +su builder +echo -e '+ mkdir repo' +mkdir repo +echo -e '+ curl -o repo/PKGBUILD -L '\''https://examplerepo.com'\''' +curl -o repo/PKGBUILD -L 'https://examplerepo.com' +echo -e '+ cd repo' +cd repo +echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' +makepkg --nobuild --syncdeps --needed --noconfirm +echo -e '+ source PKGBUILD' +source PKGBUILD +echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' +curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 +echo -e '+ [ "$(id -u)" == 0 ] && exit 0' +[ "$(id -u)" == 0 ] && exit 0 +echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' +MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done diff --git a/src/build/shell.v b/src/build/shell.v index c5944bf..e573d53 100644 --- a/src/build/shell.v +++ b/src/build/shell.v @@ -23,20 +23,45 @@ pub fn echo_commands(cmds []string) []string { } // create_build_script generates a shell script that builds a given Target. -fn create_build_script(address string, repo &Target, build_arch string) string { - repo_url := '$address/$repo.repo' +fn create_build_script(address string, target &Target, build_arch string) string { + repo_url := '$address/$target.repo' - commands := echo_commands([ + mut commands := [ // This will later be replaced by a proper setting for changing the // mirrorlist - "echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" + "echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" // We need to update the package list of the repo we just added above. // This should however not pull in a lot of packages as long as the // builder image is rebuilt frequently. 'pacman -Syu --needed --noconfirm', // makepkg can't run as root 'su builder', - 'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', + ] + + commands << match target.kind { + 'git' { + if target.branch == '' { + [ + "git clone --single-branch --depth 1 '$target.url' repo", + ] + } else { + [ + "git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", + ] + } + } + 'url' { + [ + 'mkdir repo', + "curl -o repo/PKGBUILD -L '$target.url'", + ] + } + else { + panic("Invalid kind. This shouldn't be possible.") + } + } + + commands << [ 'cd repo', 'makepkg --nobuild --syncdeps --needed --noconfirm', 'source PKGBUILD', @@ -49,7 +74,7 @@ fn create_build_script(address string, repo &Target, build_arch string) string { // we're in root so we don't proceed. '[ "\$(id -u)" == 0 ] && exit 0', 'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', - ]) + ] - return commands.join('\n') + return echo_commands(commands).join('\n') } diff --git a/src/build/shell_test.v b/src/build/shell_test.v index 8534054..341df88 100644 --- a/src/build/shell_test.v +++ b/src/build/shell_test.v @@ -2,15 +2,42 @@ module build import models { Target } -fn test_create_build_script() { +fn test_create_build_script_git_branch() { target := Target{ id: 1 + kind: 'git' url: 'https://examplerepo.com' branch: 'main' repo: 'vieter' } build_script := create_build_script('https://example.com', target, 'x86_64') - expected := $embed_file('build_script.sh') + expected := $embed_file('build_script_git_branch.sh') + + assert build_script == expected.to_string().trim_space() +} + +fn test_create_build_script_git() { + target := Target{ + id: 1 + kind: 'git' + url: 'https://examplerepo.com' + repo: 'vieter' + } + build_script := create_build_script('https://example.com', target, 'x86_64') + expected := $embed_file('build_script_git.sh') + + assert build_script == expected.to_string().trim_space() +} + +fn test_create_build_script_url() { + target := Target{ + id: 1 + kind: 'url' + url: 'https://examplerepo.com' + repo: 'vieter' + } + build_script := create_build_script('https://example.com', target, 'x86_64') + expected := $embed_file('build_script_url.sh') assert build_script == expected.to_string().trim_space() } diff --git a/src/console/targets/targets.v b/src/console/targets/targets.v index a9934f1..ce1cd14 100644 --- a/src/console/targets/targets.v +++ b/src/console/targets/targets.v @@ -123,11 +123,11 @@ pub fn cmd() cli.Command { name: 'edit' required_args: 1 usage: 'id' - description: 'Edit the Git repository target that matches the given id.' + description: 'Edit the target that matches the given id.' flags: [ cli.Flag{ name: 'url' - description: 'URL of the Git repository.' + description: 'URL value. Meaning depends on kind of target.' flag: cli.FlagType.string }, cli.Flag{ @@ -211,7 +211,6 @@ fn add(conf Config, t &NewTarget) ? { // remove removes a repository from the server's list. fn remove(conf Config, id string) ? { - // id, _ := get_repo_by_prefix(conf, id_prefix) ? id_int := id.int() if id_int != 0 { From 449656eb97a016fdf5fc115414fc58d3c4445012 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Fri, 17 Jun 2022 14:52:59 +0200 Subject: [PATCH 4/4] docs: updated to new 'kind' field --- CHANGELOG.md | 6 ++++++ docs/api/source/includes/_targets.md | 4 ++++ docs/content/usage/builds/_index.md | 16 ++++++++-------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d01409a..eee1f9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Server port can now be configured +* Targets now have a 'kind' field describing whether it's a Git repository or a + URL to a PKGBUILD +* Targets with kind 'url' can provide a direct URL to a PKGBUILD instead of + providing a Git repository ### Changed * Moved all API routes under `/v1` namespace * Renamed `vieter repos` to `vieter targets` * Renamed `/api/v1/repos` namespace to `/api/v1/targets` +* Branch name for 'git' targets is now optional; if not provided, the + repository will be cloned with the default branch ### Removed diff --git a/docs/api/source/includes/_targets.md b/docs/api/source/includes/_targets.md index f7ea21e..c7061c8 100644 --- a/docs/api/source/includes/_targets.md +++ b/docs/api/source/includes/_targets.md @@ -24,6 +24,7 @@ curl \ "data": [ { "id": 1, + "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", "repo": "bur", @@ -69,6 +70,7 @@ curl \ "message": "", "data": { "id": 1, + "kind": "git", "url": "https://aur.archlinux.org/discord-ptb.git", "branch": "master", "repo": "bur", @@ -108,6 +110,7 @@ Create a new target with the given data. Parameter | Description --------- | ----------- +kind | Kind of target to add; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. repo | Vieter repository to publish built packages to. @@ -132,6 +135,7 @@ id | id of target to modify Parameter | Description --------- | ----------- +kind | Kind of target; one of 'git', 'url'. url | URL of the Git repository. branch | Branch of the Git repository. repo | Vieter repository to publish built packages to. diff --git a/docs/content/usage/builds/_index.md b/docs/content/usage/builds/_index.md index 7babb06..e6c0b1c 100644 --- a/docs/content/usage/builds/_index.md +++ b/docs/content/usage/builds/_index.md @@ -20,24 +20,24 @@ pages](https://rustybever.be/man/vieter/vieter-targets.1.html) describe this in greater detail, but the basic usage is as follows: ``` -vieter targets add some-url some-branch some-repository +vieter targets add some-url some-repository ``` Here, `some-url` is the URL of the Git repository containing the PKGBUILD. This URL is passed to `git clone`, meaning the repository should be public. Vieter expects the same format as an AUR Git repository, so you can directly use AUR -URLs here. +URLs here. Alternatively, you can also provide the URL to a PKGBUILD file +instead. See +[vieter-targets-add(1)](https://rustybever.be/man/vieter/vieter-targets-add.1.html) +for more information. -`some-branch` is the branch of the Git repository the build should check out. -If you're using an AUR package, this should be `master`. - -Finally, `some-repo` is the repository to which the built package archives -should be published. +`some-repo` is the repository to which the built package archives should be +published. The above command intentionally leaves out a few parameters to make the CLI more useable. For information on how to modify all parameters using the CLI, see -[vieter-targets-edit(1)](https://rustybever.be/man/vieter/vieter-targets-edit.1.html). +[vieter-targets(1)](https://rustybever.be/man/vieter/vieter-targets.1.html). ## Reading logs