Merge pull request 'Add direct PKGBUILD link as target option' (#254) from Chewing_Bever/vieter:direct-pkgbuild-target into dev
	
		
			
	
		
	
	
		
			
				
	
				ci/woodpecker/push/lint Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/docs Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/arch Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/build Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/man Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/test Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/docker Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/deploy Pipeline was successful
				
					Details
				
			
		
	
				
					
				
			
				
	
				ci/woodpecker/push/lint Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/docs Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/arch Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/build Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/man Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/test Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/docker Pipeline was successful
				
					Details
				
			
		
			
				
	
				ci/woodpecker/push/deploy Pipeline was successful
				
					Details
				
			
		
	Reviewed-on: vieter/vieter#254pull/257/head
						commit
						424b0651e9
					
				|  | @ -10,12 +10,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 | ||||||
| ### Added | ### Added | ||||||
| 
 | 
 | ||||||
| * Server port can now be configured | * Server port can now be configured | ||||||
|  | * Targets now have a 'kind' field describing whether it's a Git repository or a | ||||||
|  |   URL to a PKGBUILD | ||||||
|  | * Targets with kind 'url' can provide a direct URL to a PKGBUILD instead of | ||||||
|  |   providing a Git repository | ||||||
| 
 | 
 | ||||||
| ### Changed | ### Changed | ||||||
| 
 | 
 | ||||||
| * Moved all API routes under `/v1` namespace | * Moved all API routes under `/v1` namespace | ||||||
| * Renamed `vieter repos` to `vieter targets` | * Renamed `vieter repos` to `vieter targets` | ||||||
| * Renamed `/api/v1/repos` namespace to `/api/v1/targets` | * Renamed `/api/v1/repos` namespace to `/api/v1/targets` | ||||||
|  | * Branch name for 'git' targets is now optional; if not provided, the | ||||||
|  |   repository will be cloned with the default branch | ||||||
| 
 | 
 | ||||||
| ### Removed | ### Removed | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -24,6 +24,7 @@ curl \ | ||||||
|   "data": [ |   "data": [ | ||||||
|     { |     { | ||||||
|       "id": 1, |       "id": 1, | ||||||
|  |       "kind": "git", | ||||||
|       "url": "https://aur.archlinux.org/discord-ptb.git", |       "url": "https://aur.archlinux.org/discord-ptb.git", | ||||||
|       "branch": "master", |       "branch": "master", | ||||||
|       "repo": "bur", |       "repo": "bur", | ||||||
|  | @ -69,6 +70,7 @@ curl \ | ||||||
|   "message": "", |   "message": "", | ||||||
|   "data": { |   "data": { | ||||||
|     "id": 1, |     "id": 1, | ||||||
|  |     "kind": "git", | ||||||
|     "url": "https://aur.archlinux.org/discord-ptb.git", |     "url": "https://aur.archlinux.org/discord-ptb.git", | ||||||
|     "branch": "master", |     "branch": "master", | ||||||
|     "repo": "bur", |     "repo": "bur", | ||||||
|  | @ -108,6 +110,7 @@ Create a new target with the given data. | ||||||
| 
 | 
 | ||||||
| Parameter | Description | Parameter | Description | ||||||
| --------- | ----------- | --------- | ----------- | ||||||
|  | kind | Kind of target to add; one of 'git', 'url'. | ||||||
| url | URL of the Git repository. | url | URL of the Git repository. | ||||||
| branch | Branch of the Git repository. | branch | Branch of the Git repository. | ||||||
| repo | Vieter repository to publish built packages to. | repo | Vieter repository to publish built packages to. | ||||||
|  | @ -132,6 +135,7 @@ id | id of target to modify | ||||||
| 
 | 
 | ||||||
| Parameter | Description | Parameter | Description | ||||||
| --------- | ----------- | --------- | ----------- | ||||||
|  | kind | Kind of target; one of 'git', 'url'. | ||||||
| url | URL of the Git repository. | url | URL of the Git repository. | ||||||
| branch | Branch of the Git repository. | branch | Branch of the Git repository. | ||||||
| repo | Vieter repository to publish built packages to. | repo | Vieter repository to publish built packages to. | ||||||
|  |  | ||||||
|  | @ -20,24 +20,24 @@ pages](https://rustybever.be/man/vieter/vieter-targets.1.html) describe this in | ||||||
| greater detail, but the basic usage is as follows: | greater detail, but the basic usage is as follows: | ||||||
| 
 | 
 | ||||||
| ``` | ``` | ||||||
| vieter targets add some-url some-branch some-repository | vieter targets add some-url some-repository | ||||||
| ``` | ``` | ||||||
| 
 | 
 | ||||||
| Here, `some-url` is the URL of the Git repository containing the PKGBUILD. This | Here, `some-url` is the URL of the Git repository containing the PKGBUILD. This | ||||||
| URL is passed to `git clone`, meaning the repository should be public. Vieter | URL is passed to `git clone`, meaning the repository should be public. Vieter | ||||||
| expects the same format as an AUR Git repository, so you can directly use AUR | expects the same format as an AUR Git repository, so you can directly use AUR | ||||||
| URLs here. | URLs here. Alternatively, you can also provide the URL to a PKGBUILD file | ||||||
|  | instead. See | ||||||
|  | [vieter-targets-add(1)](https://rustybever.be/man/vieter/vieter-targets-add.1.html) | ||||||
|  | for more information. | ||||||
| 
 | 
 | ||||||
| `some-branch` is the branch of the Git repository the build should check out. | `some-repo` is the repository to which the built package archives should be | ||||||
| If you're using an AUR package, this should be `master`. | published. | ||||||
| 
 |  | ||||||
| Finally, `some-repo` is the repository to which the built package archives |  | ||||||
| should be published. |  | ||||||
| 
 | 
 | ||||||
| The above command intentionally leaves out a few parameters to make the CLI | The above command intentionally leaves out a few parameters to make the CLI | ||||||
| more useable. For information on how to modify all parameters using the CLI, | more useable. For information on how to modify all parameters using the CLI, | ||||||
| see | see | ||||||
| [vieter-targets-edit(1)](https://rustybever.be/man/vieter/vieter-targets-edit.1.html). | [vieter-targets(1)](https://rustybever.be/man/vieter/vieter-targets.1.html). | ||||||
| 
 | 
 | ||||||
| ## Reading logs | ## Reading logs | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -90,10 +90,10 @@ pub: | ||||||
| 	logs       string | 	logs       string | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // build_repo builds, packages & publishes a given Arch package based on the | // build_target builds, packages & publishes a given Arch package based on the | ||||||
| // provided target. The base image ID should be of an image previously created | // provided target. The base image ID should be of an image previously created | ||||||
| // by create_build_image. It returns the logs of the container. | // by create_build_image. It returns the logs of the container. | ||||||
| pub fn build_repo(address string, api_key string, base_image_id string, repo &Target) ?BuildResult { | pub fn build_target(address string, api_key string, base_image_id string, target &Target) ?BuildResult { | ||||||
| 	mut dd := docker.new_conn()? | 	mut dd := docker.new_conn()? | ||||||
| 
 | 
 | ||||||
| 	defer { | 	defer { | ||||||
|  | @ -101,7 +101,7 @@ pub fn build_repo(address string, api_key string, base_image_id string, repo &Ta | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	build_arch := os.uname().machine | 	build_arch := os.uname().machine | ||||||
| 	build_script := create_build_script(address, repo, build_arch) | 	build_script := create_build_script(address, target, build_arch) | ||||||
| 
 | 
 | ||||||
| 	// We convert the build script into a base64 string, which then gets passed | 	// We convert the build script into a base64 string, which then gets passed | ||||||
| 	// to the container as an env var | 	// to the container as an env var | ||||||
|  |  | ||||||
|  | @ -4,8 +4,8 @@ echo -e '+ pacman -Syu --needed --noconfirm' | ||||||
| pacman -Syu --needed --noconfirm | pacman -Syu --needed --noconfirm | ||||||
| echo -e '+ su builder' | echo -e '+ su builder' | ||||||
| su builder | su builder | ||||||
| echo -e '+ git clone --single-branch --depth 1 --branch main https://examplerepo.com repo' | echo -e '+ git clone --single-branch --depth 1 '\''https://examplerepo.com'\'' repo' | ||||||
| git clone --single-branch --depth 1 --branch main https://examplerepo.com repo | git clone --single-branch --depth 1 'https://examplerepo.com' repo | ||||||
| echo -e '+ cd repo' | echo -e '+ cd repo' | ||||||
| cd repo | cd repo | ||||||
| echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' | echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' | ||||||
|  | @ -0,0 +1,20 @@ | ||||||
|  | echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' | ||||||
|  | echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf | ||||||
|  | echo -e '+ pacman -Syu --needed --noconfirm' | ||||||
|  | pacman -Syu --needed --noconfirm | ||||||
|  | echo -e '+ su builder' | ||||||
|  | su builder | ||||||
|  | echo -e '+ git clone --single-branch --depth 1 --branch main '\''https://examplerepo.com'\'' repo' | ||||||
|  | git clone --single-branch --depth 1 --branch main 'https://examplerepo.com' repo | ||||||
|  | echo -e '+ cd repo' | ||||||
|  | cd repo | ||||||
|  | echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' | ||||||
|  | makepkg --nobuild --syncdeps --needed --noconfirm | ||||||
|  | echo -e '+ source PKGBUILD' | ||||||
|  | source PKGBUILD | ||||||
|  | echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' | ||||||
|  | curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 | ||||||
|  | echo -e '+ [ "$(id -u)" == 0 ] && exit 0' | ||||||
|  | [ "$(id -u)" == 0 ] && exit 0 | ||||||
|  | echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' | ||||||
|  | MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done | ||||||
|  | @ -0,0 +1,22 @@ | ||||||
|  | echo -e '+ echo -e '\''[vieter]\\nServer = https://example.com/$repo/$arch\\nSigLevel = Optional'\'' >> /etc/pacman.conf' | ||||||
|  | echo -e '[vieter]\nServer = https://example.com/$repo/$arch\nSigLevel = Optional' >> /etc/pacman.conf | ||||||
|  | echo -e '+ pacman -Syu --needed --noconfirm' | ||||||
|  | pacman -Syu --needed --noconfirm | ||||||
|  | echo -e '+ su builder' | ||||||
|  | su builder | ||||||
|  | echo -e '+ mkdir repo' | ||||||
|  | mkdir repo | ||||||
|  | echo -e '+ curl -o repo/PKGBUILD -L '\''https://examplerepo.com'\''' | ||||||
|  | curl -o repo/PKGBUILD -L 'https://examplerepo.com' | ||||||
|  | echo -e '+ cd repo' | ||||||
|  | cd repo | ||||||
|  | echo -e '+ makepkg --nobuild --syncdeps --needed --noconfirm' | ||||||
|  | makepkg --nobuild --syncdeps --needed --noconfirm | ||||||
|  | echo -e '+ source PKGBUILD' | ||||||
|  | source PKGBUILD | ||||||
|  | echo -e '+ curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0' | ||||||
|  | curl -s --head --fail https://example.com/vieter/x86_64/$pkgname-$pkgver-$pkgrel && exit 0 | ||||||
|  | echo -e '+ [ "$(id -u)" == 0 ] && exit 0' | ||||||
|  | [ "$(id -u)" == 0 ] && exit 0 | ||||||
|  | echo -e '+ MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done' | ||||||
|  | MAKEFLAGS="-j$(nproc)" makepkg -s --noconfirm --needed && for pkg in $(ls -1 *.pkg*); do curl -XPOST -T "$pkg" -H "X-API-KEY: $API_KEY" https://example.com/vieter/publish; done | ||||||
|  | @ -23,20 +23,45 @@ pub fn echo_commands(cmds []string) []string { | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // create_build_script generates a shell script that builds a given Target. | // create_build_script generates a shell script that builds a given Target. | ||||||
| fn create_build_script(address string, repo &Target, build_arch string) string { | fn create_build_script(address string, target &Target, build_arch string) string { | ||||||
| 	repo_url := '$address/$repo.repo' | 	repo_url := '$address/$target.repo' | ||||||
| 
 | 
 | ||||||
| 	commands := echo_commands([ | 	mut commands := [ | ||||||
| 		// This will later be replaced by a proper setting for changing the | 		// This will later be replaced by a proper setting for changing the | ||||||
| 		// mirrorlist | 		// mirrorlist | ||||||
| 		"echo -e '[$repo.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" | 		"echo -e '[$target.repo]\\nServer = $address/\$repo/\$arch\\nSigLevel = Optional' >> /etc/pacman.conf" | ||||||
| 		// We need to update the package list of the repo we just added above. | 		// We need to update the package list of the repo we just added above. | ||||||
| 		// This should however not pull in a lot of packages as long as the | 		// This should however not pull in a lot of packages as long as the | ||||||
| 		// builder image is rebuilt frequently. | 		// builder image is rebuilt frequently. | ||||||
| 		'pacman -Syu --needed --noconfirm', | 		'pacman -Syu --needed --noconfirm', | ||||||
| 		// makepkg can't run as root | 		// makepkg can't run as root | ||||||
| 		'su builder', | 		'su builder', | ||||||
| 		'git clone --single-branch --depth 1 --branch $repo.branch $repo.url repo', | 	] | ||||||
|  | 
 | ||||||
|  | 	commands << match target.kind { | ||||||
|  | 		'git' { | ||||||
|  | 			if target.branch == '' { | ||||||
|  | 				[ | ||||||
|  | 					"git clone --single-branch --depth 1 '$target.url' repo", | ||||||
|  | 				] | ||||||
|  | 			} else { | ||||||
|  | 				[ | ||||||
|  | 					"git clone --single-branch --depth 1 --branch $target.branch '$target.url' repo", | ||||||
|  | 				] | ||||||
|  | 			} | ||||||
|  | 		} | ||||||
|  | 		'url' { | ||||||
|  | 			[ | ||||||
|  | 				'mkdir repo', | ||||||
|  | 				"curl -o repo/PKGBUILD -L '$target.url'", | ||||||
|  | 			] | ||||||
|  | 		} | ||||||
|  | 		else { | ||||||
|  | 			panic("Invalid kind. This shouldn't be possible.") | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	commands << [ | ||||||
| 		'cd repo', | 		'cd repo', | ||||||
| 		'makepkg --nobuild --syncdeps --needed --noconfirm', | 		'makepkg --nobuild --syncdeps --needed --noconfirm', | ||||||
| 		'source PKGBUILD', | 		'source PKGBUILD', | ||||||
|  | @ -49,7 +74,7 @@ fn create_build_script(address string, repo &Target, build_arch string) string { | ||||||
| 		// we're in root so we don't proceed. | 		// we're in root so we don't proceed. | ||||||
| 		'[ "\$(id -u)" == 0 ] && exit 0', | 		'[ "\$(id -u)" == 0 ] && exit 0', | ||||||
| 		'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', | 		'MAKEFLAGS="-j\$(nproc)" makepkg -s --noconfirm --needed && for pkg in \$(ls -1 *.pkg*); do curl -XPOST -T "\$pkg" -H "X-API-KEY: \$API_KEY" $repo_url/publish; done', | ||||||
| 	]) | 	] | ||||||
| 
 | 
 | ||||||
| 	return commands.join('\n') | 	return echo_commands(commands).join('\n') | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -2,15 +2,42 @@ module build | ||||||
| 
 | 
 | ||||||
| import models { Target } | import models { Target } | ||||||
| 
 | 
 | ||||||
| fn test_create_build_script() { | fn test_create_build_script_git_branch() { | ||||||
| 	target := Target{ | 	target := Target{ | ||||||
| 		id: 1 | 		id: 1 | ||||||
|  | 		kind: 'git' | ||||||
| 		url: 'https://examplerepo.com' | 		url: 'https://examplerepo.com' | ||||||
| 		branch: 'main' | 		branch: 'main' | ||||||
| 		repo: 'vieter' | 		repo: 'vieter' | ||||||
| 	} | 	} | ||||||
| 	build_script := create_build_script('https://example.com', target, 'x86_64') | 	build_script := create_build_script('https://example.com', target, 'x86_64') | ||||||
| 	expected := $embed_file('build_script.sh') | 	expected := $embed_file('build_script_git_branch.sh') | ||||||
|  | 
 | ||||||
|  | 	assert build_script == expected.to_string().trim_space() | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | fn test_create_build_script_git() { | ||||||
|  | 	target := Target{ | ||||||
|  | 		id: 1 | ||||||
|  | 		kind: 'git' | ||||||
|  | 		url: 'https://examplerepo.com' | ||||||
|  | 		repo: 'vieter' | ||||||
|  | 	} | ||||||
|  | 	build_script := create_build_script('https://example.com', target, 'x86_64') | ||||||
|  | 	expected := $embed_file('build_script_git.sh') | ||||||
|  | 
 | ||||||
|  | 	assert build_script == expected.to_string().trim_space() | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | fn test_create_build_script_url() { | ||||||
|  | 	target := Target{ | ||||||
|  | 		id: 1 | ||||||
|  | 		kind: 'url' | ||||||
|  | 		url: 'https://examplerepo.com' | ||||||
|  | 		repo: 'vieter' | ||||||
|  | 	} | ||||||
|  | 	build_script := create_build_script('https://example.com', target, 'x86_64') | ||||||
|  | 	expected := $embed_file('build_script_url.sh') | ||||||
| 
 | 
 | ||||||
| 	assert build_script == expected.to_string().trim_space() | 	assert build_script == expected.to_string().trim_space() | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -40,18 +40,17 @@ pub fn (c &Client) get_target(id int) ?Target { | ||||||
| 	return data.data | 	return data.data | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | pub struct NewTarget { | ||||||
|  | 	kind   string | ||||||
|  | 	url    string | ||||||
|  | 	branch string | ||||||
|  | 	repo   string | ||||||
|  | 	arch   []string | ||||||
|  | } | ||||||
|  | 
 | ||||||
| // add_target adds a new target to the server. | // add_target adds a new target to the server. | ||||||
| pub fn (c &Client) add_target(url string, branch string, repo string, arch []string) ?Response<string> { | pub fn (c &Client) add_target(t NewTarget) ?Response<string> { | ||||||
| 	mut params := { | 	params := models.params_from<NewTarget>(t) | ||||||
| 		'url':    url |  | ||||||
| 		'branch': branch |  | ||||||
| 		'repo':   repo |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if arch.len > 0 { |  | ||||||
| 		params['arch'] = arch.join(',') |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	data := c.send_request<string>(Method.post, '/api/v1/targets', params)? | 	data := c.send_request<string>(Method.post, '/api/v1/targets', params)? | ||||||
| 
 | 
 | ||||||
| 	return data | 	return data | ||||||
|  |  | ||||||
|  | @ -6,9 +6,9 @@ import os | ||||||
| import build | import build | ||||||
| 
 | 
 | ||||||
| // build locally builds the target with the given id. | // build locally builds the target with the given id. | ||||||
| fn build(conf Config, repo_id int) ? { | fn build(conf Config, target_id int) ? { | ||||||
| 	c := client.new(conf.address, conf.api_key) | 	c := client.new(conf.address, conf.api_key) | ||||||
| 	repo := c.get_target(repo_id)? | 	target := c.get_target(target_id)? | ||||||
| 
 | 
 | ||||||
| 	build_arch := os.uname().machine | 	build_arch := os.uname().machine | ||||||
| 
 | 
 | ||||||
|  | @ -16,7 +16,7 @@ fn build(conf Config, repo_id int) ? { | ||||||
| 	image_id := build.create_build_image(conf.base_image)? | 	image_id := build.create_build_image(conf.base_image)? | ||||||
| 
 | 
 | ||||||
| 	println('Running build...') | 	println('Running build...') | ||||||
| 	res := build.build_repo(conf.address, conf.api_key, image_id, repo)? | 	res := build.build_target(conf.address, conf.api_key, image_id, target)? | ||||||
| 
 | 
 | ||||||
| 	println('Removing build image...') | 	println('Removing build image...') | ||||||
| 
 | 
 | ||||||
|  | @ -29,6 +29,6 @@ fn build(conf Config, repo_id int) ? { | ||||||
| 	dd.remove_image(image_id)? | 	dd.remove_image(image_id)? | ||||||
| 
 | 
 | ||||||
| 	println('Uploading logs to Vieter...') | 	println('Uploading logs to Vieter...') | ||||||
| 	c.add_build_log(repo.id, res.start_time, res.end_time, build_arch, res.exit_code, | 	c.add_build_log(target.id, res.start_time, res.end_time, build_arch, res.exit_code, | ||||||
| 		res.logs)? | 		res.logs)? | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -3,7 +3,7 @@ module targets | ||||||
| import cli | import cli | ||||||
| import vieter.vconf | import vieter.vconf | ||||||
| import cron.expression { parse_expression } | import cron.expression { parse_expression } | ||||||
| import client | import client { NewTarget } | ||||||
| import console | import console | ||||||
| import models { TargetFilter } | import models { TargetFilter } | ||||||
| 
 | 
 | ||||||
|  | @ -65,14 +65,34 @@ pub fn cmd() cli.Command { | ||||||
| 			}, | 			}, | ||||||
| 			cli.Command{ | 			cli.Command{ | ||||||
| 				name: 'add' | 				name: 'add' | ||||||
| 				required_args: 3 | 				required_args: 2 | ||||||
| 				usage: 'url branch repo' | 				usage: 'url repo' | ||||||
| 				description: 'Add a new Git repository target.' | 				description: 'Add a new target with the given URL & target repo.' | ||||||
|  | 				flags: [ | ||||||
|  | 					cli.Flag{ | ||||||
|  | 						name: 'kind' | ||||||
|  | 						description: "Kind of target to add. Defaults to 'git' if not specified. One of 'git', 'url'." | ||||||
|  | 						flag: cli.FlagType.string | ||||||
|  | 						default_value: ['git'] | ||||||
|  | 					}, | ||||||
|  | 					cli.Flag{ | ||||||
|  | 						name: 'branch' | ||||||
|  | 						description: "Which branch to clone; only applies to kind 'git'." | ||||||
|  | 						flag: cli.FlagType.string | ||||||
|  | 					}, | ||||||
|  | 				] | ||||||
| 				execute: fn (cmd cli.Command) ? { | 				execute: fn (cmd cli.Command) ? { | ||||||
| 					config_file := cmd.flags.get_string('config-file')? | 					config_file := cmd.flags.get_string('config-file')? | ||||||
| 					conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)? | 					conf := vconf.load<Config>(prefix: 'VIETER_', default_path: config_file)? | ||||||
| 
 | 
 | ||||||
| 					add(conf, cmd.args[0], cmd.args[1], cmd.args[2])? | 					t := NewTarget{ | ||||||
|  | 						kind: cmd.flags.get_string('kind')? | ||||||
|  | 						url: cmd.args[0] | ||||||
|  | 						repo: cmd.args[1] | ||||||
|  | 						branch: cmd.flags.get_string('branch') or { '' } | ||||||
|  | 					} | ||||||
|  | 
 | ||||||
|  | 					add(conf, t)? | ||||||
| 				} | 				} | ||||||
| 			}, | 			}, | ||||||
| 			cli.Command{ | 			cli.Command{ | ||||||
|  | @ -103,11 +123,11 @@ pub fn cmd() cli.Command { | ||||||
| 				name: 'edit' | 				name: 'edit' | ||||||
| 				required_args: 1 | 				required_args: 1 | ||||||
| 				usage: 'id' | 				usage: 'id' | ||||||
| 				description: 'Edit the Git repository target that matches the given id.' | 				description: 'Edit the target that matches the given id.' | ||||||
| 				flags: [ | 				flags: [ | ||||||
| 					cli.Flag{ | 					cli.Flag{ | ||||||
| 						name: 'url' | 						name: 'url' | ||||||
| 						description: 'URL of the Git repository.' | 						description: 'URL value. Meaning depends on kind of target.' | ||||||
| 						flag: cli.FlagType.string | 						flag: cli.FlagType.string | ||||||
| 					}, | 					}, | ||||||
| 					cli.Flag{ | 					cli.Flag{ | ||||||
|  | @ -130,6 +150,11 @@ pub fn cmd() cli.Command { | ||||||
| 						description: 'Cron schedule for repository.' | 						description: 'Cron schedule for repository.' | ||||||
| 						flag: cli.FlagType.string | 						flag: cli.FlagType.string | ||||||
| 					}, | 					}, | ||||||
|  | 					cli.Flag{ | ||||||
|  | 						name: 'kind' | ||||||
|  | 						description: 'Kind of target.' | ||||||
|  | 						flag: cli.FlagType.string | ||||||
|  | 					}, | ||||||
| 				] | 				] | ||||||
| 				execute: fn (cmd cli.Command) ? { | 				execute: fn (cmd cli.Command) ? { | ||||||
| 					config_file := cmd.flags.get_string('config-file')? | 					config_file := cmd.flags.get_string('config-file')? | ||||||
|  | @ -171,22 +196,21 @@ pub fn cmd() cli.Command { | ||||||
| fn list(conf Config, filter TargetFilter) ? { | fn list(conf Config, filter TargetFilter) ? { | ||||||
| 	c := client.new(conf.address, conf.api_key) | 	c := client.new(conf.address, conf.api_key) | ||||||
| 	repos := c.get_targets(filter)? | 	repos := c.get_targets(filter)? | ||||||
| 	data := repos.map([it.id.str(), it.url, it.branch, it.repo]) | 	data := repos.map([it.id.str(), it.kind, it.url, it.repo]) | ||||||
| 
 | 
 | ||||||
| 	println(console.pretty_table(['id', 'url', 'branch', 'repo'], data)?) | 	println(console.pretty_table(['id', 'kind', 'url', 'repo'], data)?) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // add adds a new repository to the server's list. | // add adds a new repository to the server's list. | ||||||
| fn add(conf Config, url string, branch string, repo string) ? { | fn add(conf Config, t &NewTarget) ? { | ||||||
| 	c := client.new(conf.address, conf.api_key) | 	c := client.new(conf.address, conf.api_key) | ||||||
| 	res := c.add_target(url, branch, repo, [])? | 	res := c.add_target(t)? | ||||||
| 
 | 
 | ||||||
| 	println(res.message) | 	println(res.message) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // remove removes a repository from the server's list. | // remove removes a repository from the server's list. | ||||||
| fn remove(conf Config, id string) ? { | fn remove(conf Config, id string) ? { | ||||||
| 	// id, _ := get_repo_by_prefix(conf, id_prefix) ? |  | ||||||
| 	id_int := id.int() | 	id_int := id.int() | ||||||
| 
 | 
 | ||||||
| 	if id_int != 0 { | 	if id_int != 0 { | ||||||
|  |  | ||||||
|  | @ -71,29 +71,31 @@ fn (mut d Daemon) start_build(sb ScheduledBuild) bool { | ||||||
| 	return false | 	return false | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // run_build actually starts the build process for a given repo. | // run_build actually starts the build process for a given target. | ||||||
| fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { | fn (mut d Daemon) run_build(build_index int, sb ScheduledBuild) { | ||||||
| 	d.linfo('started build: $sb.repo.url $sb.repo.branch') | 	d.linfo('started build: $sb.target.url -> $sb.target.repo') | ||||||
| 
 | 
 | ||||||
| 	// 0 means success, 1 means failure | 	// 0 means success, 1 means failure | ||||||
| 	mut status := 0 | 	mut status := 0 | ||||||
| 
 | 
 | ||||||
| 	res := build.build_repo(d.client.address, d.client.api_key, d.builder_images.last(), | 	res := build.build_target(d.client.address, d.client.api_key, d.builder_images.last(), | ||||||
| 		&sb.repo) or { | 		&sb.target) or { | ||||||
| 		d.ldebug('build_repo error: $err.msg()') | 		d.ldebug('build_target error: $err.msg()') | ||||||
| 		status = 1 | 		status = 1 | ||||||
| 
 | 
 | ||||||
| 		build.BuildResult{} | 		build.BuildResult{} | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	if status == 0 { | 	if status == 0 { | ||||||
| 		d.linfo('finished build: $sb.repo.url $sb.repo.branch; uploading logs...') | 		d.linfo('finished build: $sb.target.url -> $sb.target.repo; uploading logs...') | ||||||
| 
 | 
 | ||||||
| 		build_arch := os.uname().machine | 		build_arch := os.uname().machine | ||||||
| 		d.client.add_build_log(sb.repo.id, res.start_time, res.end_time, build_arch, res.exit_code, | 		d.client.add_build_log(sb.target.id, res.start_time, res.end_time, build_arch, | ||||||
| 			res.logs) or { d.lerror('Failed to upload logs for $sb.repo.url $sb.repo.arch') } | 			res.exit_code, res.logs) or { | ||||||
|  | 			d.lerror('Failed to upload logs for build: $sb.target.url -> $sb.target.repo') | ||||||
|  | 		} | ||||||
| 	} else { | 	} else { | ||||||
| 		d.linfo('failed build: $sb.repo.url $sb.repo.branch') | 		d.linfo('an error occured during build: $sb.target.url -> $sb.target.repo') | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	stdatomic.store_u64(&d.atomics[build_index], daemon.build_done) | 	stdatomic.store_u64(&d.atomics[build_index], daemon.build_done) | ||||||
|  |  | ||||||
|  | @ -20,7 +20,7 @@ const ( | ||||||
| 
 | 
 | ||||||
| struct ScheduledBuild { | struct ScheduledBuild { | ||||||
| pub: | pub: | ||||||
| 	repo      Target | 	target    Target | ||||||
| 	timestamp time.Time | 	timestamp time.Time | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | @ -37,9 +37,9 @@ mut: | ||||||
| 	global_schedule         CronExpression | 	global_schedule         CronExpression | ||||||
| 	api_update_frequency    int | 	api_update_frequency    int | ||||||
| 	image_rebuild_frequency int | 	image_rebuild_frequency int | ||||||
| 	// Repos currently loaded from API. | 	// Targets currently loaded from API. | ||||||
| 	repos []Target | 	targets []Target | ||||||
| 	// At what point to update the list of repositories. | 	// At what point to update the list of targets. | ||||||
| 	api_update_timestamp  time.Time | 	api_update_timestamp  time.Time | ||||||
| 	image_build_timestamp time.Time | 	image_build_timestamp time.Time | ||||||
| 	queue                 MinHeap<ScheduledBuild> | 	queue                 MinHeap<ScheduledBuild> | ||||||
|  | @ -51,7 +51,7 @@ mut: | ||||||
| 	logger  shared log.Log | 	logger  shared log.Log | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // init_daemon initializes a new Daemon object. It renews the repositories & | // init_daemon initializes a new Daemon object. It renews the targets & | ||||||
| // populates the build queue for the first time. | // populates the build queue for the first time. | ||||||
| pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { | pub fn init_daemon(logger log.Log, address string, api_key string, base_image string, global_schedule CronExpression, max_concurrent_builds int, api_update_frequency int, image_rebuild_frequency int) ?Daemon { | ||||||
| 	mut d := Daemon{ | 	mut d := Daemon{ | ||||||
|  | @ -65,8 +65,8 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st | ||||||
| 		logger: logger | 		logger: logger | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	// Initialize the repos & queue | 	// Initialize the targets & queue | ||||||
| 	d.renew_repos() | 	d.renew_targets() | ||||||
| 	d.renew_queue() | 	d.renew_queue() | ||||||
| 	if !d.rebuild_base_image() { | 	if !d.rebuild_base_image() { | ||||||
| 		return error('The base image failed to build. The Vieter cron daemon cannot run without an initial builder image.') | 		return error('The base image failed to build. The Vieter cron daemon cannot run without an initial builder image.') | ||||||
|  | @ -76,21 +76,21 @@ pub fn init_daemon(logger log.Log, address string, api_key string, base_image st | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // run starts the actual daemon process. It runs builds when possible & | // run starts the actual daemon process. It runs builds when possible & | ||||||
| // periodically refreshes the list of repositories to ensure we stay in sync. | // periodically refreshes the list of targets to ensure we stay in sync. | ||||||
| pub fn (mut d Daemon) run() { | pub fn (mut d Daemon) run() { | ||||||
| 	for { | 	for { | ||||||
| 		finished_builds := d.clean_finished_builds() | 		finished_builds := d.clean_finished_builds() | ||||||
| 
 | 
 | ||||||
| 		// Update the API's contents if needed & renew the queue | 		// Update the API's contents if needed & renew the queue | ||||||
| 		if time.now() >= d.api_update_timestamp { | 		if time.now() >= d.api_update_timestamp { | ||||||
| 			d.renew_repos() | 			d.renew_targets() | ||||||
| 			d.renew_queue() | 			d.renew_queue() | ||||||
| 		} | 		} | ||||||
| 		// The finished builds should only be rescheduled if the API contents | 		// The finished builds should only be rescheduled if the API contents | ||||||
| 		// haven't been renewed. | 		// haven't been renewed. | ||||||
| 		else { | 		else { | ||||||
| 			for sb in finished_builds { | 			for sb in finished_builds { | ||||||
| 				d.schedule_build(sb.repo) | 				d.schedule_build(sb.target) | ||||||
| 			} | 			} | ||||||
| 		} | 		} | ||||||
| 
 | 
 | ||||||
|  | @ -114,7 +114,7 @@ pub fn (mut d Daemon) run() { | ||||||
| 		// every second to clean up any finished builds & start new ones. | 		// every second to clean up any finished builds & start new ones. | ||||||
| 		mut delay := time.Duration(1 * time.second) | 		mut delay := time.Duration(1 * time.second) | ||||||
| 
 | 
 | ||||||
| 		// Sleep either until we have to refresh the repos or when the next | 		// Sleep either until we have to refresh the targets or when the next | ||||||
| 		// build has to start, with a minimum of 1 second. | 		// build has to start, with a minimum of 1 second. | ||||||
| 		if d.current_build_count() == 0 { | 		if d.current_build_count() == 0 { | ||||||
| 			now := time.now() | 			now := time.now() | ||||||
|  | @ -148,12 +148,13 @@ pub fn (mut d Daemon) run() { | ||||||
| 	} | 	} | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // schedule_build adds the next occurence of the given repo build to the queue. | // schedule_build adds the next occurence of the given targets build to the | ||||||
| fn (mut d Daemon) schedule_build(repo Target) { | // queue. | ||||||
| 	ce := if repo.schedule != '' { | fn (mut d Daemon) schedule_build(target Target) { | ||||||
| 		parse_expression(repo.schedule) or { | 	ce := if target.schedule != '' { | ||||||
|  | 		parse_expression(target.schedule) or { | ||||||
| 			// TODO This shouldn't return an error if the expression is empty. | 			// TODO This shouldn't return an error if the expression is empty. | ||||||
| 			d.lerror("Error while parsing cron expression '$repo.schedule' (id $repo.id): $err.msg()") | 			d.lerror("Error while parsing cron expression '$target.schedule' (id $target.id): $err.msg()") | ||||||
| 
 | 
 | ||||||
| 			d.global_schedule | 			d.global_schedule | ||||||
| 		} | 		} | ||||||
|  | @ -161,41 +162,41 @@ fn (mut d Daemon) schedule_build(repo Target) { | ||||||
| 		d.global_schedule | 		d.global_schedule | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	// A repo that can't be scheduled will just be skipped for now | 	// A target that can't be scheduled will just be skipped for now | ||||||
| 	timestamp := ce.next_from_now() or { | 	timestamp := ce.next_from_now() or { | ||||||
| 		d.lerror("Couldn't calculate next timestamp from '$repo.schedule'; skipping") | 		d.lerror("Couldn't calculate next timestamp from '$target.schedule'; skipping") | ||||||
| 		return | 		return | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	d.queue.insert(ScheduledBuild{ | 	d.queue.insert(ScheduledBuild{ | ||||||
| 		repo: repo | 		target: target | ||||||
| 		timestamp: timestamp | 		timestamp: timestamp | ||||||
| 	}) | 	}) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // renew_repos requests the newest list of Git repos from the server & replaces | // renew_targets requests the newest list of targets from the server & replaces | ||||||
| // the old one. | // the old one. | ||||||
| fn (mut d Daemon) renew_repos() { | fn (mut d Daemon) renew_targets() { | ||||||
| 	d.linfo('Renewing repos...') | 	d.linfo('Renewing targets...') | ||||||
| 
 | 
 | ||||||
| 	mut new_repos := d.client.get_all_targets() or { | 	mut new_targets := d.client.get_all_targets() or { | ||||||
| 		d.lerror('Failed to renew repos. Retrying in ${daemon.api_update_retry_timeout}s...') | 		d.lerror('Failed to renew targets. Retrying in ${daemon.api_update_retry_timeout}s...') | ||||||
| 		d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) | 		d.api_update_timestamp = time.now().add_seconds(daemon.api_update_retry_timeout) | ||||||
| 
 | 
 | ||||||
| 		return | 		return | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	// Filter out any repos that shouldn't run on this architecture | 	// Filter out any targets that shouldn't run on this architecture | ||||||
| 	cur_arch := os.uname().machine | 	cur_arch := os.uname().machine | ||||||
| 	new_repos = new_repos.filter(it.arch.any(it.value == cur_arch)) | 	new_targets = new_targets.filter(it.arch.any(it.value == cur_arch)) | ||||||
| 
 | 
 | ||||||
| 	d.repos = new_repos | 	d.targets = new_targets | ||||||
| 
 | 
 | ||||||
| 	d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) | 	d.api_update_timestamp = time.now().add_seconds(60 * d.api_update_frequency) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // renew_queue replaces the old queue with a new one that reflects the newest | // renew_queue replaces the old queue with a new one that reflects the newest | ||||||
| // values in repos_map. | // values in targets. | ||||||
| fn (mut d Daemon) renew_queue() { | fn (mut d Daemon) renew_queue() { | ||||||
| 	d.linfo('Renewing queue...') | 	d.linfo('Renewing queue...') | ||||||
| 	mut new_queue := MinHeap<ScheduledBuild>{} | 	mut new_queue := MinHeap<ScheduledBuild>{} | ||||||
|  | @ -225,10 +226,10 @@ fn (mut d Daemon) renew_queue() { | ||||||
| 
 | 
 | ||||||
| 	d.queue = new_queue | 	d.queue = new_queue | ||||||
| 
 | 
 | ||||||
| 	// For each repository in repos_map, parse their cron expression (or use | 	// For each target in targets, parse their cron expression (or use the | ||||||
| 	// the default one if not present) & add them to the queue | 	// default one if not present) & add them to the queue | ||||||
| 	for repo in d.repos { | 	for target in d.targets { | ||||||
| 		d.schedule_build(repo) | 		d.schedule_build(target) | ||||||
| 	} | 	} | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -16,9 +16,13 @@ const ( | ||||||
| 	migrations_up   = [ | 	migrations_up   = [ | ||||||
| 		$embed_file('migrations/001-initial/up.sql'), | 		$embed_file('migrations/001-initial/up.sql'), | ||||||
| 		$embed_file('migrations/002-rename-to-targets/up.sql'), | 		$embed_file('migrations/002-rename-to-targets/up.sql'), | ||||||
|  | 		$embed_file('migrations/003-target-url-type/up.sql'), | ||||||
|  | 	] | ||||||
|  | 	migrations_down = [ | ||||||
|  | 		$embed_file('migrations/001-initial/down.sql'), | ||||||
|  | 		$embed_file('migrations/002-rename-to-targets/down.sql'), | ||||||
|  | 		$embed_file('migrations/003-target-url-type/down.sql'), | ||||||
| 	] | 	] | ||||||
| 	migrations_down = [$embed_file('migrations/001-initial/down.sql'), |  | ||||||
| 		$embed_file('migrations/002-rename-to-targets/down.sql')] |  | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| // init initializes a database & adds the correct tables. | // init initializes a database & adds the correct tables. | ||||||
|  |  | ||||||
|  | @ -0,0 +1,4 @@ | ||||||
|  | -- I'm not sure whether I should remove any non-git targets here. Keeping them | ||||||
|  | -- will result in invalid targets, but removing them means losing data. | ||||||
|  | ALTER TABLE Target DROP COLUMN kind; | ||||||
|  | 
 | ||||||
|  | @ -0,0 +1 @@ | ||||||
|  | ALTER TABLE Target ADD COLUMN kind TEXT NOT NULL DEFAULT 'git'; | ||||||
|  | @ -1,5 +1,7 @@ | ||||||
| module models | module models | ||||||
| 
 | 
 | ||||||
|  | pub const valid_kinds = ['git', 'url'] | ||||||
|  | 
 | ||||||
| pub struct TargetArch { | pub struct TargetArch { | ||||||
| pub: | pub: | ||||||
| 	id        int    [primary; sql: serial] | 	id        int    [primary; sql: serial] | ||||||
|  | @ -15,10 +17,13 @@ pub fn (gra &TargetArch) str() string { | ||||||
| pub struct Target { | pub struct Target { | ||||||
| pub mut: | pub mut: | ||||||
| 	id   int    [primary; sql: serial] | 	id   int    [primary; sql: serial] | ||||||
| 	// URL of the Git repository | 	kind string [nonull] | ||||||
|  | 	// If kind is git: URL of the Git repository | ||||||
|  | 	// If kind is url: URL to PKGBUILD file | ||||||
| 	url string [nonull] | 	url string [nonull] | ||||||
| 	// Branch of the Git repository to use | 	// Branch of the Git repository to use; only applicable when kind is git. | ||||||
| 	branch string [nonull] | 	// If not provided, the repository is cloned with the default branch. | ||||||
|  | 	branch string | ||||||
| 	// Which repo the builder should publish packages to | 	// Which repo the builder should publish packages to | ||||||
| 	repo string [nonull] | 	repo string [nonull] | ||||||
| 	// Cron schedule describing how frequently to build the repo. | 	// Cron schedule describing how frequently to build the repo. | ||||||
|  | @ -32,6 +37,7 @@ pub mut: | ||||||
| pub fn (gr &Target) str() string { | pub fn (gr &Target) str() string { | ||||||
| 	mut parts := [ | 	mut parts := [ | ||||||
| 		'id: $gr.id', | 		'id: $gr.id', | ||||||
|  | 		'kind: $gr.kind', | ||||||
| 		'url: $gr.url', | 		'url: $gr.url', | ||||||
| 		'branch: $gr.branch', | 		'branch: $gr.branch', | ||||||
| 		'repo: $gr.repo', | 		'repo: $gr.repo', | ||||||
|  |  | ||||||
|  | @ -52,6 +52,11 @@ fn (mut app App) v1_post_target() web.Result { | ||||||
| 		return app.json(http.Status.bad_request, new_response(err.msg())) | 		return app.json(http.Status.bad_request, new_response(err.msg())) | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
|  | 	// Ensure someone doesn't submit an invalid kind | ||||||
|  | 	if new_repo.kind !in models.valid_kinds { | ||||||
|  | 		return app.json(http.Status.bad_request, new_response('Invalid kind.')) | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
| 	app.db.add_target(new_repo) | 	app.db.add_target(new_repo) | ||||||
| 
 | 
 | ||||||
| 	return app.json(http.Status.ok, new_response('Repo added successfully.')) | 	return app.json(http.Status.ok, new_response('Repo added successfully.')) | ||||||
|  |  | ||||||
		Loading…
	
		Reference in New Issue