Merge pull request 'Move over documentation' (#143) from docs-move into dev

Reviewed-on: Chewing_Bever/vieter#143
cli-docs
Jef Roosens 2022-05-01 12:17:39 +02:00
commit d9d7272b44
23 changed files with 656 additions and 13 deletions

14
.gitignore vendored
View File

@ -1,13 +1,13 @@
*.c
data/
/data/
# Build artifacts
vieter
dvieter
pvieter
suvieter
afvieter
vieter.c
/vieter
/dvieter
/pvieter
/suvieter
/afvieter
/vieter.c
# Ignore testing files
*.pkg*

3
.gitmodules vendored 100644
View File

@ -0,0 +1,3 @@
[submodule "docs/themes/hugo-book"]
path = docs/themes/hugo-book
url = https://github.com/alex-shpak/hugo-book

View File

@ -0,0 +1,16 @@
branches: 'dev'
platform: 'linux/amd64'
depends_on:
- 'docker'
skip_clone: true
pipeline:
webhooks:
image: 'curlimages/curl'
secrets:
- 'webhook_app'
- 'webhook_cron'
commands:
- 'curl -XPOST -s --fail $WEBHOOK_APP'
- 'curl -XPOST -s --fail $WEBHOOK_CRON'

View File

@ -0,0 +1,42 @@
platform: 'linux/amd64'
branches:
exclude: [ main ]
pipeline:
docs:
image: 'klakegg/hugo:alpine'
group: 'generate'
commands:
- apk add git
- make docs
- 'cd docs/public && tar czvf ../../docs.tar.gz *'
api-docs:
image: 'chewingbever/vlang:latest'
pull: true
group: 'generate'
commands:
- make api-docs
- 'cd src/_docs && tar czvf ../../api-docs.tar.gz *'
deploy-docs:
image: 'curlimages/curl'
group: 'deploy'
secrets:
- 'site_api_key'
commands:
- 'curl -XPOST --fail -s -H "Authorization: Bearer $SITE_API_KEY" -T docs.tar.gz https://rustybever.be/api/deploy?dir=docs-vieter'
when:
event: push
branch: dev
deploy-api-docs:
image: 'curlimages/curl'
group: 'deploy'
secrets:
- 'site_api_key'
commands:
- 'curl -XPOST --fail -s -H "Authorization: Bearer $SITE_API_KEY" -T api-docs.tar.gz https://rustybever.be/api/deploy?dir=api-docs-vieter'
when:
event: push
branch: dev

View File

@ -49,6 +49,18 @@ run-prod: prod
./pvieter -f vieter.toml server
# =====DOCS=====
.PHONY: docs
docs:
rm -rf 'docs/public'
cd docs && hugo
.PHONY: api-docs
api-docs:
rm -rf '$(SRC_DIR)/_docs'
cd '$(SRC_DIR)' && v doc -all -f html -m -readme .
# =====OTHER=====
.PHONY: lint
lint:
@ -75,12 +87,7 @@ v/v:
.PHONY: clean
clean:
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs'
.PHONY: api-docs
api-docs:
rm -rf '$(SRC_DIR)/_docs'
cd '$(SRC_DIR)' && v doc -all -f html -m -readme .
rm -rf 'data' 'vieter' 'dvieter' 'pvieter' 'vieter.c' 'dvieterctl' 'vieterctl' 'pkg' 'src/vieter' *.pkg.tar.zst 'suvieter' 'afvieter' '$(SRC_DIR)/_docs' 'docs/public'
# =====EXPERIMENTAL=====

2
docs/.gitignore vendored 100644
View File

@ -0,0 +1,2 @@
.hugo_build.lock
/public/

View File

@ -0,0 +1,17 @@
platform: 'linux/amd64'
branches: 'main'
pipeline:
release:
image: 'klakegg/hugo:alpine'
commands:
- apk add git
- hugo
- 'cd public && tar czvf ../public.tar.gz *'
deploy:
image: 'curlimages/curl'
secrets:
- 'api_key'
commands:
- 'curl -XPOST --fail -s -H "Authorization: Bearer $API_KEY" -T public.tar.gz https://rustybever.be/api/deploy?dir=docs'

9
docs/LICENSE 100644
View File

@ -0,0 +1,9 @@
MIT License
Copyright (c) <year> <copyright holders>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

3
docs/README.md 100644
View File

@ -0,0 +1,3 @@
# docs
Repository containing docs for various personal projects I've made.

View File

@ -0,0 +1,6 @@
---
title: "{{ replace .Name "-" " " | title }}"
date: {{ .Date }}
draft: true
---

108
docs/config.toml 100644
View File

@ -0,0 +1,108 @@
# hugo server --minify --themesDir ... --baseURL=http://0.0.0.0:1313/theme/hugo-book/
baseURL = 'https://rustybever.be/docs/vieter/'
title = 'The Rusty Bever - Docs'
theme = 'hugo-book'
# Book configuration
disablePathToLower = true
enableGitInfo = true
# Needed for mermaid/katex shortcodes
[markup]
[markup.goldmark.renderer]
unsafe = true
[markup.tableOfContents]
startLevel = 1
# Multi-lingual mode config
# There are different options to translate files
# See https://gohugo.io/content-management/multilingual/#translation-by-filename
# And https://gohugo.io/content-management/multilingual/#translation-by-content-directory
[languages]
[languages.en]
languageName = 'English'
contentDir = 'content'
weight = 1
[menu]
# [[menu.before]]
[[menu.after]]
name = "Source"
url = "https://git.rustybever.be/Chewing_Bever/docs"
weight = 10
[[menu.after]]
name = "Hugo Theme"
url = "https://github.com/alex-shpak/hugo-book"
weight = 20
[params]
# (Optional, default light) Sets color theme: light, dark or auto.
# Theme 'auto' switches between dark and light modes based on browser/os preferences
BookTheme = 'auto'
# (Optional, default true) Controls table of contents visibility on right side of pages.
# Start and end levels can be controlled with markup.tableOfContents setting.
# You can also specify this parameter per page in front matter.
BookToC = true
# (Optional, default none) Set the path to a logo for the book. If the logo is
# /static/logo.png then the path would be logo.png
# BookLogo = 'logo.png'
# (Optional, default none) Set leaf bundle to render as side menu
# When not specified file structure and weights will be used
# BookMenuBundle = '/menu'
# (Optional, default docs) Specify root page to render child pages as menu.
# Page is resoled by .GetPage function: https://gohugo.io/functions/getpage/
# For backward compatibility you can set '*' to render all sections to menu. Acts same as '/'
BookSection = '/'
# Set source repository location.
# Used for 'Last Modified' and 'Edit this page' links.
BookRepo = 'https://git.rustybever.be/Chewing_Bever/docs'
# (Optional, default 'commit') Specifies commit portion of the link to the page's last modified
# commit hash for 'doc' page type.
# Requires 'BookRepo' param.
# Value used to construct a URL consisting of BookRepo/BookCommitPath/<commit-hash>
# Github uses 'commit', Bitbucket uses 'commits'
# BookCommitPath = 'commit'
# Enable "Edit this page" links for 'doc' page type.
# Disabled by default. Uncomment to enable. Requires 'BookRepo' param.
# Edit path must point to root directory of repo.
# BookEditPath = 'edit/main/exampleSite'
# Configure the date format used on the pages
# - In git information
# - In blog posts
BookDateFormat = 'January 2, 2006'
# (Optional, default true) Enables search function with flexsearch,
# Index is built on fly, therefore it might slowdown your website.
# Configuration for indexing can be adjusted in i18n folder per language.
BookSearch = true
# (Optional, default true) Enables comments template on pages
# By default partals/docs/comments.html includes Disqus template
# See https://gohugo.io/content-management/comments/#configure-disqus
# Can be overwritten by same param in page frontmatter
BookComments = false
# /!\ This is an experimental feature, might be removed or changed at any time
# (Optional, experimental, default false) Enables portable links and link checks in markdown pages.
# Portable links meant to work with text editors and let you write markdown without {{< relref >}} shortcode
# Theme will print warning if page referenced in markdown does not exists.
BookPortableLinks = true
# /!\ This is an experimental feature, might be removed or changed at any time
# (Optional, experimental, default false) Enables service worker that caches visited pages and resources for offline use.
BookServiceWorker = true
# /!\ This is an experimental feature, might be removed or changed at any time
# (Optional, experimental, default false) Enables a drop-down menu for translations only if a translation is present.
BookTranslatedOnly = false

View File

@ -0,0 +1,27 @@
# Vieter CLI
I provide a simple CLI tool that currently only allows changing the Git
repository API. Its usage is quite simple.
First, you need to create a file in your home directory called `.vieterrc` with
the following content:
```toml
address = "https://example.com"
api_key = "your-api-key"
```
You can also use a different file or use environment variables, as described in
[Configuration](/configuration).
Now you're ready to use the CLI tool.
## Usage
* `vieter repos list` returns all repositories currently stored in the API.
* `vieter repos add url branch repo arch...` adds the repository with the given
URL, branch, repo & arch to the API.
* `vieter repos remove id` removes the repository with the given ID prefix.
You can always check `vieter -help` or `vieter repos -help` for more
information about the commands.

View File

@ -0,0 +1,62 @@
# Vieter
{{< hint warning >}}
**Important**
Because this project is still in heavy development, this documentation tries to
follow the development branch & not the latest release. This means that the
documentation might not be relevant anymore for the latest release.
{{< /hint >}}
## Overview
Vieter has a few main features:
* It's a simple & lightweight implementation of an Arch repository server
* It allows for uploading of built package archives
* It supports a basic build system to periodically re-build packages & upload
them to the server
{{< hint info >}}
**Note**
While I mention Vieter being an "Arch" repository server, it works with any
distribution that uses Pacman as the package manager. I do recommend using a
base docker image for your distribution if you wish to use the build system as
well.
{{< /hint >}}
### Why?
Vieter is my personal solution for a problem I've been facing for months:
extremely long AUR package build times. I run EndeavourOS on both my laptops,
one of which being a rather old MacBook Air. I really like being a beta-tester
for projects & run development builds for multiple packages (nheko,
newsflash...). The issue with this is that I have to regularly re-build these
packages in order to stay up to date with development & these builds can take a
really long time on the old MacBook. This project is a solution to that
problem: instead of building the packages locally, I can build them
automatically in the cloud & just download them whenever I update my system!
Thanks to this solution, I'm able to shave 10-15 minutes off my update times,
just from not having to compile everything every time there's an update.
Besides this, it's also just really useful to have a repository server that you
control & can upload your own packages to. For example, I package my st
terminal using a CI pipeline & upload it to my repository!
### Why V?
I had been interested in learning V for a couple of months ever since I
stumbled upon it by accident. It looked like a promising language & turned out
to be very fun to use! It's fast & easy to learn, & it's a nice contrast with
my usual Rust-based projects, which tend to get quite complex.
I recommend checking out their [homepage](https://vlang.io/)!
### What's with the name?
Before deciding to write this project in V, I wrote a prototype in Python,
called [Pieter](https://git.rustybever.be/Chewing_Bever/pieter). The name
Pieter came from Pieter Post, the Dutch name for [Postname
Pat](https://en.wikipedia.org/wiki/Postman_Pat). The idea was that the server
"delivered packages", & a good friend of mine suggested the name. When I
decided to switch over to Vieter, I changed the P (for Python) to a V, it
seemed fitting.

View File

@ -0,0 +1,84 @@
# API Reference
All routes that return JSON use the following shape:
```json
{
"message": "some message",
"data": {}
}
```
Here, data can be any JSON object, so it's not guaranteed to be a struct.
### `GET /<repo>/<arch>/<filename>`
This route serves the contents of a specific architecture' repo.
If `<filename>` is one of `<repo>.db`, `<repo>.files`, `<repo>.db.tar.gz` or
`<repo>.files.tar.gz`, it will serve the respective archive file from the
repository.
If `<filename>` contains `.pkg`, it assumes the request to be for a package
archive & will serve that file from the specific arch-repo's package directory.
Finally, if none of the above are true, Vieter assumes it to be request for a
package version's desc file & tries to serve this instead. This functionality
is very useful for the build system for checking whether a package needs to be
rebuilt or not.
### `HEAD /<repo>/<arch>/<filename>`
Behaves the same as the above route, but instead of returning actual data, it
returns either 200 or 404, depending on whether the file exists. This route is
used by the build system to determine whether a package needs to be rebuilt.
### `POST /<repo>/publish`
This route is used to upload packages to a repository. It requires the API
key to be provided using the `X-Api-Key` HTTP header. Vieter will parse the
package's contents & update the repository files accordingely. I find the
easiest way to use this route is using cURL:
```sh
curl -XPOST -T "path-to-package.pkg.tar.zst" -H "X-API-KEY: your-api-key" https://example.com/somerepo/publish
```
Packages are automatically added to the correct arch-repo. If a package type is
`any`, the package is added to the configured `default_arch`, as well as all
already present arch-repos. To prevent unnecessary duplication of package
files, these packages are shared between arch-repos' package directories using
hard links.
{{< hint info >}}
**Note**
Vieter only supports uploading archives compressed using either gzip, zstd or
xz at the moment.
{{< /hint >}}
## API
All API routes require the API key to provided using the `X-Api-Key` header.
Otherwise, they'll return a status code 401.
### `GET /api/repos`
Returns the current list of Git repositories.
### `GET /api/repos/<id>`
Get the information for the Git repo with the given ID.
### `POST /api/repos?<url>&<branch>&<arch>&<repo>`
Adds a new Git repository with the provided URL, Git branch & comma-separated
list of architectures.
### `DELETE /api/repos/<id>`
Deletes the Git repository with the provided ID.
### `PATCH /api/repos/<id>?<url>&<branch>&<arch>&<repo>`
Updates the provided parameters for the repo with the given ID. All arguments
are optional.

View File

@ -0,0 +1,56 @@
# Builder
Vieter supports a basic build system that allows you to build the packages
defined using the Git repositories API by running `vieter build`. For
configuration, see [here](/configuration#builder).
## How it works
The build system works in two stages. First it pulls down the
`archlinux:latest` image from Docker Hub, runs `pacman -Syu` & configures a
non-root build user. It then creates a new Docker image from this container.
This is to prevent each build having to fully update the container's
repositories. After the image has been created, each repository returned by
`/api/repos` is built sequentially by starting up a new container with the
previously created image as a base. Each container goes through the following steps:
1. The repository is cloned
2. `makepkg --nobuild --nodeps` is ran to update the `pkgver` variable inside
the `PKGBUILD` file
3. A HEAD request is sent to the Vieter server to check whether the specific
version of the package is already present. If it is, the container exits.
4. `makepkg` is ran with `MAKEFLAGS="-j\$(nproc)`
5. Each produced package archive is uploaded to the Vieter instance's
repository, as defined in the API for that specific Git repo.
## Cron image
The Vieter Docker image contains crond & a cron config that runs `vieter build`
every night at 3AM. This value is currently hardcoded, but I wish to change
that down the line (work is in progress). There's also some other caveats you
should be aware of, namely that the image should be run as root & that the
healthcheck will always fail, so you might have to disable it. This boils down
to the following docker-compose file:
```yaml
version: '3'
services:
cron:
image: 'chewingbever/vieter:dev'
command: crond -f
user: root
healthcheck:
disable: true
environment:
- 'VIETER_API_KEY=some-key'
- 'VIETER_ADDRESS=https://example.com'
volumes:
- '/var/run/docker.sock:/var/run/docker.sock'
```
Important to note is that the container also requires the host's Docker socket
to be mounted as this is how it spawns the necessary containers, as well as a
change to the container's command.

View File

@ -0,0 +1,64 @@
---
weight: 20
---
# Configuration
All vieter operations by default try to read in the TOML file `~/.vieterrc` for
configuration. The location of this file can be changed by using the `-f` flag.
If the above file doesn't exist or you wish to override some of its settings,
configuration is also possible using environment variables. Every variable in
the config file has a respective environment variable of the following form:
say the variable is called `api_key`, then the respective environment variable
would be `VIETER_API_KEY`. In essence, it's the variable in uppercase prepended
with `VIETER_`.
If a variable is both present in the config file & as an environment variable,
the value in the environment variable is used.
{{< hint info >}}
**Note**
All environment variables can also be provided from a file by appending them
with `_FILE`. This for example allows you to provide the API key from a docker
secrets file.
{{< /hint >}}
## Modes
The vieter binary can run in several "modes", indicated by the first argument
passed to them. Each mode requires a different configuration.
### Server
* `log_level`: defines how much logs to show. Valid values are one of `FATAL`,
`ERROR`, `WARN`, `INFO` or `DEBUG`. Defaults to `WARN`
* `log_file`: log file to write logs to. Defaults to `vieter.log` in the
current directory.
* `pkg_dir`: where Vieter should store the actual package archives.
* `download_dir`: where Vieter should initially download uploaded files.
* `api_key`: the API key to use when authenticating requests.
* `repo_dir`: where Vieter should store the contents of the repository.
* `repos_file`: JSON file where the list of Git repositories is saved
* `default_arch`: architecture to always add packages of arch `any` to.
{{< hint info >}}
**Note**
Because Vieter hard links files between `download_dir` & `pkg_dir`, they need
to be on the same file system.
{{< /hint >}}
### Builder
* `api_key`: the API key to use when authenticating requests.
* `address`: Base your URL of your Vieter instance, e.g. https://example.com
* `base_image`: image to use when building a package. It should be an Archlinux
image. The default if not configured is `archlinux:base-devel`, but this
image only supports arm64. If you require aarch64 support as well, consider
using
[`menci/archlinuxarm:base-devel`](https://hub.docker.com/r/menci/archlinuxarm)
([GH](https://github.com/Menci/docker-archlinuxarm))
### Repos
* `api_key`: the API key to use when authenticating requests.
* `address`: Base your URL of your Vieter instance, e.g. https://example.com

View File

@ -0,0 +1,78 @@
---
weight: 10
---
# Installation
## Docker
Docker is the recommended way to install vieter. The images can be pulled from
[`chewingbever/vieter`](https://hub.docker.com/r/chewingbever/vieter). You can
either pull a release tag (e.g. `chewingbever/vieter:0.1.0-rc1`), or pull the
`chewingbever/vieter:dev` tag. The latter is updated every time a new commit is
pushed to the development branch. This branch will be the most up to date, but
does not give any guarantees about stability, so beware!
The simplest way to run the Docker image is using a plain Docker command:
```sh
docker run \
--rm \
-d \
-v /path/to/data:/data \
-e VIETER_API_KEY=changeme \
-e VIETER_DEFAULT_ARCH=x86_64 \
-p 8000:8000 \
chewingbever/vieter:dev
```
Here, you should change `/path/to/data` to the path on your host where you want
vieter to store its files.
The default configuration will store everything inside the `/data` directory.
Inside the container, the Vieter server runs on port 8000. This port should be
exposed to the public accordingely.
For an overview of how to configure vieter & which environment variables can be
used, see the [Configuration](/configuration) page.
## Binary
On the [releases](https://git.rustybever.be/Chewing_Bever/vieter/releases)
page, you can find statically compiled binaries for all released versions. You
can download the binary for your host's architecture & run it that way.
For more information about configuring the binary, check out the
[Configuration](/configuration) page.
## Building from source
Because the project is still in heavy development, it might be useful to build
from source instead. Luckily, this process is very easy. You'll need make,
libarchive & openssl; all of which should be present on an every-day Arch
install. Then, after cloning the repository, you can use the following commands:
```sh
# Builds the compiler; should usually only be ran once. Vieter compiles using
# the default compiler, but I maintain my own mirror to ensure nothing breaks
# without me knowing.
make v
# Build vieter
# Alternatively, use `make prod` to build the production build.
make
```
{{< hint info >}}
**Note**
My version of the V compiler is also available on my Vieter instance,
https://arch.r8r.be. It's in the `vieter` repository, with the package being
named `vieter-v`. The compiler is available for both x86_64 & aarch64.
{{< /hint >}}
## My Vieter instance
Besides uploading development Docker images, my CI also publishes x86_64 &
aarch64 packages to my personal Vieter instance, https://arch.r8r.be. If you'd
like, you can use this repository as well by adding it to your Pacman
configuration as described [here](/usage#configuring-pacman). Both the
repository & the package are called `vieter`.

View File

@ -0,0 +1,54 @@
---
weight: 30
---
# Usage
## Starting the server
To start a server, either install it using Docker (see
[Installation](/installation)) or run it locally by executing `vieter
server`. See [Configuration](/configuration) for more information about
configuring the binary.
## Multiple repositories
Vieter works with multiple repositories. This means that a single Vieter server
can serve multiple repositories in Pacman. It also automatically divides files
with specific architectures among arch-repos. Arch-repos are the actual
repositories you add to your `/etc/pacman.conf` file. See [Configuring
Pacman](/usage#configuring-pacman) below for more info.
## Adding packages
Using Vieter is currently very simple. If you wish to add a package to Vieter,
build it using makepkg & POST that file to the `/<repo>/publish` endpoint of
your server. This will add the package to the repository. Authentification
requires you to add the API key as the `X-Api-Key` header.
All of this can be combined into a simple cURL call:
```
curl -XPOST -H "X-API-KEY: your-key" -T some-package.pkg.tar.zst https://example.com/somerepo/publish
```
`somerepo` is automatically created if it doesn't exist yet.
## Configuring Pacman
Configuring Pacman to use a Vieter instance is very simple. In your
`/etc/pacman.conf` file, add the following lines:
```
[vieter]
Server = https://example.com/$repo/$arch
SigLevel = Optional
```
Here, you see two important placeholder variables. `$repo` is replaced by the
name within the square brackets, which in this case would be `vieter`. `$arch`
is replaced by the output of `uname -m`. Because Vieter supports multiple
repositories & architectures per repository, using this notation makes sure you
always use the correct endpoint for fetching files.
I recommend placing this below all other repository entries, as the order
decides which repository should be used if there's ever a naming conflict.

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"Target":"book.min.97cfda4f5e3c9fa49a2bf8d401f4ddc0eec576c99cdcf6afbec19173200c37db.css","MediaType":"text/css","Data":{"Integrity":"sha256-l8/aT148n6SaK/jUAfTdwO7Fdsmc3PavvsGRcyAMN9s="}}

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
{"Target":"book.min.97cfda4f5e3c9fa49a2bf8d401f4ddc0eec576c99cdcf6afbec19173200c37db.css","MediaType":"text/css","Data":{"Integrity":"sha256-l8/aT148n6SaK/jUAfTdwO7Fdsmc3PavvsGRcyAMN9s="}}

1
docs/themes/hugo-book vendored 160000

@ -0,0 +1 @@
Subproject commit 4ef38f3bbf5dae9a11a711d2ed1ced9294c6af5f