.woodpecker/.image.yml

@@ -19,3 +19,6 @@ pipeline:
     when:
       event: pull_request
       branch: dev
+
+depends_on:
+  - build

.woodpecker/.publish.yml

@@ -1,8 +1,5 @@
 branches: [main, dev]
 platform: linux/amd64
-depends_on:
-  - builder
-  - build
 
 pipeline:
   dev:
@@ -34,3 +31,7 @@ pipeline:
     when:
       event: tag
       branch: main
+
+depends_on:
+  - builder
+  - build

.woodpecker/.release.yml

@@ -1,8 +1,6 @@
 # Yeah so this only works on tags so we'll worry about this later
 platform: linux/amd64
 branches: main
-depends_on:
-  - build
 
 # We need the entire repo in order for the release names to work
 skip_clone: true
@@ -36,3 +34,6 @@ pipeline:
       title: title
     when:
       event: push
+
+depends_on:
+  - build

src/archive.v

@@ -12,9 +12,6 @@ fn C.archive_read_new() &C.archive
 // Configure the archive to work with zstd compression
 fn C.archive_read_support_filter_zstd(&C.archive)
 
-// Configure the archive to work with gzip compression
-fn C.archive_read_support_filter_gzip(&C.archive)
-
 // Configure the archive to work with a tarball content
 fn C.archive_read_support_format_tar(&C.archive)
 

src/package.v

@@ -107,24 +107,24 @@ pub fn read_pkg(pkg_path string) ?Pkg {
 
 	a := C.archive_read_new()
 	entry := C.archive_entry_new()
+	mut r := 0
 
 	// Sinds 2020, all newly built Arch packages use zstd
 	C.archive_read_support_filter_zstd(a)
-	C.archive_read_support_filter_gzip(a)
 	// The content should always be a tarball
 	C.archive_read_support_format_tar(a)
 
 	// TODO find out where does this 10240 come from
-	r := C.archive_read_open_filename(a, &char(pkg_path.str), 10240)
+	r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240)
+	defer {
+		C.archive_read_free(a)
+	}
 
 	if r != C.ARCHIVE_OK {
 		return error('Failed to open package.')
 	}
 
-	defer {
-		C.archive_read_free(a)
-	}
-
+	mut buf := voidptr(0)
 	mut files := []string{}
 	mut pkg_info := PkgInfo{}
 
@@ -142,7 +142,7 @@ pub fn read_pkg(pkg_path string) ?Pkg {
 			size := C.archive_entry_size(entry)
 
 			// TODO can this unsafe block be avoided?
-			buf := unsafe { malloc(size) }
+			buf = unsafe { malloc(size) }
 			defer {
 				unsafe {
 					free(buf)
@@ -150,9 +150,10 @@ pub fn read_pkg(pkg_path string) ?Pkg {
 			}
 			C.archive_read_data(a, buf, size)
 
-			pkg_text := unsafe { buf.vstring_with_len(size).clone() }
-
-			pkg_info = parse_pkg_info_string(pkg_text) ?
+			unsafe {
+				println(cstring_to_vstring(buf))
+			}
+			pkg_info = parse_pkg_info_string(unsafe { cstring_to_vstring(buf) }) ?
 		} else {
 			C.archive_read_data_skip(a)
 		}

src/repo.v

@@ -93,3 +93,53 @@ fn (r &Repo) add(pkg &package.Pkg) ?bool {
 fn (r &Repo) pkg_path(pkg &package.Pkg) string {
 	return os.join_path(r.repo_dir, '$pkg.info.name-$pkg.info.version')
 }
+
+// Re-generate the repo archive files
+fn (r &Repo) sync() ? {
+	lock r.mutex {
+		a := C.archive_write_new()
+		entry := C.archive_entry_new()
+		st := C.stat{}
+		buf := [8192]byte{}
+
+		// This makes the archive a gzip-compressed tarball
+		C.archive_write_add_filter_gzip(a)
+		C.archive_write_set_format_pax_restricted(a)
+
+		repo_path := os.join_path_single(r.repo_dir, 'repo.db')
+
+		C.archive_write_open_filename(a, &char(repo_path.str))
+
+		// Iterate over each directory
+		for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir,
+			it))) {
+			inner_path := os.join_path_single(d, 'desc')
+			actual_path := os.join_path_single(r.repo_dir, inner_path)
+
+			unsafe {
+				C.stat(&char(actual_path.str), &st)
+			}
+
+			C.archive_entry_set_pathname(entry, &char(inner_path.str))
+			C.archive_entry_copy_stat(entry, &st)
+			// C.archive_entry_set_size(entry, st.st_size)
+			// C.archive_entry_set_filetype(entry, C.AE_IFREG)
+			// C.archive_entry_set_perm(entry, 0o644)
+			C.archive_write_header(a, entry)
+
+			fd := C.open(&char(actual_path.str), C.O_RDONLY)
+			mut len := C.read(fd, &buf, sizeof(buf))
+
+			for len > 0 {
+				C.archive_write_data(a, &buf, len)
+				len = C.read(fd, &buf, sizeof(buf))
+			}
+			C.close(fd)
+
+			C.archive_entry_clear(entry)
+		}
+
+		C.archive_write_close(a)
+		C.archive_write_free(a)
+	}
+}

src/repo/sync.v

@@ -1,80 +0,0 @@
-module repo
-
-import os
-
-fn archive_add_entry(archive &C.archive, entry &C.archive_entry, file_path &string, inner_path &string) {
-	st := C.stat{}
-
-	unsafe {
-		C.stat(&char(file_path.str), &st)
-	}
-
-	C.archive_entry_set_pathname(entry, &char(inner_path.str))
-	C.archive_entry_copy_stat(entry, &st)
-	C.archive_write_header(archive, entry)
-
-	mut fd := C.open(&char(file_path.str), C.O_RDONLY)
-	defer {
-		C.close(fd)
-	}
-
-	// Write the file to the archive
-	buf := [8192]byte{}
-	mut len := C.read(fd, &buf, sizeof(buf))
-
-	for len > 0 {
-		C.archive_write_data(archive, &buf, len)
-
-		len = C.read(fd, &buf, sizeof(buf))
-	}
-}
-
-// Re-generate the repo archive files
-fn (r &Repo) sync() ? {
-	// TODO also write files archive
-	lock r.mutex {
-		a_db := C.archive_write_new()
-		a_files := C.archive_write_new()
-
-		entry := C.archive_entry_new()
-
-		// This makes the archive a gzip-compressed tarball
-		C.archive_write_add_filter_gzip(a_db)
-		C.archive_write_set_format_pax_restricted(a_db)
-		C.archive_write_add_filter_gzip(a_files)
-		C.archive_write_set_format_pax_restricted(a_files)
-
-		// TODO add symlink to .tar.gz version
-		db_path := os.join_path_single(r.repo_dir, 'repo.db')
-		files_path := os.join_path_single(r.repo_dir, 'repo.files')
-
-		C.archive_write_open_filename(a_db, &char(db_path.str))
-		C.archive_write_open_filename(a_files, &char(files_path.str))
-
-		// Iterate over each directory
-		for d in os.ls(r.repo_dir) ?.filter(os.is_dir(os.join_path_single(r.repo_dir,
-			it))) {
-			// desc
-			mut inner_path := os.join_path_single(d, 'desc')
-			mut actual_path := os.join_path_single(r.repo_dir, inner_path)
-
-			archive_add_entry(a_db, entry, actual_path, inner_path)
-			archive_add_entry(a_files, entry, actual_path, inner_path)
-
-			C.archive_entry_clear(entry)
-
-			// files
-			inner_path = os.join_path_single(d, 'files')
-			actual_path = os.join_path_single(r.repo_dir, inner_path)
-
-			archive_add_entry(a_files, entry, actual_path, inner_path)
-
-			C.archive_entry_clear(entry)
-		}
-
-		C.archive_write_close(a_db)
-		C.archive_write_free(a_db)
-		C.archive_write_close(a_files)
-		C.archive_write_free(a_files)
-	}
-}

src/routes.v

@@ -76,13 +76,9 @@ fn (mut app App) put_package() web.Result {
 	added := app.repo.add_from_path(pkg_path) or {
 		app.lerror('Error while adding package: $err.msg')
 
-		os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path'.") }
-
 		return app.text('Failed to add package.')
 	}
 	if !added {
-		os.rm(pkg_path) or { app.lerror("Failed to remove download '$pkg_path'.") }
-
 		app.lwarn('Duplicate package.')
 
 		return app.text('File already exists.')

test.py

@@ -1,145 +0,0 @@
-import random
-import tempfile
-import tarfile
-from pathlib import Path
-import uuid
-import argparse
-import asyncio
-import aiohttp
-import sys
-
-
-# A list of words the program can choose from
-WORDS = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf",
-         "hotel", "india", "juliet", "kilo", "lima", "mike", "november",
-         "oscar", "papa", "quebec", "romeo", "sierra", "tango", "uniform",
-         "victor", "whiskey", "xray", "yankee", "zulu"]
-SEED = 2022
-
-
-def random_words(words, min_len, max_len=None):
-    """
-    Returns a random list of words, with a length randomly choosen between
-    min_len and max_len. If max_len is None, it is equal to the length of
-    words.
-    """
-    if max_len is None:
-        max_len = len(words)
-
-    k = random.randint(min_len, max_len)
-
-    return random.choices(words, k=k)
-
-def random_lists(words, n, min_len, max_len=None):
-    return [random_words(words, min_len, max_len) for _ in range(n)]
-
-def create_random_pkginfo(words, name_min_len, name_max_len):
-    """
-    Generates a random .PKGINFO
-    """
-    name = "-".join(random_words(words, name_min_len, name_max_len))
-    ver = "0.1.0"  # doesn't matter what it is anyways
-
-    # TODO add random dependencies (all types)
-
-    data = {
-        "pkgname": name,
-        "pkgbase": name,
-        "pkgver": ver,
-        "arch": "x86_64"
-    }
-
-    return "\n".join(f"{key} = {value}" for key, value in data.items())
-
-def create_random_package(tmpdir, words, pkg_name_min_len, pkg_name_max_len, min_files, max_files, min_filename_len, max_filename_len):
-    """
-    Creates a random, but valid Arch package, using the provided tmpdir. Output
-    is the path to the created package tarball.
-    """
-
-    sub_path = tmpdir / uuid.uuid4().hex
-    sub_path.mkdir()
-
-    tar_path = sub_path / "archive.pkg.tar.gz"
-
-    def remove_prefix(tar_info):
-        tar_info.name = tar_info.name[len(str(sub_path)):]
-
-        return tar_info
-
-    with tarfile.open(tar_path, "w") as tar:
-        # Add random .PKGINFO file
-        pkginfo_file = sub_path / ".PKGINFO"
-        pkginfo_file.write_text(create_random_pkginfo(words, pkg_name_min_len, pkg_name_max_len))
-        tar.add(pkginfo_file, filter=remove_prefix)
-
-        # Create random files
-        file_count = random.randint(min_files, max_files)
-
-        for words in random_lists(words, file_count, min_filename_len, max_filename_len):
-            path = sub_path / 'usr' / ('/'.join(words) + ".txt")
-            path.parent.mkdir(parents=True, exist_ok=True)
-            path.write_text(' '.join(words))
-
-            tar.add(path, filter=remove_prefix)
-
-    return tar_path
-
-
-async def check_output(r):
-    good = {"File already exists.", "Package added successfully."}
-    txt = await r.text()
-
-    return (txt in good, txt)
-
-
-async def upload_random_package(tar_path, sem):
-    async with sem:
-        with open(tar_path, 'rb') as f:
-            async with aiohttp.ClientSession() as s:
-                async with s.post("http://localhost:8000/publish", data=f.read(), headers={"x-api-key": "test"}) as r:
-                    return await check_output(r)
-
-
-async def main():
-    parser = argparse.ArgumentParser(description="Test vieter by uploading random package files.")
-    
-    parser.add_argument("count", help="How many packages to upload.", default=1, type=int)
-    parser.add_argument("-p", "--parallel", help="How many uploads to run in parallel.", default=1, type=int)
-    parser.add_argument("-s", "--seed", help="Seed for the randomizer.", default=SEED, type=int)
-    parser.add_argument("--min-files", help="Minimum amount of files to add to an archive.", default=5, type=int)
-    parser.add_argument("--max-files", help="Max amount of files to add to an archive.", default=10, type=int)
-    parser.add_argument("--min-filename-length", help="Minimum amount of words to use for generating filenames.", default=1, type=int)
-    parser.add_argument("--max-filename-length", help="Max amount of words to use for generating filenames.", default=5, type=int)
-    parser.add_argument("--min-pkg-name-length", help="Minimum amount of words to use for creating package name.", default=1, type=int)
-    parser.add_argument("--max-pkg-name-length", help="Max amount of words to use for creating package name.", default=3, type=int)
-    parser.add_argument("--words", help="Words to use for randomizing.", default=WORDS, type=lambda s: s.split(','))
-    # parser.add_argument("--words", help="Words to use for randomizing.", default=WORDS, type=)
-    # parser.add_argument("-d", "--dir", help="Directory to create ")
-
-    args = parser.parse_args()
-
-    sem = asyncio.BoundedSemaphore(args.parallel)
-    random.seed(args.seed)
-
-
-    with tempfile.TemporaryDirectory() as tmpdirname:
-        tmpdir = Path(tmpdirname)
-
-        # We generate the tars in advance because they're not async anyways
-        print("Generating tarballs...")
-        tars = {
-            create_random_package(tmpdir, args.words, args.min_pkg_name_length, args.max_pkg_name_length, args.min_files, args.max_files, args.min_filename_length, args.max_filename_length)
-            for _ in range(args.count)
-        }
-
-        print("Sending requests...")
-        res = await asyncio.gather(*(upload_random_package(tar, sem) for tar in tars))
-
-        # Generate status report
-        if any(not x[0] for x in res):
-            sys.exit(1)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())