diff --git a/src/archive.v b/src/archive.v index bdc18462..3eaad5c3 100644 --- a/src/archive.v +++ b/src/archive.v @@ -12,6 +12,9 @@ fn C.archive_read_new() &C.archive // Configure the archive to work with zstd compression fn C.archive_read_support_filter_zstd(&C.archive) +// Configure the archive to work with gzip compression +fn C.archive_read_support_filter_gzip(&C.archive) + // Configure the archive to work with a tarball content fn C.archive_read_support_format_tar(&C.archive) diff --git a/src/package.v b/src/package.v index 7db58494..2d783bdd 100644 --- a/src/package.v +++ b/src/package.v @@ -111,19 +111,21 @@ pub fn read_pkg(pkg_path string) ?Pkg { // Sinds 2020, all newly built Arch packages use zstd C.archive_read_support_filter_zstd(a) + C.archive_read_support_filter_gzip(a) // The content should always be a tarball C.archive_read_support_format_tar(a) // TODO find out where does this 10240 come from r = C.archive_read_open_filename(a, &char(pkg_path.str), 10240) - defer { - C.archive_read_free(a) - } if r != C.ARCHIVE_OK { return error('Failed to open package.') } + defer { + C.archive_read_free(a) + } + mut buf := voidptr(0) mut files := []string{} mut pkg_info := PkgInfo{} @@ -150,10 +152,8 @@ pub fn read_pkg(pkg_path string) ?Pkg { } C.archive_read_data(a, buf, size) - unsafe { - println(cstring_to_vstring(buf)) - } - pkg_info = parse_pkg_info_string(unsafe { cstring_to_vstring(buf) }) ? + pkg_text := unsafe { cstring_to_vstring(buf) } + pkg_info = parse_pkg_info_string(pkg_text) ? } else { C.archive_read_data_skip(a) } diff --git a/src/repo.v b/src/repo.v index cca6b848..fcf48089 100644 --- a/src/repo.v +++ b/src/repo.v @@ -96,6 +96,7 @@ fn (r &Repo) pkg_path(pkg &package.Pkg) string { // Re-generate the repo archive files fn (r &Repo) sync() ? { + // TODO also write files archive lock r.mutex { a := C.archive_write_new() entry := C.archive_entry_new() @@ -106,6 +107,7 @@ fn (r &Repo) sync() ? { C.archive_write_add_filter_gzip(a) C.archive_write_set_format_pax_restricted(a) + // TODO add symlink to .tar.gz version repo_path := os.join_path_single(r.repo_dir, 'repo.db') C.archive_write_open_filename(a, &char(repo_path.str)) diff --git a/src/routes.v b/src/routes.v index 8ad4a4a0..c1e888b9 100644 --- a/src/routes.v +++ b/src/routes.v @@ -76,9 +76,17 @@ fn (mut app App) put_package() web.Result { added := app.repo.add_from_path(pkg_path) or { app.lerror('Error while adding package: $err.msg') + os.rm(pkg_path) or { + app.lerror("Failed to remove download '$pkg_path'.") + } + return app.text('Failed to add package.') } if !added { + os.rm(pkg_path) or { + app.lerror("Failed to remove download '$pkg_path'.") + } + app.lwarn('Duplicate package.') return app.text('File already exists.') diff --git a/test.py b/test.py new file mode 100644 index 00000000..3a0b51cf --- /dev/null +++ b/test.py @@ -0,0 +1,116 @@ +import random +import tempfile +import tarfile +from pathlib import Path +import uuid +import argparse +import asyncio +import aiohttp + + +# A list of words the program can choose from +WORDS = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", + "hotel", "india", "juliet", "kilo", "lima", "mike", "november", + "oscar", "papa", "quebec", "romeo", "sierra", "tango", "uniform", + "victor", "whiskey", "xray", "yankee", "zulu"] +SEED = 2022 + + +def random_words(max_len=None, min_len=1): + """ + Returns a random list of words, with a length randomly choosen between + min_len and max_len. If max_len is None, it is equal to the length of + WORDS. + """ + if max_len is None: + max_len = len(WORDS) + + k = random.randint(min_len, max_len) + + return random.choices(WORDS, k=k) + +def random_lists(n, max_len=None, min_len=1): + return [random_words(max_len, min_len) for _ in range(n)] + +def create_random_pkginfo(): + """ + Generates a random .PKGINFO + """ + name = "-".join(random_words(3)) + ver = "0.1.0" # doesn't matter what it is anyways + + # TODO add random dependencies (all types) + + data = { + "pkgname": name, + "pkgbase": name, + "pkgver": ver, + "arch": "x86_64" + } + + return "\n".join(f"{key} = {value}" for key, value in data.items()) + +def create_random_package(tmpdir): + """ + Creates a random, but valid Arch package, using the provided tmpdir. Output + is the path to the created package tarball. + """ + + sub_path = tmpdir / uuid.uuid4().hex + sub_path.mkdir() + + tar_path = sub_path / "archive.pkg.tar.gz" + + def remove_prefix(tar_info): + tar_info.name = tar_info.name[len(str(sub_path)):] + + return tar_info + + with tarfile.open(tar_path, "w") as tar: + # Add random .PKGINFO file + pkginfo_file = sub_path / ".PKGINFO" + pkginfo_file.write_text(create_random_pkginfo()) + tar.add(pkginfo_file, filter=remove_prefix) + + # Create random files + for words in random_lists(10, max_len=5): + path = sub_path / 'usr' / ('/'.join(words) + ".txt") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(' '.join(words)) + + tar.add(path, filter=remove_prefix) + + return tar_path + + +async def upload_random_package(dir, sem): + tar_path = create_random_package(dir) + + async with sem: + with open(tar_path, 'rb') as f: + async with aiohttp.ClientSession() as s: + async with s.post("http://localhost:8000/publish", data=f.read(), headers={"x-api-key": "test"}) as r: + print(r.text) + + +async def main(): + parser = argparse.ArgumentParser(description="Test vieter by uploading random package files.") + + parser.add_argument("count", help="How many packages to upload.", default=1, type=int) + parser.add_argument("-p", "--parallel", help="How many uploads to run in parallel.", default=1, type=int) + parser.add_argument("-s", "--seed", help="Seed for the randomizer.", default=SEED, type=int) + # parser.add_argument("-d", "--dir", help="Directory to create ") + + args = parser.parse_args() + + sem = asyncio.BoundedSemaphore(args.parallel) + random.seed(args.seed) + + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdir = Path(tmpdirname) + + await asyncio.gather(*(upload_random_package(tmpdir, sem) for _ in range(args.count))) + + +if __name__ == "__main__": + asyncio.run(main())