From 4288c40bee80552205204f66354bce890735c2c4 Mon Sep 17 00:00:00 2001 From: ChAoS_UnItY Date: Fri, 3 Jun 2022 14:00:11 +0800 Subject: [PATCH] compress: Add gzip module & refactor compress & compress.zlib module (#14599) --- vlib/compress/README.md | 2 +- vlib/compress/compress.v | 44 +++++++++++++++++++++++++++++++ vlib/compress/gzip/README.md | 17 ++++++++++++ vlib/compress/gzip/gzip.v | 16 ++++++++++++ vlib/compress/gzip/gzip_test.v | 8 ++++++ vlib/compress/zlib/zlib.v | 47 +++------------------------------- 6 files changed, 89 insertions(+), 45 deletions(-) create mode 100644 vlib/compress/compress.v create mode 100644 vlib/compress/gzip/README.md create mode 100644 vlib/compress/gzip/gzip.v create mode 100644 vlib/compress/gzip/gzip_test.v diff --git a/vlib/compress/README.md b/vlib/compress/README.md index 158c2c432b..166e052b47 100644 --- a/vlib/compress/README.md +++ b/vlib/compress/README.md @@ -1,4 +1,4 @@ ## Description: `compress` is a namespace for (multiple) compression algorithms supported by V. -At the moment, only `compress.zlib` is implemented. +At the moment, only `compress.zlib` and `compress.gzip` are implemented. diff --git a/vlib/compress/compress.v b/vlib/compress/compress.v new file mode 100644 index 0000000000..062efff48a --- /dev/null +++ b/vlib/compress/compress.v @@ -0,0 +1,44 @@ +module compress + +#flag -I @VEXEROOT/thirdparty/zip +#include "miniz.h" + +pub const max_size = u64(1 << 31) + +fn C.tdefl_compress_mem_to_heap(source_buf voidptr, source_buf_len usize, out_len &usize, flags int) voidptr +fn C.tinfl_decompress_mem_to_heap(source_buf voidptr, source_buf_len usize, out_len &usize, flags int) voidptr + +// compresses an array of bytes based on providing flags and returns the compressed bytes in a new array +// see `gzip.compress([]u8)` and `zlib.compress([]u8)` for default implementations. +[manualfree] +pub fn compress(data []u8, flags int) ?[]u8 { + if u64(data.len) > compress.max_size { + return error('data too large ($data.len > $compress.max_size)') + } + mut out_len := usize(0) + + address := C.tdefl_compress_mem_to_heap(data.data, data.len, &out_len, flags) + if address == 0 { + return error('compression failed') + } + if u64(out_len) > compress.max_size { + return error('compressed data is too large ($out_len > $compress.max_size)') + } + return unsafe { address.vbytes(int(out_len)) } +} + +// decompresses an array of bytes based on providing flags and returns the decompressed bytes in a new array +// see `gzip.decompress([]u8)` and `zlib.decompress([]u8)` for default implementations. +[manualfree] +pub fn decompress(data []u8, flags int) ?[]u8 { + mut out_len := usize(0) + + address := C.tinfl_decompress_mem_to_heap(data.data, data.len, &out_len, flags) + if address == 0 { + return error('decompression failed') + } + if u64(out_len) > compress.max_size { + return error('decompressed data is too large ($out_len > $compress.max_size)') + } + return unsafe { address.vbytes(int(out_len)) } +} diff --git a/vlib/compress/gzip/README.md b/vlib/compress/gzip/README.md new file mode 100644 index 0000000000..20f5aefa1f --- /dev/null +++ b/vlib/compress/gzip/README.md @@ -0,0 +1,17 @@ +## Description: + +`compress.gzip` is a module that assists in the compression and +decompression of binary data using `gzip` compression + +## Examples: + +```v +import compress.gzip + +fn main() { + uncompressed := 'Hello world!' + compressed := gzip.compress(uncompressed.bytes())? + decompressed := gzip.decompress(compressed)? + assert decompressed == uncompressed.bytes() +} +``` \ No newline at end of file diff --git a/vlib/compress/gzip/gzip.v b/vlib/compress/gzip/gzip.v new file mode 100644 index 0000000000..5213e319d8 --- /dev/null +++ b/vlib/compress/gzip/gzip.v @@ -0,0 +1,16 @@ +module gzip + +import compress + +// compresses an array of bytes using gzip and returns the compressed bytes in a new array +// Example: compressed := gzip.compress(b)? +pub fn compress(data []u8) ?[]u8 { + return compress.compress(data, 0) +} + +// decompresses an array of bytes using zlib and returns the decompressed bytes in a new array +// Example: decompressed := zlib.decompress(b)? +[manualfree] +pub fn decompress(data []u8) ?[]u8 { + return compress.decompress(data, 0) +} diff --git a/vlib/compress/gzip/gzip_test.v b/vlib/compress/gzip/gzip_test.v new file mode 100644 index 0000000000..2c91137be8 --- /dev/null +++ b/vlib/compress/gzip/gzip_test.v @@ -0,0 +1,8 @@ +module gzip + +fn test_gzip() ? { + uncompressed := 'Hello world!' + compressed := compress(uncompressed.bytes())? + decompressed := decompress(compressed)? + assert decompressed == uncompressed.bytes() +} diff --git a/vlib/compress/zlib/zlib.v b/vlib/compress/zlib/zlib.v index d72800e613..44e4e2abc1 100644 --- a/vlib/compress/zlib/zlib.v +++ b/vlib/compress/zlib/zlib.v @@ -1,60 +1,19 @@ module zlib -#flag -I @VEXEROOT/thirdparty/zip -#include "miniz.h" - -pub const max_size = u64(1 << 31) - -fn C.tdefl_compress_mem_to_heap(source_buf voidptr, source_buf_len usize, out_len &usize, flags int) voidptr -fn C.tinfl_decompress_mem_to_heap(source_buf voidptr, source_buf_len usize, out_len &usize, flags int) voidptr +import compress // compresses an array of bytes using zlib and returns the compressed bytes in a new array // Example: compressed := zlib.compress(b)? [manualfree] pub fn compress(data []u8) ?[]u8 { - if u64(data.len) > zlib.max_size { - return error('data too large ($data.len > $zlib.max_size)') - } - mut out_len := usize(0) - // flags = TDEFL_WRITE_ZLIB_HEADER (0x01000) - address := C.tdefl_compress_mem_to_heap(data.data, data.len, &out_len, 0x01000) - if address == 0 { - return error('compression failed') - } - if u64(out_len) > zlib.max_size { - return error('compressed data is too large ($out_len > $zlib.max_size)') - } - compressed := unsafe { - address.vbytes(int(out_len)) - } - copy := compressed.clone() - unsafe { - free(address) - } - return copy + return compress.compress(data, 0x01000) } // decompresses an array of bytes using zlib and returns the decompressed bytes in a new array // Example: decompressed := zlib.decompress(b)? [manualfree] pub fn decompress(data []u8) ?[]u8 { - mut out_len := usize(0) - // flags = TINFL_FLAG_PARSE_ZLIB_HEADER (0x1) - address := C.tinfl_decompress_mem_to_heap(data.data, data.len, &out_len, 0x1) - if address == 0 { - return error('decompression failed') - } - if u64(out_len) > zlib.max_size { - return error('decompressed data is too large ($out_len > $zlib.max_size)') - } - decompressed := unsafe { - address.vbytes(int(out_len)) - } - copy := decompressed.clone() - unsafe { - free(address) - } - return copy + return compress.decompress(data, 0x1) }