From 9995f6cca14b6c93ebef02fc96f27104aa63ee2b Mon Sep 17 00:00:00 2001 From: Sebastian Schicho <64368773+schicho@users.noreply.github.com> Date: Fri, 6 Aug 2021 23:09:55 +0200 Subject: [PATCH] encoding.base64: speed up encoding and decoding (#11055) * add new function headers * new encoding function * rename to decode_micro and add helper functions * implement new decoding function * add test for buffer to buffer decoding * - add notice to GO code - v fmt base64.v * implement new decoding function * fix base64_memory_test.v by commenting a few lines * vfmt base64.v * add some more asserts to base64_memory_test.v * remove unused decoding function * add bounds check, when detecting the padding * use union for storing the decoded data Co-authored-by: Delyan Angelov --- vlib/encoding/base64/base64.v | 228 +++++++++++++++------- vlib/encoding/base64/base64_memory_test.v | 27 ++- vlib/encoding/base64/base64_test.v | 18 ++ 3 files changed, 198 insertions(+), 75 deletions(-) diff --git a/vlib/encoding/base64/base64.v b/vlib/encoding/base64/base64.v index 203ff0b812..61ffd4f583 100644 --- a/vlib/encoding/base64/base64.v +++ b/vlib/encoding/base64/base64.v @@ -1,6 +1,8 @@ // Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved. // Use of this source code is governed by an MIT license // that can be found in the LICENSE file. +// Based off: https://github.com/golang/go/blob/master/src/encoding/base64/base64.go +// Last commit: https://github.com/golang/go/commit/9a93baf4d7d13d7d5c67388c93960d78abc8e11e module base64 const ( @@ -13,6 +15,18 @@ const ( enc_table = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' ) +union B64_64_datablock { +mut: + data u64 + data_byte [8]byte +} + +union B64_32_datablock { +mut: + data u32 + data_byte [4]byte +} + // decode decodes the base64 encoded `string` value passed in `data`. // Please note: If you need to decode many strings repeatedly, take a look at `decode_in_buffer`. // Example: assert base64.decode('ViBpbiBiYXNlIDY0') == 'V in base 64' @@ -104,63 +118,122 @@ pub fn url_encode_str(data string) string { return encode_str(data).replace_each(['+', '-', '/', '_', '=', '']) } +// assemble64 assembles 8 base64 digits into 6 bytes. +// Each digit comes from the decode map. +// Please note: Invalid base64 digits are not expected and not handled. +fn assemble64(n1 byte, n2 byte, n3 byte, n4 byte, n5 byte, n6 byte, n7 byte, n8 byte) u64 { + return u64(n1) << 58 | u64(n2) << 52 | u64(n3) << 46 | u64(n4) << 40 | u64(n5) << 34 | u64(n6) << 28 | u64(n7) << 22 | u64(n8) << 16 +} + +// assemble32 assembles 4 base64 digits into 3 bytes. +// Each digit comes from the decode map. +// Please note: Invalid base64 digits are not expected and not handled. +fn assemble32(n1 byte, n2 byte, n3 byte, n4 byte) u32 { + return u32(n1) << 26 | u32(n2) << 20 | u32(n3) << 14 | u32(n4) << 8 +} + // decode_in_buffer decodes the base64 encoded `string` reference passed in `data` into `buffer`. // decode_in_buffer returns the size of the decoded data in the buffer. // Please note: The `buffer` should be large enough (i.e. 3/4 of the data.len, or larger) // to hold the decoded data. // Please note: This function does NOT allocate new memory, and is thus suitable for handling very large strings. pub fn decode_in_buffer(data &string, buffer &byte) int { + return decode_from_buffer(buffer, data.str, data.len) +} + +// decode_from_buffer decodes the base64 encoded ASCII bytes from `data` into `buffer`. +// decode_from_buffer returns the size of the decoded data in the buffer. +// Please note: The `buffer` should be large enough (i.e. 3/4 of the data.len, or larger) +// to hold the decoded data. +// Please note: This function does NOT allocate new memory, and is thus suitable for handling very large strings. +pub fn decode_in_buffer_bytes(data []byte, buffer &byte) int { + return decode_from_buffer(buffer, data.data, data.len) +} + +// decode_from_buffer decodes the base64 encoded ASCII bytes from `src` into `dest`. +// decode_from_buffer returns the size of the decoded data in the buffer. +// Please note: The `dest` buffer should be large enough (i.e. 3/4 of the `src_len`, or larger) +// to hold the decoded data. +// Please note: This function does NOT allocate new memory, and is thus suitable for handling very large strings. +// Please note: This function is for internal base64 decoding +fn decode_from_buffer(dest &byte, src &byte, src_len int) int { + if src_len < 4 { + return 0 + } + mut padding := 0 - if data.ends_with('=') { - if data.ends_with('==') { + if unsafe { src[src_len - 1] == `=` } { + if unsafe { src[src_len - 2] == `=` } { padding = 2 } else { padding = 1 } } - // input_length is the length of meaningful data - input_length := data.len - padding - output_length := input_length * 3 / 4 - mut i := 0 - mut j := 0 - mut b := &byte(0) - mut d := &byte(0) + mut d := unsafe { src } + mut b := unsafe { dest } + unsafe { - d = &byte(data.str) - b = &byte(buffer) - } - for i < input_length { - mut char_a := 0 - mut char_b := 0 - mut char_c := 0 - mut char_d := 0 - if i < input_length { - char_a = base64.index[unsafe { d[i] }] - i++ + mut n_decoded_bytes := 0 // padding bytes are also counted towards this. + mut si := 0 + + mut datablock_64 := B64_64_datablock{ + data: 0 } - if i < input_length { - char_b = base64.index[unsafe { d[i] }] - i++ - } - if i < input_length { - char_c = base64.index[unsafe { d[i] }] - i++ - } - if i < input_length { - char_d = base64.index[unsafe { d[i] }] - i++ + mut datablock_32 := B64_32_datablock{ + data: 0 } - decoded_bytes := (char_a << 18) | (char_b << 12) | (char_c << 6) | (char_d << 0) - unsafe { - b[j] = byte(decoded_bytes >> 16) - b[j + 1] = byte((decoded_bytes >> 8) & 0xff) - b[j + 2] = byte((decoded_bytes >> 0) & 0xff) + for src_len - si >= 8 { + // Converting 8 bytes of input into 6 bytes of output. Storing these in the upper bytes of an u64. + datablock_64.data = assemble64(byte(base64.index[d[si + 0]]), byte(base64.index[d[si + 1]]), + byte(base64.index[d[si + 2]]), byte(base64.index[d[si + 3]]), byte(base64.index[d[ + si + 4]]), byte(base64.index[d[si + 5]]), byte(base64.index[d[si + 6]]), + byte(base64.index[d[si + 7]])) + + // Reading out the individual bytes from the u64. Watch out with endianess. + $if little_endian { + b[n_decoded_bytes + 0] = datablock_64.data_byte[7] + b[n_decoded_bytes + 1] = datablock_64.data_byte[6] + b[n_decoded_bytes + 2] = datablock_64.data_byte[5] + b[n_decoded_bytes + 3] = datablock_64.data_byte[4] + b[n_decoded_bytes + 4] = datablock_64.data_byte[3] + b[n_decoded_bytes + 5] = datablock_64.data_byte[2] + } $else { + b[n_decoded_bytes + 0] = datablock_64.data_byte[0] + b[n_decoded_bytes + 1] = datablock_64.data_byte[1] + b[n_decoded_bytes + 2] = datablock_64.data_byte[2] + b[n_decoded_bytes + 3] = datablock_64.data_byte[3] + b[n_decoded_bytes + 4] = datablock_64.data_byte[4] + b[n_decoded_bytes + 5] = datablock_64.data_byte[5] + } + + n_decoded_bytes += 6 + si += 8 } - j += 3 + + for src_len - si >= 4 { + datablock_32.data = assemble32(byte(base64.index[d[si + 0]]), byte(base64.index[d[si + 1]]), + byte(base64.index[d[si + 2]]), byte(base64.index[d[si + 3]])) + + $if little_endian { + b[n_decoded_bytes + 0] = datablock_32.data_byte[3] + b[n_decoded_bytes + 1] = datablock_32.data_byte[2] + b[n_decoded_bytes + 2] = datablock_32.data_byte[1] + b[n_decoded_bytes + 3] = datablock_32.data_byte[0] + } $else { + b[n_decoded_bytes + 0] = datablock_32.data_byte[0] + b[n_decoded_bytes + 1] = datablock_32.data_byte[1] + b[n_decoded_bytes + 2] = datablock_32.data_byte[2] + b[n_decoded_bytes + 3] = datablock_32.data_byte[3] + } + + n_decoded_bytes += 3 + si += 4 + } + + return n_decoded_bytes - padding } - return output_length } // encode_in_buffer base64 encodes the `[]byte` passed in `data` into `buffer`. @@ -176,48 +249,59 @@ pub fn encode_in_buffer(data []byte, buffer &byte) int { // Please note: The `dest` buffer should be large enough (i.e. 4/3 of the src_len, or larger) to hold the encoded data. // Please note: This function is for internal base64 encoding fn encode_from_buffer(dest &byte, src &byte, src_len int) int { - input_length := src_len - output_length := 4 * ((input_length + 2) / 3) - - mut i := 0 - mut j := 0 + if src_len == 0 { + return 0 + } + output_length := 4 * ((src_len + 2) / 3) mut d := unsafe { src } mut b := unsafe { dest } - mut etable := base64.enc_table.str - for i < input_length { - mut octet_a := 0 - mut octet_b := 0 - mut octet_c := 0 - - if i < input_length { - octet_a = int(unsafe { d[i] }) - i++ - } - if i < input_length { - octet_b = int(unsafe { d[i] }) - i++ - } - if i < input_length { - octet_c = int(unsafe { d[i] }) - i++ - } - - triple := ((octet_a << 0x10) + (octet_b << 0x08) + octet_c) + etable := base64.enc_table.str + mut di := 0 + mut si := 0 + n := (src_len / 3) * 3 + for si < n { + // Convert 3x 8bit source bytes into 4 bytes unsafe { - b[j] = etable[(triple >> 3 * 6) & 63] // 63 is 0x3F - b[j + 1] = etable[(triple >> 2 * 6) & 63] - b[j + 2] = etable[(triple >> 1 * 6) & 63] - b[j + 3] = etable[(triple >> 0 * 6) & 63] + val := u32(d[si + 0]) << 16 | u32(d[si + 1]) << 8 | u32(d[si + 2]) + + b[di + 0] = etable[val >> 18 & 0x3F] + b[di + 1] = etable[val >> 12 & 0x3F] + b[di + 2] = etable[val >> 6 & 0x3F] + b[di + 3] = etable[val & 0x3F] } - j += 4 + si += 3 + di += 4 } - padding_length := base64.ending_table[input_length % 3] - for i = 0; i < padding_length; i++ { - unsafe { - b[output_length - 1 - i] = `=` + remain := src_len - si + if remain == 0 { + return output_length + } + + // Add the remaining small block and padding + unsafe { + mut val := u32(d[si + 0]) << 16 + if remain == 2 { + val |= u32(d[si + 1]) << 8 + } + + b[di + 0] = etable[val >> 18 & 0x3F] + b[di + 1] = etable[val >> 12 & 0x3F] + + match remain { + 2 { + b[di + 2] = etable[val >> 6 & 0x3F] + b[di + 3] = byte(`=`) + } + 1 { + b[di + 2] = byte(`=`) + b[di + 3] = byte(`=`) + } + else { + panic('base64: This case should never occur.') + } } } return output_length diff --git a/vlib/encoding/base64/base64_memory_test.v b/vlib/encoding/base64/base64_memory_test.v index a1ac47a59b..6bd9f99faf 100644 --- a/vlib/encoding/base64/base64_memory_test.v +++ b/vlib/encoding/base64/base64_memory_test.v @@ -11,16 +11,37 @@ fn test_long_encoding() { assert s_encoded.len > s_original.len assert s_original == s_decoded - mut s := 0 - ebuffer := unsafe { malloc(s_encoded.len) } + dbuffer := unsafe { malloc(s_decoded.len) } + // + encoded_size := base64.encode_in_buffer(s_original, ebuffer) + mut encoded_in_buf := []byte{len: encoded_size} + unsafe { C.memcpy(encoded_in_buf.data, ebuffer, encoded_size) } + assert input_size * 4 / 3 == encoded_size + assert encoded_in_buf[0] == `Y` + assert encoded_in_buf[1] == `W` + assert encoded_in_buf[2] == `F` + assert encoded_in_buf[3] == `h` + + assert encoded_in_buf[encoded_size - 4] == `Y` + assert encoded_in_buf[encoded_size - 3] == `W` + assert encoded_in_buf[encoded_size - 2] == `F` + assert encoded_in_buf[encoded_size - 1] == `h` + assert encoded_in_buf == s_encoded.bytes() + + decoded_size := base64.decode_in_buffer(s_encoded, dbuffer) + assert decoded_size == input_size + mut decoded_in_buf := []byte{len: decoded_size} + unsafe { C.memcpy(decoded_in_buf.data, dbuffer, decoded_size) } + assert decoded_in_buf == s_original + + mut s := 0 for _ in 0 .. repeats { resultsize := base64.encode_in_buffer(s_original, ebuffer) s += resultsize assert resultsize == s_encoded.len } - dbuffer := unsafe { malloc(s_decoded.len) } for _ in 0 .. repeats { resultsize := base64.decode_in_buffer(s_encoded, dbuffer) s += resultsize diff --git a/vlib/encoding/base64/base64_test.v b/vlib/encoding/base64/base64_test.v index e461330b92..8d08de24ad 100644 --- a/vlib/encoding/base64/base64_test.v +++ b/vlib/encoding/base64/base64_test.v @@ -130,3 +130,21 @@ fn test_decode_null_byte_str() { s := [byte(`A`), 0, `C`].bytestr() assert base64.decode_str('QQBD') == s } + +fn test_decode_in_buffer_bytes() { + rfc4648_pairs := [ + TestPair{'foob', 'Zm9vYg=='}, + TestPair{'fooba', 'Zm9vYmE='}, + TestPair{'foobar', 'Zm9vYmFy'}, + ] + mut src_dec_buf := []byte{len: 8} + mut src_enc_buf := []byte{len: 8} + mut out_buf := []byte{len: 8} + + for p in rfc4648_pairs { + src_dec_buf = p.decoded.bytes() + src_enc_buf = p.encoded.bytes() + n := base64.decode_in_buffer_bytes(src_enc_buf, out_buf.data) + assert src_dec_buf == out_buf[..n] + } +}