From 4961d3ea17b4d15fc271c5abb73603dcacf7ea0f Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Tue, 16 Feb 2021 22:53:46 +0200 Subject: [PATCH] io: fix detection of end_of_stream, when reading files through io.new_buffered_reader(reader: io.make_reader(f)) --- examples/buf_reader.v | 12 +++++------- vlib/io/buffered_reader.v | 32 +++++++++++++++++++++++--------- vlib/io/os_file_reader_test.v | 29 +++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 vlib/io/os_file_reader_test.v diff --git a/examples/buf_reader.v b/examples/buf_reader.v index b2eb6cb2e0..a4268367ad 100644 --- a/examples/buf_reader.v +++ b/examples/buf_reader.v @@ -5,17 +5,15 @@ import io fn main() { // Make a new connection - mut conn := net.dial_tcp('google.com:80')? + mut conn := net.dial_tcp('google.com:80') ? // Simple http HEAD request for a file - conn.write_str('GET /index.html HTTP/1.0\r\n\r\n')? + conn.write_str('GET /index.html HTTP/1.0\r\n\r\n') ? // Wrap in a buffered reader mut r := io.new_buffered_reader(reader: io.make_reader(conn)) for { - l := r.read_line() or { - break - } + l := r.read_line() or { break } println('$l') // Make it nice and obvious that we are doing this line by line - time.sleep_ms(10) + time.sleep_ms(100) } -} +} diff --git a/vlib/io/buffered_reader.v b/vlib/io/buffered_reader.v index 860ba5356c..3f1ccb7a87 100644 --- a/vlib/io/buffered_reader.v +++ b/vlib/io/buffered_reader.v @@ -3,19 +3,21 @@ module io // BufferedReader provides a buffered interface for a reader struct BufferedReader { mut: - reader Reader - buf []byte - // current offset in the buffer - offset int - len int - // Whether we reached the end of the upstream reader - end_of_stream bool + reader Reader + buf []byte + offset int // current offset in the buffer + len int + fails int // how many times fill_buffer has read 0 bytes in a row + mfails int // maximum fails, after which we can assume that the stream has ended +pub mut: + end_of_stream bool // whether we reached the end of the upstream reader } // BufferedReaderConfig are options that can be given to a reader pub struct BufferedReaderConfig { - reader Reader - cap int = 128 * 1024 // large for fast reading of big(ish) files + reader Reader + cap int = 128 * 1024 // large for fast reading of big(ish) files + retries int = 2 // how many times to retry before assuming the stream ended } // new_buffered_reader creates new BufferedReader @@ -26,6 +28,7 @@ pub fn new_buffered_reader(o BufferedReaderConfig) &BufferedReader { reader: o.reader buf: []byte{len: o.cap, cap: o.cap} offset: 0 + mfails: o.retries } return r } @@ -59,6 +62,17 @@ fn (mut r BufferedReader) fill_buffer() bool { r.end_of_stream = true return false } + if r.len == 0 { + r.fails++ + } else { + r.fails = 0 + } + if r.fails >= r.mfails { + // When reading 0 bytes several times in a row, assume the stream has ended. + // This prevents infinite loops ¯\_(ツ)_/¯ ... + r.end_of_stream = true + return false + } // we got some data return true } diff --git a/vlib/io/os_file_reader_test.v b/vlib/io/os_file_reader_test.v new file mode 100644 index 0000000000..39606c8a03 --- /dev/null +++ b/vlib/io/os_file_reader_test.v @@ -0,0 +1,29 @@ +import os +import io + +fn read_file(file string, cap int) []string { + mut lines := []string{} + mut f := os.open(file) or { panic(err) } + defer { + f.close() + } + mut r := io.new_buffered_reader(reader: io.make_reader(f), cap: cap) + for { + l := r.read_line() or { break } + lines << l + // println('Line: $l') + } + assert lines.len > 0 + assert r.end_of_stream == true + println('------------------------------------------------ cap: ${cap:6}; read: ${lines.len:3} lines') + return lines +} + +fn test_file_reader() { + for cap := 64; cap <= 10000; cap += 256 { + lines := read_file(@FILE, cap) + assert lines.last() == '// my last line' + } +} + +// my last line