vlib/io: fix reader bugs, make read_all take a config struct (#7361)
parent
1a2c7cd336
commit
553ecf63e7
|
@ -8,7 +8,7 @@ fn main() {
|
||||||
// Simple http HEAD request for a file
|
// Simple http HEAD request for a file
|
||||||
conn.write_str('HEAD /index.html HTTP/1.0\r\n\r\n')?
|
conn.write_str('HEAD /index.html HTTP/1.0\r\n\r\n')?
|
||||||
// Read all the data that is waiting
|
// Read all the data that is waiting
|
||||||
result := io.read_all(conn)?
|
result := io.read_all(reader: conn)?
|
||||||
// Cast to string and print result
|
// Cast to string and print result
|
||||||
println(result.bytestr())
|
println(result.bytestr())
|
||||||
}
|
}
|
||||||
|
|
|
@ -649,9 +649,15 @@ pub fn (a []int) reduce(iter fn (int, int) int, accum_start int) int {
|
||||||
return accum_
|
return accum_
|
||||||
}
|
}
|
||||||
|
|
||||||
// grow grows the array's capacity by `amount` elements.
|
// grow_cap grows the array's capacity by `amount` elements.
|
||||||
pub fn (mut a array) grow(amount int) {
|
pub fn (mut a array) grow_cap(amount int) {
|
||||||
|
a.ensure_cap(a.cap + amount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// grow_len ensures that an array has a.len + amount of length
|
||||||
|
pub fn (mut a array) grow_len(amount int) {
|
||||||
a.ensure_cap(a.len + amount)
|
a.ensure_cap(a.len + amount)
|
||||||
|
a.len += amount
|
||||||
}
|
}
|
||||||
|
|
||||||
// array_eq<T> checks if two arrays contain all the same elements in the same order.
|
// array_eq<T> checks if two arrays contain all the same elements in the same order.
|
||||||
|
|
|
@ -8,22 +8,23 @@ mut:
|
||||||
// current offset in the buffer
|
// current offset in the buffer
|
||||||
offset int
|
offset int
|
||||||
len int
|
len int
|
||||||
|
// Whether we reached the end of the upstream reader
|
||||||
|
end_of_stream bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// BufferedReaderConfig are options that can be given to a reader
|
// BufferedReaderConfig are options that can be given to a reader
|
||||||
pub struct BufferedReaderConfig {
|
pub struct BufferedReaderConfig {
|
||||||
reader Reader
|
reader Reader
|
||||||
buf_cap int = 128 * 1024 // large for fast reading of big(ish) files
|
cap int = 128 * 1024 // large for fast reading of big(ish) files
|
||||||
}
|
}
|
||||||
|
|
||||||
// new_buffered_reader creates new BufferedReader
|
// new_buffered_reader creates new BufferedReader
|
||||||
pub fn new_buffered_reader(o BufferedReaderConfig) &BufferedReader {
|
pub fn new_buffered_reader(o BufferedReaderConfig) &BufferedReader {
|
||||||
assert o.buf_cap >= 2
|
assert o.cap >= 2
|
||||||
|
|
||||||
// create
|
// create
|
||||||
r := &BufferedReader{
|
r := &BufferedReader{
|
||||||
reader: o.reader
|
reader: o.reader
|
||||||
buf: []byte{len: o.buf_cap, cap: o.buf_cap}
|
buf: []byte{len: o.cap, cap: o.cap}
|
||||||
offset: 0
|
offset: 0
|
||||||
}
|
}
|
||||||
return r
|
return r
|
||||||
|
@ -31,46 +32,70 @@ pub fn new_buffered_reader(o BufferedReaderConfig) &BufferedReader {
|
||||||
|
|
||||||
// read fufills the Reader interface
|
// read fufills the Reader interface
|
||||||
pub fn (mut r BufferedReader) read(mut buf []byte) ?int {
|
pub fn (mut r BufferedReader) read(mut buf []byte) ?int {
|
||||||
// read data out of the buffer if we dont have any
|
if r.end_of_stream {
|
||||||
if r.offset >= r.len-1 {
|
return none
|
||||||
r.fill_buffer()?
|
}
|
||||||
|
// read data out of the buffer if we dont have any
|
||||||
|
if r.needs_fill() {
|
||||||
|
if !r.fill_buffer() {
|
||||||
|
// end of stream
|
||||||
|
return none
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
read := copy(buf, r.buf[r.offset..r.len])
|
read := copy(buf, r.buf[r.offset..r.len])
|
||||||
r.offset += read
|
r.offset += read
|
||||||
|
|
||||||
return read
|
return read
|
||||||
}
|
}
|
||||||
|
|
||||||
// fill buffer attempts to refill the internal buffer
|
// fill_buffer attempts to refill the internal buffer
|
||||||
fn (mut r BufferedReader) fill_buffer() ? {
|
// and returns whether it got any data
|
||||||
// TODO we should keep track of when we get an end of stream
|
fn (mut r BufferedReader) fill_buffer() bool {
|
||||||
// from the upstream reader so that we dont have to keep
|
if r.end_of_stream {
|
||||||
// trying to call this
|
// we know we have already reached the end of stream
|
||||||
|
// so return early
|
||||||
|
return false
|
||||||
|
}
|
||||||
r.offset = 0
|
r.offset = 0
|
||||||
new_len := r.reader.read(mut r.buf) or {
|
r.len = 0
|
||||||
if errcode != 0 || err.len != 0 {
|
r.len = r.reader.read(mut r.buf) or {
|
||||||
eprintln('>> BufferedReader.reader.read err: $err | errcode: $errcode')
|
// end of stream was reached
|
||||||
|
r.end_of_stream = true
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
0
|
// we got some data
|
||||||
}
|
return true
|
||||||
r.len = new_len
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// read_line reads a line from the buffered reader
|
// needs_fill returns whether the buffer needs refilling
|
||||||
|
fn (r BufferedReader) needs_fill() bool {
|
||||||
|
return r.offset >= r.len - 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// end_of_stream returns whether the end of the stream was reached
|
||||||
|
pub fn (r BufferedReader) end_of_stream() bool {
|
||||||
|
return r.end_of_stream
|
||||||
|
}
|
||||||
|
|
||||||
|
// read_line attempts to read a line from the buffered reader
|
||||||
|
// it will read until it finds a new line character (\n) or
|
||||||
|
// the end of stream
|
||||||
pub fn (mut r BufferedReader) read_line() ?string {
|
pub fn (mut r BufferedReader) read_line() ?string {
|
||||||
mut line := []byte{}
|
if r.end_of_stream {
|
||||||
for {
|
|
||||||
if r.offset >= (r.len-1) {
|
|
||||||
// go fetch some new data
|
|
||||||
r.fill_buffer()?
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.len == 0 {
|
|
||||||
// if we have no data then return nothing
|
|
||||||
return none
|
return none
|
||||||
}
|
}
|
||||||
|
mut line := []byte{}
|
||||||
|
for {
|
||||||
|
if r.needs_fill() {
|
||||||
|
// go fetch some new data
|
||||||
|
if !r.fill_buffer() {
|
||||||
|
// We are at the end of the stream
|
||||||
|
if line.len == 0 {
|
||||||
|
// we had nothing so return nothing
|
||||||
|
return none
|
||||||
|
}
|
||||||
|
return line.bytestr()
|
||||||
|
}
|
||||||
|
}
|
||||||
// try and find a newline character
|
// try and find a newline character
|
||||||
mut i := r.offset
|
mut i := r.offset
|
||||||
for ; i < r.len; i++ {
|
for ; i < r.len; i++ {
|
||||||
|
@ -78,22 +103,17 @@ pub fn (mut r BufferedReader) read_line() ?string {
|
||||||
if c == `\n` {
|
if c == `\n` {
|
||||||
// great, we hit something
|
// great, we hit something
|
||||||
// do some checking for whether we hit \r\n or just \n
|
// do some checking for whether we hit \r\n or just \n
|
||||||
|
|
||||||
if i != 0 && r.buf[i - 1] == `\r` {
|
if i != 0 && r.buf[i - 1] == `\r` {
|
||||||
x := i - 1
|
x := i - 1
|
||||||
line << r.buf[r.offset..x]
|
line << r.buf[r.offset..x]
|
||||||
} else {
|
} else {
|
||||||
line << r.buf[r.offset..i]
|
line << r.buf[r.offset..i]
|
||||||
}
|
}
|
||||||
|
|
||||||
r.offset = i + 1
|
r.offset = i + 1
|
||||||
|
|
||||||
return line.bytestr()
|
return line.bytestr()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
line << r.buf[r.offset..i]
|
line << r.buf[r.offset..i]
|
||||||
|
|
||||||
r.offset = i
|
r.offset = i
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,29 +4,57 @@ module io
|
||||||
pub interface Reader {
|
pub interface Reader {
|
||||||
// read reads up to buf.len bytes and places
|
// read reads up to buf.len bytes and places
|
||||||
// them into buf.
|
// them into buf.
|
||||||
// A struct that implements this should return
|
// A type that implements this should return
|
||||||
// `none` on end of stream (EOF) instead of just returning 0
|
// `none` on end of stream (EOF) instead of just returning 0
|
||||||
read(mut buf []byte) ?int
|
read(mut buf []byte) ?int
|
||||||
}
|
}
|
||||||
|
|
||||||
// make_reader is a temp that converts a struct to a reader
|
// make_reader is a temp that converts a type to a reader
|
||||||
// (e.g. for use in struct initialisation)
|
// (e.g. for use in struct initialisation)
|
||||||
|
// (this shouldnt need to be a thing but until coercion gets made better
|
||||||
|
// it is required)
|
||||||
pub fn make_reader(r Reader) Reader {
|
pub fn make_reader(r Reader) Reader {
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
pub const (
|
|
||||||
eof_code = -1
|
|
||||||
eof = error_with_code('EOF', eof_code)
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
read_all_len = 10 * 1024
|
read_all_len = 10 * 1024
|
||||||
read_all_grow_len = 1024
|
read_all_grow_len = 1024
|
||||||
)
|
)
|
||||||
|
|
||||||
// read_all reads all available bytes from a reader
|
// ReadAllConfig allows options to be passed for the behaviour
|
||||||
pub fn read_all(r Reader) ?[]byte {
|
// of read_all
|
||||||
|
pub struct ReadAllConfig {
|
||||||
|
reader Reader
|
||||||
|
read_to_end_of_stream bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// read_all reads all bytes from a reader until either a 0 length read
|
||||||
|
// or if read_to_end_of_stream is true then the end of the stream (`none`)
|
||||||
|
pub fn read_all(config ReadAllConfig) ?[]byte {
|
||||||
|
r := config.reader
|
||||||
|
read_till_eof := config.read_to_end_of_stream
|
||||||
|
|
||||||
|
mut b := []byte{len: read_all_len}
|
||||||
|
mut read := 0
|
||||||
|
for {
|
||||||
|
new_read := r.read(mut b[read..]) or {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
read += new_read
|
||||||
|
if !read_till_eof && read == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if b.len == read {
|
||||||
|
b.grow_len(read_all_grow_len)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b[..read]
|
||||||
|
}
|
||||||
|
|
||||||
|
// read_any reads any available bytes from a reader
|
||||||
|
// (until the reader returns a read of 0 length)
|
||||||
|
pub fn read_any(r Reader) ?[]byte {
|
||||||
mut b := []byte{len: read_all_len}
|
mut b := []byte{len: read_all_len}
|
||||||
mut read := 0
|
mut read := 0
|
||||||
for {
|
for {
|
||||||
|
@ -38,7 +66,7 @@ pub fn read_all(r Reader) ?[]byte {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if b.len == read {
|
if b.len == read {
|
||||||
b.grow(read_all_grow_len)
|
b.grow_len(read_all_grow_len)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return b[..read]
|
return b[..read]
|
||||||
|
|
|
@ -9,9 +9,9 @@ mut:
|
||||||
|
|
||||||
fn (mut b Buf) read(mut buf []byte) ?int {
|
fn (mut b Buf) read(mut buf []byte) ?int {
|
||||||
if !(b.i < b.bytes.len) {
|
if !(b.i < b.bytes.len) {
|
||||||
return eof
|
return none
|
||||||
}
|
}
|
||||||
n := copy(buf, b.bytes[b.i..b.bytes.len])
|
n := copy(buf, b.bytes[b.i..])
|
||||||
b.i += n
|
b.i += n
|
||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
|
@ -20,21 +20,117 @@ fn test_read_all() {
|
||||||
buf := Buf{
|
buf := Buf{
|
||||||
bytes: '123'.repeat(10).bytes()
|
bytes: '123'.repeat(10).bytes()
|
||||||
}
|
}
|
||||||
res := read_all(buf) or {
|
res := read_all(reader: buf) or {
|
||||||
assert false
|
assert false
|
||||||
''.bytes()
|
''.bytes()
|
||||||
}
|
}
|
||||||
assert res == '123'.repeat(10).bytes()
|
assert res == '123'.repeat(10).bytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
TODO: This test failed by a bug of read_all
|
|
||||||
fn test_read_all_huge() {
|
fn test_read_all_huge() {
|
||||||
buf := Buf{bytes: "123".repeat(100000).bytes()}
|
buf := Buf{
|
||||||
res := read_all(buf) or {
|
bytes: '123'.repeat(100000).bytes()
|
||||||
|
}
|
||||||
|
res := read_all(reader: buf) or {
|
||||||
assert false
|
assert false
|
||||||
"".bytes()
|
''.bytes()
|
||||||
}
|
}
|
||||||
assert res == "123".repeat(100000).bytes()
|
assert res == '123'.repeat(100000).bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
struct StringReader {
|
||||||
|
text string
|
||||||
|
mut:
|
||||||
|
place int
|
||||||
|
}
|
||||||
|
|
||||||
|
fn (mut s StringReader) read(mut buf []byte) ?int {
|
||||||
|
if s.place >= s.text.len {
|
||||||
|
return none
|
||||||
|
}
|
||||||
|
read := copy(buf, s.text[s.place..].bytes())
|
||||||
|
s.place += read
|
||||||
|
return read
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
newline_count = 100000
|
||||||
|
)
|
||||||
|
|
||||||
|
fn test_stringreader() {
|
||||||
|
text := '12345\n'.repeat(newline_count)
|
||||||
|
mut s := StringReader{
|
||||||
|
text: text
|
||||||
|
}
|
||||||
|
mut r := new_buffered_reader({
|
||||||
|
reader: make_reader(s)
|
||||||
|
})
|
||||||
|
for i := 0; true; i++ {
|
||||||
|
if _ := r.read_line() {
|
||||||
|
} else {
|
||||||
|
assert i == newline_count
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _ := r.read_line() {
|
||||||
|
assert false
|
||||||
|
}
|
||||||
|
leftover := read_all(reader: r) or {
|
||||||
|
assert false
|
||||||
|
panic('bad')
|
||||||
|
}
|
||||||
|
if leftover.len > 0 {
|
||||||
|
assert false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_stringreader2() {
|
||||||
|
text := '12345\r\n'.repeat(newline_count)
|
||||||
|
mut s := StringReader{
|
||||||
|
text: text
|
||||||
|
}
|
||||||
|
mut r := new_buffered_reader({
|
||||||
|
reader: make_reader(s)
|
||||||
|
})
|
||||||
|
for i := 0; true; i++ {
|
||||||
|
if _ := r.read_line() {
|
||||||
|
} else {
|
||||||
|
assert i == newline_count
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _ := r.read_line() {
|
||||||
|
assert false
|
||||||
|
}
|
||||||
|
leftover := read_all(reader: io.make_reader(r)) or {
|
||||||
|
assert false
|
||||||
|
panic('bad')
|
||||||
|
}
|
||||||
|
if leftover.len > 0 {
|
||||||
|
assert false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_leftover() {
|
||||||
|
text := 'This is a test\r\nNice try!'
|
||||||
|
mut s := StringReader{
|
||||||
|
text: text
|
||||||
|
}
|
||||||
|
mut r := new_buffered_reader({
|
||||||
|
reader: make_reader(s)
|
||||||
|
})
|
||||||
|
_ := r.read_line() or {
|
||||||
|
assert false
|
||||||
|
panic('bad')
|
||||||
|
}
|
||||||
|
line2 := r.read_line() or {
|
||||||
|
assert false
|
||||||
|
panic('bad')
|
||||||
|
}
|
||||||
|
assert line2 == 'Nice try!'
|
||||||
|
if _ := r.read_line() {
|
||||||
|
assert false
|
||||||
|
panic('bad')
|
||||||
|
}
|
||||||
|
assert r.end_of_stream()
|
||||||
}
|
}
|
||||||
*/
|
|
||||||
|
|
|
@ -4,7 +4,6 @@ module io
|
||||||
pub interface ReaderWriter {
|
pub interface ReaderWriter {
|
||||||
// from Reader
|
// from Reader
|
||||||
read(mut buf []byte) ?int
|
read(mut buf []byte) ?int
|
||||||
|
|
||||||
// from Writer
|
// from Writer
|
||||||
write(buf []byte) ?int
|
write(buf []byte) ?int
|
||||||
}
|
}
|
||||||
|
@ -27,7 +26,10 @@ pub fn (mut r ReaderWriterImpl) write(buf []byte) ?int {
|
||||||
// make_readerwriter takes a rstream and a wstream and makes
|
// make_readerwriter takes a rstream and a wstream and makes
|
||||||
// an rwstream with them
|
// an rwstream with them
|
||||||
pub fn make_readerwriter(r Reader, w Writer) ReaderWriterImpl {
|
pub fn make_readerwriter(r Reader, w Writer) ReaderWriterImpl {
|
||||||
return {r: r, w: w}
|
return {
|
||||||
|
r: r
|
||||||
|
w: w
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Zzz_CoerceInterfaceTableGeneration {
|
struct Zzz_CoerceInterfaceTableGeneration {
|
||||||
|
|
|
@ -398,7 +398,7 @@ fn (req &Request) http_do(host string, method Method, path string) ?Response {
|
||||||
mut client := net.dial_tcp(host)?
|
mut client := net.dial_tcp(host)?
|
||||||
// TODO this really needs to be exposed somehow
|
// TODO this really needs to be exposed somehow
|
||||||
client.write(s.bytes())?
|
client.write(s.bytes())?
|
||||||
mut bytes := io.read_all(client)?
|
mut bytes := io.read_all(reader: client)?
|
||||||
client.close()
|
client.close()
|
||||||
return parse_response(bytes.bytestr())
|
return parse_response(bytes.bytestr())
|
||||||
}
|
}
|
||||||
|
|
|
@ -95,7 +95,7 @@ pub fn (mut c Client) quit() ? {
|
||||||
|
|
||||||
// expect_reply checks if the SMTP server replied with the expected reply code
|
// expect_reply checks if the SMTP server replied with the expected reply code
|
||||||
fn (c Client) expect_reply(expected ReplyCode) ? {
|
fn (c Client) expect_reply(expected ReplyCode) ? {
|
||||||
bytes := io.read_all(c.conn)?
|
bytes := io.read_all(reader: c.conn)?
|
||||||
|
|
||||||
str := bytes.bytestr().trim_space()
|
str := bytes.bytestr().trim_space()
|
||||||
$if smtp_debug? {
|
$if smtp_debug? {
|
||||||
|
|
|
@ -259,7 +259,7 @@ $config.content'
|
||||||
eprintln('sending:\n$message')
|
eprintln('sending:\n$message')
|
||||||
}
|
}
|
||||||
client.write(message.bytes()) ?
|
client.write(message.bytes()) ?
|
||||||
read := io.read_all(client) ?
|
read := io.read_all(reader: client) ?
|
||||||
$if debug_net_socket_client ? {
|
$if debug_net_socket_client ? {
|
||||||
eprintln('received:\n$read')
|
eprintln('received:\n$read')
|
||||||
}
|
}
|
||||||
|
|
|
@ -285,7 +285,7 @@ fn handle_conn<T>(mut conn net.TcpConn, mut app T) {
|
||||||
//}
|
//}
|
||||||
|
|
||||||
// read body
|
// read body
|
||||||
read_body := io.read_all(reader) or { []byte{} }
|
read_body := io.read_all(reader: reader) or { []byte{} }
|
||||||
body += read_body.bytestr()
|
body += read_body.bytestr()
|
||||||
|
|
||||||
break
|
break
|
||||||
|
|
Loading…
Reference in New Issue