os: make os module handle large files (#9439)
parent
91ea76797a
commit
69dff4b384
|
@ -120,6 +120,6 @@ pub fn used_tools_must_exist(tools []string) {
|
||||||
pub fn show_sizes_of_files(files []string) {
|
pub fn show_sizes_of_files(files []string) {
|
||||||
for f in files {
|
for f in files {
|
||||||
size := os.file_size(f)
|
size := os.file_size(f)
|
||||||
println('${size:10d} $f')
|
println('$size $f') // println('${size:10d} $f')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -70,5 +70,5 @@ pub fn read_any(r Reader) ?[]byte {
|
||||||
|
|
||||||
// RandomReader represents a stream of data that can be read from at a random location
|
// RandomReader represents a stream of data that can be read from at a random location
|
||||||
interface RandomReader {
|
interface RandomReader {
|
||||||
read_from(pos int, mut buf []byte) ?int
|
read_from(pos u64, mut buf []byte) ?int
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,5 +8,5 @@ pub interface Writer {
|
||||||
// RandomWriter represents a stream of data that can be wrote to
|
// RandomWriter represents a stream of data that can be wrote to
|
||||||
// at a random pos
|
// at a random pos
|
||||||
pub interface RandomWriter {
|
pub interface RandomWriter {
|
||||||
write_to(pos int, buf []byte) ?int
|
write_to(pos u64, buf []byte) ?int
|
||||||
}
|
}
|
||||||
|
|
103
vlib/os/file.c.v
103
vlib/os/file.c.v
|
@ -13,6 +13,10 @@ struct FileInfo {
|
||||||
size int
|
size int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn C.fseeko(voidptr, u64, int) int
|
||||||
|
|
||||||
|
fn C._fseeki64(voidptr, u64, int) int
|
||||||
|
|
||||||
// open_file can be used to open or create a file with custom flags and permissions and returns a `File` object.
|
// open_file can be used to open or create a file with custom flags and permissions and returns a `File` object.
|
||||||
pub fn open_file(path string, mode string, options ...int) ?File {
|
pub fn open_file(path string, mode string, options ...int) ?File {
|
||||||
mut flags := 0
|
mut flags := 0
|
||||||
|
@ -202,10 +206,30 @@ pub fn (mut f File) write_string(s string) ?int {
|
||||||
// write_to implements the RandomWriter interface.
|
// write_to implements the RandomWriter interface.
|
||||||
// It returns how many bytes were actually written.
|
// It returns how many bytes were actually written.
|
||||||
// It resets the seek position to the end of the file.
|
// It resets the seek position to the end of the file.
|
||||||
pub fn (mut f File) write_to(pos int, buf []byte) ?int {
|
pub fn (mut f File) write_to(pos u64, buf []byte) ?int {
|
||||||
if !f.is_opened {
|
if !f.is_opened {
|
||||||
return error('file is not opened')
|
return error('file is not opened')
|
||||||
}
|
}
|
||||||
|
$if x64 {
|
||||||
|
$if windows {
|
||||||
|
C._fseeki64(f.cfile, pos, C.SEEK_SET)
|
||||||
|
res := int(C.fwrite(buf.data, 1, buf.len, f.cfile))
|
||||||
|
if res == 0 && buf.len != 0 {
|
||||||
|
return error('0 bytes written')
|
||||||
|
}
|
||||||
|
C._fseeki64(f.cfile, 0, C.SEEK_END)
|
||||||
|
return res
|
||||||
|
} $else {
|
||||||
|
C.fseeko(f.cfile, pos, C.SEEK_SET)
|
||||||
|
res := int(C.fwrite(buf.data, 1, buf.len, f.cfile))
|
||||||
|
if res == 0 && buf.len != 0 {
|
||||||
|
return error('0 bytes written')
|
||||||
|
}
|
||||||
|
C.fseeko(f.cfile, 0, C.SEEK_END)
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$if x32 {
|
||||||
C.fseek(f.cfile, pos, C.SEEK_SET)
|
C.fseek(f.cfile, pos, C.SEEK_SET)
|
||||||
res := int(C.fwrite(buf.data, 1, buf.len, f.cfile))
|
res := int(C.fwrite(buf.data, 1, buf.len, f.cfile))
|
||||||
if res == 0 && buf.len != 0 {
|
if res == 0 && buf.len != 0 {
|
||||||
|
@ -214,6 +238,8 @@ pub fn (mut f File) write_to(pos int, buf []byte) ?int {
|
||||||
C.fseek(f.cfile, 0, C.SEEK_END)
|
C.fseek(f.cfile, 0, C.SEEK_END)
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
return error('Could not write to file')
|
||||||
|
}
|
||||||
|
|
||||||
// write_bytes writes `size` bytes to the file, starting from the address in `data`.
|
// write_bytes writes `size` bytes to the file, starting from the address in `data`.
|
||||||
// NB: write_bytes is unsafe and should be used carefully, since if you pass invalid
|
// NB: write_bytes is unsafe and should be used carefully, since if you pass invalid
|
||||||
|
@ -230,7 +256,7 @@ pub fn (mut f File) write_bytes(data voidptr, size int) int {
|
||||||
// pointers to it, it will cause your programs to segfault.
|
// pointers to it, it will cause your programs to segfault.
|
||||||
[deprecated: 'use File.write_ptr_at() instead']
|
[deprecated: 'use File.write_ptr_at() instead']
|
||||||
[unsafe]
|
[unsafe]
|
||||||
pub fn (mut f File) write_bytes_at(data voidptr, size int, pos int) int {
|
pub fn (mut f File) write_bytes_at(data voidptr, size int, pos u64) int {
|
||||||
return unsafe { f.write_ptr_at(data, size, pos) }
|
return unsafe { f.write_ptr_at(data, size, pos) }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -247,12 +273,28 @@ pub fn (mut f File) write_ptr(data voidptr, size int) int {
|
||||||
// NB: write_ptr_at is unsafe and should be used carefully, since if you pass invalid
|
// NB: write_ptr_at is unsafe and should be used carefully, since if you pass invalid
|
||||||
// pointers to it, it will cause your programs to segfault.
|
// pointers to it, it will cause your programs to segfault.
|
||||||
[unsafe]
|
[unsafe]
|
||||||
pub fn (mut f File) write_ptr_at(data voidptr, size int, pos int) int {
|
pub fn (mut f File) write_ptr_at(data voidptr, size int, pos u64) int {
|
||||||
|
$if x64 {
|
||||||
|
$if windows {
|
||||||
|
C._fseeki64(f.cfile, pos, C.SEEK_SET)
|
||||||
|
res := int(C.fwrite(data, 1, size, f.cfile))
|
||||||
|
C._fseeki64(f.cfile, 0, C.SEEK_END)
|
||||||
|
return res
|
||||||
|
} $else {
|
||||||
|
C.fseeko(f.cfile, pos, C.SEEK_SET)
|
||||||
|
res := int(C.fwrite(data, 1, size, f.cfile))
|
||||||
|
C.fseeko(f.cfile, 0, C.SEEK_END)
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$if x32 {
|
||||||
C.fseek(f.cfile, pos, C.SEEK_SET)
|
C.fseek(f.cfile, pos, C.SEEK_SET)
|
||||||
res := int(C.fwrite(data, 1, size, f.cfile))
|
res := int(C.fwrite(data, 1, size, f.cfile))
|
||||||
C.fseek(f.cfile, 0, C.SEEK_END)
|
C.fseek(f.cfile, 0, C.SEEK_END)
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
// **************************** Read ops ***************************
|
// **************************** Read ops ***************************
|
||||||
// read_bytes reads bytes from the beginning of the file.
|
// read_bytes reads bytes from the beginning of the file.
|
||||||
|
@ -262,7 +304,7 @@ pub fn (f &File) read_bytes(size int) []byte {
|
||||||
}
|
}
|
||||||
|
|
||||||
// read_bytes_at reads `size` bytes at the given position in the file.
|
// read_bytes_at reads `size` bytes at the given position in the file.
|
||||||
pub fn (f &File) read_bytes_at(size int, pos int) []byte {
|
pub fn (f &File) read_bytes_at(size int, pos u64) []byte {
|
||||||
mut arr := []byte{len: size}
|
mut arr := []byte{len: size}
|
||||||
nreadbytes := f.read_bytes_into(pos, mut arr) or {
|
nreadbytes := f.read_bytes_into(pos, mut arr) or {
|
||||||
// return err
|
// return err
|
||||||
|
@ -274,23 +316,51 @@ pub fn (f &File) read_bytes_at(size int, pos int) []byte {
|
||||||
// read_bytes_into fills `buf` with bytes at the given position in the file.
|
// read_bytes_into fills `buf` with bytes at the given position in the file.
|
||||||
// `buf` *must* have length greater than zero.
|
// `buf` *must* have length greater than zero.
|
||||||
// Returns the number of read bytes, or an error.
|
// Returns the number of read bytes, or an error.
|
||||||
pub fn (f &File) read_bytes_into(pos int, mut buf []byte) ?int {
|
pub fn (f &File) read_bytes_into(pos u64, mut buf []byte) ?int {
|
||||||
if buf.len == 0 {
|
if buf.len == 0 {
|
||||||
panic(@FN + ': `buf.len` == 0')
|
panic(@FN + ': `buf.len` == 0')
|
||||||
}
|
}
|
||||||
|
$if x64 {
|
||||||
|
$if windows {
|
||||||
// Note: fseek errors if pos == os.file_size, which we accept
|
// Note: fseek errors if pos == os.file_size, which we accept
|
||||||
C.fseek(f.cfile, pos, C.SEEK_SET)
|
C._fseeki64(f.cfile, pos, C.SEEK_SET)
|
||||||
// errno is only set if fread fails, so clear it first to tell
|
// errno is only set if fread fails, so clear it first to tell
|
||||||
C.errno = 0
|
C.errno = 0
|
||||||
nbytes := int(C.fread(buf.data, 1, buf.len, f.cfile))
|
nbytes := int(C.fread(buf.data, 1, buf.len, f.cfile))
|
||||||
if C.errno != 0 {
|
if C.errno != 0 {
|
||||||
return error(posix_get_error_msg(C.errno))
|
return error(posix_get_error_msg(C.errno))
|
||||||
}
|
}
|
||||||
|
$if debug {
|
||||||
|
C._fseeki64(f.cfile, 0, C.SEEK_SET)
|
||||||
|
}
|
||||||
|
return nbytes
|
||||||
|
} $else {
|
||||||
|
C.fseeko(f.cfile, pos, C.SEEK_SET)
|
||||||
|
C.errno = 0
|
||||||
|
nbytes := int(C.fread(buf.data, 1, buf.len, f.cfile))
|
||||||
|
if C.errno != 0 {
|
||||||
|
return error(posix_get_error_msg(C.errno))
|
||||||
|
}
|
||||||
|
$if debug {
|
||||||
|
C.fseeko(f.cfile, 0, C.SEEK_SET)
|
||||||
|
}
|
||||||
|
return nbytes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$if x32 {
|
||||||
|
C.fseek(f.cfile, pos, C.SEEK_SET)
|
||||||
|
C.errno = 0
|
||||||
|
nbytes := int(C.fread(buf.data, 1, buf.len, f.cfile))
|
||||||
|
if C.errno != 0 {
|
||||||
|
return error(posix_get_error_msg(C.errno))
|
||||||
|
}
|
||||||
$if debug {
|
$if debug {
|
||||||
C.fseek(f.cfile, 0, C.SEEK_SET)
|
C.fseek(f.cfile, 0, C.SEEK_SET)
|
||||||
}
|
}
|
||||||
return nbytes
|
return nbytes
|
||||||
}
|
}
|
||||||
|
return error('Could not read file')
|
||||||
|
}
|
||||||
|
|
||||||
// read implements the Reader interface.
|
// read implements the Reader interface.
|
||||||
pub fn (f &File) read(mut buf []byte) ?int {
|
pub fn (f &File) read(mut buf []byte) ?int {
|
||||||
|
@ -307,15 +377,30 @@ pub fn (f &File) read(mut buf []byte) ?int {
|
||||||
|
|
||||||
// read_at reads `buf.len` bytes starting at file byte offset `pos`, in `buf`.
|
// read_at reads `buf.len` bytes starting at file byte offset `pos`, in `buf`.
|
||||||
[deprecated: 'use File.read_from() instead']
|
[deprecated: 'use File.read_from() instead']
|
||||||
pub fn (f &File) read_at(pos int, mut buf []byte) ?int {
|
pub fn (f &File) read_at(pos u64, mut buf []byte) ?int {
|
||||||
return f.read_from(pos, mut buf)
|
return f.read_from(pos, mut buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
// read_from implements the RandomReader interface.
|
// read_from implements the RandomReader interface.
|
||||||
pub fn (f &File) read_from(pos int, mut buf []byte) ?int {
|
pub fn (f &File) read_from(pos u64, mut buf []byte) ?int {
|
||||||
if buf.len == 0 {
|
if buf.len == 0 {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
$if x64 {
|
||||||
|
$if windows {
|
||||||
|
C._fseeki64(f.cfile, pos, C.SEEK_SET)
|
||||||
|
} $else {
|
||||||
|
C.fseeko(f.cfile, pos, C.SEEK_SET)
|
||||||
|
}
|
||||||
|
|
||||||
|
C.errno = 0
|
||||||
|
nbytes := int(C.fread(buf.data, 1, buf.len, f.cfile))
|
||||||
|
if C.errno != 0 {
|
||||||
|
return error(posix_get_error_msg(C.errno))
|
||||||
|
}
|
||||||
|
return nbytes
|
||||||
|
}
|
||||||
|
$if x32 {
|
||||||
C.fseek(f.cfile, pos, C.SEEK_SET)
|
C.fseek(f.cfile, pos, C.SEEK_SET)
|
||||||
C.errno = 0
|
C.errno = 0
|
||||||
nbytes := int(C.fread(buf.data, 1, buf.len, f.cfile))
|
nbytes := int(C.fread(buf.data, 1, buf.len, f.cfile))
|
||||||
|
@ -324,6 +409,8 @@ pub fn (f &File) read_from(pos int, mut buf []byte) ?int {
|
||||||
}
|
}
|
||||||
return nbytes
|
return nbytes
|
||||||
}
|
}
|
||||||
|
return error('Could not read file')
|
||||||
|
}
|
||||||
|
|
||||||
// **************************** Utility ops ***********************
|
// **************************** Utility ops ***********************
|
||||||
// flush writes any buffered unwritten data left in the file stream.
|
// flush writes any buffered unwritten data left in the file stream.
|
||||||
|
|
|
@ -25,9 +25,19 @@ fn C.CopyFile(&u32, &u32, int) int
|
||||||
|
|
||||||
fn C.execvp(file charptr, argv &charptr) int
|
fn C.execvp(file charptr, argv &charptr) int
|
||||||
|
|
||||||
|
// fn C.lstat(charptr, voidptr) u64
|
||||||
|
|
||||||
|
fn C._wstat64(charptr, voidptr) u64
|
||||||
|
|
||||||
// fn C.proc_pidpath(int, byteptr, int) int
|
// fn C.proc_pidpath(int, byteptr, int) int
|
||||||
struct C.stat {
|
struct C.stat {
|
||||||
st_size int
|
st_size u64
|
||||||
|
st_mode u32
|
||||||
|
st_mtime int
|
||||||
|
}
|
||||||
|
|
||||||
|
struct C.__stat64 {
|
||||||
|
st_size u64
|
||||||
st_mode u32
|
st_mode u32
|
||||||
st_mtime int
|
st_mtime int
|
||||||
}
|
}
|
||||||
|
@ -101,20 +111,33 @@ pub fn read_file(path string) ?string {
|
||||||
|
|
||||||
// ***************************** OS ops ************************
|
// ***************************** OS ops ************************
|
||||||
// file_size returns the size of the file located in `path`.
|
// file_size returns the size of the file located in `path`.
|
||||||
pub fn file_size(path string) int {
|
pub fn file_size(path string) u64 {
|
||||||
mut s := C.stat{}
|
mut s := C.stat{}
|
||||||
unsafe {
|
unsafe {
|
||||||
|
$if x64 {
|
||||||
$if windows {
|
$if windows {
|
||||||
$if tinyc {
|
mut swin := C.__stat64{}
|
||||||
C.stat(charptr(path.str), &s)
|
C._wstat64(path.to_wide(), voidptr(&swin))
|
||||||
|
return swin.st_size
|
||||||
} $else {
|
} $else {
|
||||||
|
C.stat(charptr(path.str), &s)
|
||||||
|
return u64(s.st_size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$if x32 {
|
||||||
|
$if debug {
|
||||||
|
println('Using os.file_size() on 32bit systems may not work on big files.')
|
||||||
|
}
|
||||||
|
$if windows {
|
||||||
C._wstat(path.to_wide(), voidptr(&s))
|
C._wstat(path.to_wide(), voidptr(&s))
|
||||||
}
|
return u64(s.st_size)
|
||||||
} $else {
|
} $else {
|
||||||
C.stat(charptr(path.str), &s)
|
C.stat(charptr(path.str), &s)
|
||||||
|
return u64(s.st_size)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return s.st_size
|
}
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// mv moves files or folders from `src` to `dst`.
|
// mv moves files or folders from `src` to `dst`.
|
||||||
|
@ -172,7 +195,9 @@ pub fn cp(src string, dst string) ? {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
from_attr := C.stat{}
|
from_attr := C.stat{}
|
||||||
unsafe { C.stat(charptr(src.str), &from_attr) }
|
unsafe {
|
||||||
|
C.stat(charptr(src.str), &from_attr)
|
||||||
|
}
|
||||||
if C.chmod(charptr(dst.str), from_attr.st_mode) < 0 {
|
if C.chmod(charptr(dst.str), from_attr.st_mode) < 0 {
|
||||||
return error_with_code('failed to set permissions for $dst', int(-1))
|
return error_with_code('failed to set permissions for $dst', int(-1))
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue