From 6937074e7a3b11c9d5b59e5bd3e04aa09ba107bd Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Sun, 31 Oct 2021 12:58:55 +0200 Subject: [PATCH] strings,builder: reduce V cgen and parser memory consumption, fix strings.Builder leak (#12342) --- vlib/builtin/array.v | 5 +++++ vlib/strings/builder.c.v | 7 ++++--- vlib/v/depgraph/depgraph.v | 10 ++++++++-- vlib/v/parser/parser.v | 25 ++++++++++++++++++++----- vlib/v/scanner/scanner.v | 18 +++++++++++++----- vlib/v/tests/valgrind/strings_builder.v | 22 ++++++++++++++++++++++ 6 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 vlib/v/tests/valgrind/strings_builder.v diff --git a/vlib/builtin/array.v b/vlib/builtin/array.v index ae9af636b1..45c00a1266 100644 --- a/vlib/builtin/array.v +++ b/vlib/builtin/array.v @@ -238,6 +238,11 @@ pub fn (mut a array) delete_many(i int, size int) { vmemcpy(&byte(a.data) + i * a.element_size, &byte(old_data) + (i + size) * a.element_size, (a.len - i - size) * a.element_size) } + if a.flags.has(.noslices) { + unsafe { + free(old_data) + } + } a.len = new_size a.cap = new_cap } diff --git a/vlib/strings/builder.c.v b/vlib/strings/builder.c.v index b1fd3d6fff..c6041d0091 100644 --- a/vlib/strings/builder.c.v +++ b/vlib/strings/builder.c.v @@ -11,7 +11,9 @@ pub type Builder = []byte // new_builder returns a new string builder, with an initial capacity of `initial_size` pub fn new_builder(initial_size int) Builder { - return Builder([]byte{cap: initial_size}) + mut res := Builder([]byte{cap: initial_size}) + unsafe { res.flags.set(.noslices) } + return res } // write_ptr writes `len` bytes provided byteptr to the accumulated buffer @@ -161,9 +163,8 @@ pub fn (mut b Builder) str() string { pub fn (mut b Builder) free() { if b.data != 0 { unsafe { free(b.data) } - mut pd := &b.data unsafe { - (*pd) = voidptr(0) + b.data = voidptr(0) } } } diff --git a/vlib/v/depgraph/depgraph.v b/vlib/v/depgraph/depgraph.v index cc6a8eb9f7..f6706dae98 100644 --- a/vlib/v/depgraph/depgraph.v +++ b/vlib/v/depgraph/depgraph.v @@ -25,6 +25,12 @@ mut: data map[string][]string } +pub fn new_ordered_dependency_map() OrderedDepMap { + mut res := OrderedDepMap{} + unsafe { res.keys.flags.set(.noslices) } + return res +} + pub fn (mut o OrderedDepMap) set(name string, deps []string) { if name !in o.data { o.keys << name @@ -92,8 +98,8 @@ pub fn (mut graph DepGraph) add(mod string, deps []string) { } pub fn (graph &DepGraph) resolve() &DepGraph { - mut node_names := OrderedDepMap{} - mut node_deps := OrderedDepMap{} + mut node_names := new_ordered_dependency_map() + mut node_deps := new_ordered_dependency_map() for node in graph.nodes { node_names.add(node.name, node.deps) node_deps.add(node.name, node.deps) diff --git a/vlib/v/parser/parser.v b/vlib/v/parser/parser.v index 4d6830b64c..22c7e62df8 100644 --- a/vlib/v/parser/parser.v +++ b/vlib/v/parser/parser.v @@ -111,7 +111,9 @@ pub fn parse_comptime(text string, table &ast.Table, pref &pref.Preferences, sco errors: []errors.Error{} warnings: []errors.Warning{} } - return p.parse() + res := p.parse() + unsafe { p.free_scanner() } + return res } pub fn parse_text(text string, path string, table &ast.Table, comments_mode scanner.CommentsMode, pref &pref.Preferences) &ast.File { @@ -128,13 +130,23 @@ pub fn parse_text(text string, path string, table &ast.Table, comments_mode scan warnings: []errors.Warning{} } p.set_path(path) - return p.parse() + res := p.parse() + unsafe { p.free_scanner() } + return res } [unsafe] pub fn (mut p Parser) free() { + unsafe { p.free_scanner() } +} + +[unsafe] +pub fn (mut p Parser) free_scanner() { unsafe { - p.scanner.free() + if p.scanner != 0 { + p.scanner.free() + p.scanner = &scanner.Scanner(0) + } } } @@ -190,7 +202,9 @@ pub fn parse_file(path string, table &ast.Table, comments_mode scanner.CommentsM warnings: []errors.Warning{} } p.set_path(path) - return p.parse() + res := p.parse() + unsafe { p.free_scanner() } + return res } pub fn parse_vet_file(path string, table_ &ast.Table, pref &pref.Preferences) (&ast.File, []vet.Error) { @@ -225,6 +239,7 @@ pub fn parse_vet_file(path string, table_ &ast.Table, pref &pref.Preferences) (& } p.vet_errors << p.scanner.vet_errors file := p.parse() + unsafe { p.free_scanner() } return file, p.vet_errors } @@ -367,7 +382,7 @@ pub fn parse_files(paths []string, table &ast.Table, pref &pref.Preferences) []& } */ } - mut files := []&ast.File{} + mut files := []&ast.File{cap: paths.len} for path in paths { timers.start('parse_file $path') files << parse_file(path, table, .skip_comments, pref) diff --git a/vlib/v/scanner/scanner.v b/vlib/v/scanner/scanner.v index c8ad180e48..3116ede07e 100644 --- a/vlib/v/scanner/scanner.v +++ b/vlib/v/scanner/scanner.v @@ -151,7 +151,11 @@ fn (mut s Scanner) init_scanner() { [unsafe] pub fn (mut s Scanner) free() { unsafe { - s.text.free() + // NB: s.text is not freed here, because it is shared with all other util.read_file instances, + // and strings are not reference counted yet: + // s.text.free() + // .all_tokens however are not shared with anything, and can be freed: + s.all_tokens.free() } } @@ -1455,15 +1459,19 @@ pub fn verror(s string) { util.verror('scanner error', s) } +// codegen allows you to generate V code, so that it can be parsed, +// checked, markused, cgen-ed etc further, just like user's V code. pub fn (mut s Scanner) codegen(newtext string) { $if debug_codegen ? { eprintln('scanner.codegen:\n $newtext') } - // codegen makes sense only during normal compilation - // feeding code generated V code to vfmt or vdoc will - // cause them to output/document ephemeral stuff. if s.comments_mode == .skip_comments { - s.all_tokens.delete_last() // remove .eof from end of .all_tokens + // Calling codegen makes sense only during normal compilation, since + // feeding code generated V code to vfmt or vdoc will cause them to + // output/document ephemeral stuff. + for s.all_tokens.len > 0 && s.all_tokens.last().kind == .eof { + s.all_tokens.delete_last() + } s.text += newtext old_tidx := s.tidx s.tidx = s.all_tokens.len diff --git a/vlib/v/tests/valgrind/strings_builder.v b/vlib/v/tests/valgrind/strings_builder.v new file mode 100644 index 0000000000..5e54c079e1 --- /dev/null +++ b/vlib/v/tests/valgrind/strings_builder.v @@ -0,0 +1,22 @@ +import strings + +fn main() { + mut sb := strings.new_builder(4) + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + sb.write_string('abcd') + println(sb.str()) +}