tools: add cmd/tools/measure/scanner_speed.v and cmd/tools/measure/parser_speed.v

master
Delyan Angelov 2022-06-02 13:35:13 +03:00
parent 9a0ec7f367
commit 031629faa1
No known key found for this signature in database
GPG Key ID: 66886C0F12D595ED
3 changed files with 115 additions and 5 deletions

View File

@ -0,0 +1,67 @@
import os
import time
import v.ast
import v.pref
import v.parser
import v.errors
import v.scanner
fn main() {
files := os.args#[1..]
if files.len > 0 && files[0].starts_with('@') {
lst_path := files[0].all_after('@')
listed_files := os.read_file(lst_path)?.split('\n')
process_files(listed_files)?
return
}
process_files(files)?
}
fn process_files(files []string) ? {
mut table := ast.new_table()
mut pref := pref.new_preferences()
pref.is_fmt = true
pref.skip_warnings = true
pref.output_mode = .silent
mut sw := time.new_stopwatch()
mut total_us := i64(0)
mut total_bytes := i64(0)
mut total_tokens := i64(0)
for f in files {
if f == '' {
continue
}
if f.ends_with('_test.v') {
continue
}
// do not measure the scanning, but only the parsing:
mut p := new_parser(f, .skip_comments, table, pref)
///
sw.restart()
_ := p.parse()
f_us := sw.elapsed().microseconds()
///
total_us += f_us
total_bytes += p.scanner.text.len
total_tokens += p.scanner.all_tokens.len
println('${f_us:10}us ${p.scanner.all_tokens.len:10} ${p.scanner.text.len:10} ${(f64(p.scanner.text.len) / p.scanner.all_tokens.len):7.3} ${p.errors.len:4} $f')
}
println('${total_us:10}us ${total_tokens:10} ${total_bytes:10} ${(f64(total_tokens) / total_bytes):7.3} | speed: ${(f64(total_bytes) / total_us):2.5f} MB/s')
}
fn new_parser(path string, comments_mode scanner.CommentsMode, table &ast.Table, pref &pref.Preferences) &parser.Parser {
mut p := &parser.Parser{
scanner: scanner.new_scanner_file(path, comments_mode, pref) or { panic(err) }
comments_mode: comments_mode
table: table
pref: pref
scope: &ast.Scope{
start_pos: 0
parent: table.global_scope
}
errors: []errors.Error{}
warnings: []errors.Warning{}
}
p.set_path(path)
return p
}

View File

@ -0,0 +1,42 @@
import os
import time
import v.scanner
import v.pref
fn main() {
files := os.args#[1..]
if files.len > 0 && files[0].starts_with('@') {
lst_path := files[0].all_after('@')
listed_files := os.read_file(lst_path)?.split('\n')
process_files(listed_files)?
return
}
process_files(files)?
}
fn process_files(files []string) ? {
mut pref := pref.new_preferences()
pref.is_fmt = true
pref.skip_warnings = true
pref.output_mode = .silent
mut sw := time.new_stopwatch()
mut total_us := i64(0)
mut total_bytes := i64(0)
mut total_tokens := i64(0)
for f in files {
if f == '' {
continue
}
if f.ends_with('_test.v') {
continue
}
sw.restart()
s := scanner.new_scanner_file(f, .skip_comments, pref)?
f_us := sw.elapsed().microseconds()
total_us += f_us
total_bytes += s.text.len
total_tokens += s.all_tokens.len
println('${f_us:10}us ${s.all_tokens.len:10} ${s.text.len:10} ${(f64(s.text.len) / s.all_tokens.len):7.3f} $f')
}
println('${total_us:10}us ${total_tokens:10} ${total_bytes:10} ${(f64(total_tokens) / total_bytes):7.3f} | speed: ${(f64(total_bytes) / total_us):2.5f} MB/s')
}

View File

@ -23,7 +23,6 @@ mut:
file_name_dir string // "/home/user" file_name_dir string // "/home/user"
unique_prefix string // a hash of p.file_name, used for making anon fn generation unique unique_prefix string // a hash of p.file_name, used for making anon fn generation unique
file_backend_mode ast.Language // .c for .c.v|.c.vv|.c.vsh files; .js for .js.v files, .amd64/.rv32/other arches for .amd64.v/.rv32.v/etc. files, .v otherwise. file_backend_mode ast.Language // .c for .c.v|.c.vv|.c.vsh files; .js for .js.v files, .amd64/.rv32/other arches for .amd64.v/.rv32.v/etc. files, .v otherwise.
scanner &scanner.Scanner
comments_mode scanner.CommentsMode = .skip_comments comments_mode scanner.CommentsMode = .skip_comments
// see comment in parse_file // see comment in parse_file
tok token.Token tok token.Token
@ -78,10 +77,6 @@ mut:
returns bool returns bool
is_stmt_ident bool // true while the beginning of a statement is an ident/selector is_stmt_ident bool // true while the beginning of a statement is an ident/selector
expecting_type bool // `is Type`, expecting type expecting_type bool // `is Type`, expecting type
errors []errors.Error
warnings []errors.Warning
notices []errors.Notice
vet_errors []vet.Error
cur_fn_name string cur_fn_name string
label_names []string label_names []string
name_error bool // indicates if the token is not a name or the name is on another line name_error bool // indicates if the token is not a name or the name is on another line
@ -95,6 +90,12 @@ mut:
if_cond_comments []ast.Comment if_cond_comments []ast.Comment
script_mode bool script_mode bool
script_mode_start_token token.Token script_mode_start_token token.Token
pub mut:
scanner &scanner.Scanner
errors []errors.Error
warnings []errors.Warning
notices []errors.Notice
vet_errors []vet.Error
} }
__global codegen_files = []&ast.File{} __global codegen_files = []&ast.File{}