vieter-v
/
v
2
0
Fork 0
Code Issues Pull Requests Releases Activity

fmt: comments

pull/3883/head
Alexander Medvednikov 2020-02-29 17:51:35 +01:00
parent 5a5f4ce99a
commit d5faf36aa9
9 changed files with 110 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/tools/vfmt.v
View File

@ -153,7 +153,7 @@ fn (foptions &FormatOptions) format_file(file string) {
eprintln('vfmt2 running fmt.fmt over file: $file')
}
table := table.new_table()
file_ast := parser.parse_file(file, table)
file_ast := parser.parse_file(file, table, .parse_comments)
formatted_content := fmt.fmt(file_ast, table)
file_name := filepath.filename(file)
vfmt_output_path := filepath.join(os.tmpdir(), 'vfmt_' + file_name)
@ -302,7 +302,7 @@ Options:
-diff display only diffs between the formatted source and the original source.
-l list files whose formatting differs from vfmt.
-w write result to (source) file(s) instead of to stdout.
-2 Use the new V parser/vfmt. NB: this is EXPERIMENTAL for now.
-2 Use the new V parser/vfmt. NB: this is EXPERIMENTAL for now.
The new vfmt is much faster and more forgiving.
It also may EAT some of your code for now.
Please be carefull, and make frequent BACKUPS, when running with -vfmt2 .

4
vlib/compiler/aparser.v
View File

@ -22,9 +22,10 @@ struct Parser {
// the #include directives in the parsed .v file
file_pcguard string
v &V
pref &pref.Preferences // Preferences shared from V struct
pref &pref.Preferences
mut:
scanner &Scanner
// Preferences shared from V struct
tokens []Token
token_idx int
prev_stuck_token_idx int
@ -261,6 +262,7 @@ fn (p mut Parser) next() {
// (only when vfmt compile time flag is enabled, otherwise this function
// is not even generated)
p.fnext()
//
p.prev_tok2 = p.prev_tok
p.prev_tok = p.tok
p.scanner.prev_tok = p.tok

6
vlib/compiler/scanner.v
View File

@ -137,7 +137,7 @@ fn (s mut Scanner) ident_bin_number() string {
if !c.is_bin_digit() && c != num_sep {
if (!c.is_digit() && !c.is_letter()) || s.inside_string {
break
}
}
else if !has_wrong_digit {
has_wrong_digit = true
first_wrong_digit = c
@ -166,7 +166,7 @@ fn (s mut Scanner) ident_hex_number() string {
if !c.is_hex_digit() && c != num_sep {
if !c.is_letter() || s.inside_string {
break
}
}
else if !has_wrong_digit {
has_wrong_digit = true
first_wrong_digit = c
@ -195,7 +195,7 @@ fn (s mut Scanner) ident_oct_number() string {
if !c.is_oct_digit() && c != num_sep {
if (!c.is_digit() && !c.is_letter()) || s.inside_string {
break
}
}
else if !has_wrong_digit {
has_wrong_digit = true
first_wrong_digit = c

3
vlib/v/ast/str.v
View File

@ -14,6 +14,9 @@ import (
pub fn (node &FnDecl) str(t &table.Table) string {
mut f := strings.new_builder(30)
if node.is_pub {
f.write('pub ')
}
mut receiver := ''
if node.is_method {
sym := t.get_type_symbol(node.receiver.typ)

6
vlib/v/doc/doc.v
View File

@ -19,7 +19,7 @@ mut:
stmts []ast.Stmt // all module statements from all files
}
type FilterFn fn(node ast.FnDecl) bool
type FilterFn fn(node ast.FnDecl)bool
pub fn doc(mod string, table &table.Table) string {
mut d := Doc{
@ -45,7 +45,7 @@ pub fn doc(mod string, table &table.Table) string {
if file.ends_with('_test.v') || file.ends_with('_windows.v') || file.ends_with('_macos.v') {
continue
}
file_ast := parser.parse_file(filepath.join(path,file), table)
file_ast := parser.parse_file(filepath.join(path,file), table, .skip_comments)
d.stmts << file_ast.stmts
}
d.print_fns()
@ -92,7 +92,7 @@ fn (d Doc) get_fn_signatures(filter_fn FilterFn) []string {
}
}
else {}
}
}
}
fn_signatures.sort()
return fn_signatures

6
vlib/v/fmt/fmt.v
View File

@ -195,6 +195,9 @@ fn (f mut Fmt) stmt(node ast.Stmt) {
f.stmts(it.stmts)
f.writeln('}')
}
ast.LineComment {
f.writeln('// $it.text')
}
ast.Return {
f.write('return')
// multiple returns
@ -244,6 +247,9 @@ fn (f mut Fmt) stmt(node ast.Stmt) {
}
fn (f mut Fmt) struct_decl(node ast.StructDecl) {
if node.is_pub {
f.write('pub ')
}
f.writeln('struct $node.name {')
mut max := 0
for field in node.fields {

12
vlib/v/fmt/fmt_test.v
View File

@ -15,7 +15,7 @@ const (
fn test_fmt() {
fmt_message := 'vfmt tests'
eprintln(term.header(fmt_message,'-'))
eprintln(term.header(fmt_message, '-'))
vexe := os.getenv('VEXE')
if vexe.len == 0 || !os.exists(vexe) {
eprintln('VEXE must be set')
@ -23,11 +23,13 @@ fn test_fmt() {
}
vroot := filepath.dir(vexe)
tmpfolder := os.tmpdir()
diff_cmd := find_working_diff_command() or { '' }
diff_cmd := find_working_diff_command() or {
''
}
mut fmt_bench := benchmark.new_benchmark()
// Lookup the existing test _input.vv files:
input_files := os.walk_ext('$vroot/vlib/v/fmt/tests', '_input.vv')
fmt_bench.set_total_expected_steps( input_files.len )
fmt_bench.set_total_expected_steps(input_files.len)
for istep, ipath in input_files {
fmt_bench.cstep = istep
fmt_bench.step()
@ -39,12 +41,12 @@ fn test_fmt() {
continue
}
expected_ocontent := os.read_file(opath) or {
fmt_bench.fail()
fmt_bench.fail()
eprintln(fmt_bench.step_message_fail('cannot read from ${opath}'))
continue
}
table := table.new_table()
file_ast := parser.parse_file(ipath, table)
file_ast := parser.parse_file(ipath, table, .skip_comments)
result_ocontent := fmt.fmt(file_ast, table)
if expected_ocontent != result_ocontent {
fmt_bench.fail()

58
vlib/v/parser/parser.v
View File

@ -46,11 +46,12 @@ mut:
expected_type table.Type
scope &ast.Scope
imports map[string]string
ast_imports []ast.Import
}
// for tests
pub fn parse_stmt(text string, table &table.Table, scope &ast.Scope) ast.Stmt {
s := scanner.new_scanner(text)
s := scanner.new_scanner(text, .skip_comments)
mut p := Parser{
scanner: s
table: table
@ -64,10 +65,15 @@ pub fn parse_stmt(text string, table &table.Table, scope &ast.Scope) ast.Stmt {
return p.stmt()
}
pub fn parse_file(path string, table &table.Table) ast.File {
pub fn parse_file(path string, table &table.Table, comments_mode scanner.CommentsMode) ast.File {
// println('parse_file("$path")')
text := os.read_file(path) or {
panic(err)
}
mut stmts := []ast.Stmt
mut p := Parser{
scanner: scanner.new_scanner_file(path)
// scanner: scanner.new_scanner(text, comments_mode)
scanner: scanner.new_scanner_file(path, comments_mode)
table: table
file_name: path
pref: &pref.Preferences{}
@ -75,6 +81,8 @@ pub fn parse_file(path string, table &table.Table) ast.File {
start_pos: 0
parent: 0
}
// comments_mode: comments_mode
}
p.read_first_token()
// p.scope = &ast.Scope{start_pos: p.tok.position(), parent: 0}
@ -84,10 +92,12 @@ pub fn parse_file(path string, table &table.Table) ast.File {
p.mod = module_decl.name
p.builtin_mod = p.mod == 'builtin'
// imports
/*
mut imports := []ast.Import
for p.tok.kind == .key_import {
imports << p.import_stmt()
}
*/
// TODO: import only mode
for {
// res := s.scan()
@ -104,7 +114,7 @@ pub fn parse_file(path string, table &table.Table) ast.File {
return ast.File{
path: path
mod: module_decl
imports: imports
imports: p.ast_imports
stmts: stmts
scope: p.scope
}
@ -113,7 +123,7 @@ pub fn parse_file(path string, table &table.Table) ast.File {
pub fn parse_files(paths []string, table &table.Table) []ast.File {
mut files := []ast.File
for path in paths {
files << parse_file(path, table)
files << parse_file(path, table, .skip_comments)
}
return files
}
@ -164,12 +174,20 @@ pub fn (p mut Parser) parse_block() []ast.Stmt {
}
fn (p mut Parser) next() {
// for {
p.tok = p.peek_tok
p.peek_tok = p.scanner.scan()
// if !(p.tok.kind in [.line_comment, .mline_comment]) {
// break
// }
// }
// println(p.tok.str())
}
fn (p mut Parser) check(expected token.Kind) {
// for p.tok.kind in [.line_comment, .mline_comment] {
// p.next()
// }
if p.tok.kind != expected {
s := 'syntax error: unexpected `${p.tok.kind.str()}`, expecting `${expected.str()}`'
p.error(s)
@ -211,6 +229,14 @@ pub fn (p mut Parser) top_stmt() ast.Stmt {
.lsbr {
return p.attr()
}
.key_module {
return p.module_decl()
}
.key_import {
node := p.import_stmt()
p.ast_imports << node
return node[0]
}
.key_global {
return p.global_decl()
}
@ -239,10 +265,7 @@ pub fn (p mut Parser) top_stmt() ast.Stmt {
return p.struct_decl()
}
.line_comment {
// p.next()
return ast.LineComment{
text: p.scanner.line_comment
}
return p.line_comment()
}
.mline_comment {
// p.next()
@ -258,6 +281,14 @@ pub fn (p mut Parser) top_stmt() ast.Stmt {
}
}
pub fn (p mut Parser) line_comment() ast.LineComment {
text := p.tok.lit
p.next()
return ast.LineComment{
text: text
}
}
pub fn (p mut Parser) stmt() ast.Stmt {
match p.tok.kind {
.key_assert {
@ -273,6 +304,9 @@ pub fn (p mut Parser) stmt() ast.Stmt {
.key_for {
return p.for_statement()
}
.line_comment {
return p.line_comment()
}
.key_return {
return p.return_stmt()
}
@ -553,7 +587,8 @@ pub fn (p mut Parser) name_expr() ast.Expr {
if p.peek_tok.kind == .dot && (is_c || p.known_import(p.tok.lit) || p.mod.all_after('.') == p.tok.lit) {
if is_c {
mod = 'C'
} else {
}
else {
// prepend the full import
mod = p.imports[p.tok.lit]
}
@ -1472,7 +1507,8 @@ fn (p mut Parser) struct_decl() ast.StructDecl {
p.check(.rcbr)
if is_c {
name = 'C.$name'
} else {
}
else {
name = p.prepend_mod(name)
}
t := table.TypeSymbol{

52
vlib/v/scanner/scanner.v
View File

@ -41,9 +41,16 @@ mut:
is_vh bool // Keep newlines
is_fmt bool // Used only for skipping ${} in strings, since we need literal
// string values when generating formatted code.
comments_mode CommentsMode
}
pub enum CommentsMode {
skip_comments
parse_comments
}
// new scanner from file.
pub fn new_scanner_file(file_path string) &Scanner {
pub fn new_scanner_file(file_path string, comments_mode CommentsMode) &Scanner {
if !os.exists(file_path) {
verror("$file_path doesn't exist")
}
@ -60,7 +67,7 @@ pub fn new_scanner_file(file_path string) &Scanner {
raw_text = tos(c_text[offset_from_begin], vstrlen(c_text) - offset_from_begin)
}
}
mut s := new_scanner(raw_text)
mut s := new_scanner(raw_text, comments_mode) // .skip_comments)
// s.init_fmt()
s.file_path = file_path
return s
@ -70,13 +77,14 @@ const (
is_fmt = os.getenv('VEXE').contains('vfmt')
)
// new scanner from string.
pub fn new_scanner(text string) &Scanner {
pub fn new_scanner(text string, comments_mode CommentsMode) &Scanner {
return &Scanner{
text: text
print_line_on_error: true
print_colored_error: true
print_rel_paths_on_error: true
is_fmt: is_fmt
comments_mode: comments_mode
}
}
@ -105,7 +113,7 @@ const (
)
fn filter_num_sep(txt byteptr, start int, end int) string {
unsafe {
unsafe{
mut b := malloc(end - start + 1) // add a byte for the endstring 0
mut i := start
mut i1 := 0
@ -134,7 +142,7 @@ fn (s mut Scanner) ident_bin_number() string {
if !c.is_bin_digit() && c != num_sep {
if (!c.is_digit() && !c.is_letter()) || s.inside_string {
break
}
}
else if !has_wrong_digit {
has_wrong_digit = true
first_wrong_digit = c
@ -163,7 +171,7 @@ fn (s mut Scanner) ident_hex_number() string {
if !c.is_hex_digit() && c != num_sep {
if !c.is_letter() || s.inside_string {
break
}
}
else if !has_wrong_digit {
has_wrong_digit = true
first_wrong_digit = c
@ -192,7 +200,7 @@ fn (s mut Scanner) ident_oct_number() string {
if !c.is_oct_digit() && c != num_sep {
if (!c.is_digit() && !c.is_letter()) || s.inside_string {
break
}
}
else if !has_wrong_digit {
has_wrong_digit = true
first_wrong_digit = c
@ -332,6 +340,9 @@ fn (s mut Scanner) end_of_file() token.Token {
}
pub fn (s mut Scanner) scan() token.Token {
// if s.comments_mode == .parse_comments {
// println('\nscan()')
// }
// if s.line_comment != '' {
// s.fgenln('// LC "$s.line_comment"')
// s.line_comment = ''
@ -400,17 +411,17 @@ pub fn (s mut Scanner) scan() token.Token {
// `123`, `.123`
else if c.is_digit() || (c == `.` && nextc.is_digit()) {
if !s.inside_string {
// In C ints with `0` prefix are octal (in V they're decimal), so discarding heading zeros is needed.
// In C ints with `0` prefix are octal (in V they're decimal), so discarding heading zeros is needed.
mut start_pos := s.pos
for start_pos < s.text.len && s.text[start_pos] == `0` {
start_pos++
}
mut prefix_zero_num := start_pos - s.pos // how many prefix zeros should be jumped
mut prefix_zero_num := start_pos - s.pos // how many prefix zeros should be jumped
// for 0b, 0o, 0x the heading zero shouldn't be jumped
if start_pos == s.text.len || (c == `0` && !s.text[start_pos].is_digit()) {
prefix_zero_num--
}
s.pos += prefix_zero_num // jump these zeros
s.pos += prefix_zero_num // jump these zeros
}
num := s.ident_number()
return s.scan_res(.number, num)
@ -712,11 +723,19 @@ pub fn (s mut Scanner) scan() token.Token {
start := s.pos + 1
s.ignore_line()
s.line_comment = s.text[start + 1..s.pos]
s.line_comment = s.line_comment.trim_space()
if s.is_fmt {
s.pos-- // fix line_nr, \n was read, and the comment is marked on the next line
s.line_nr--
return s.scan_res(.line_comment, s.line_comment)
// if s.comments_mode == .parse_comments {
// println('line c $s.line_comment')
// }
comment := s.line_comment.trim_space()
// s.line_comment = comment
if s.comments_mode == .parse_comments {
// println('line c "$comment" z=')
// fix line_nr, \n was read, and the comment is marked
// on the next line
s.pos--
// println("'" + s.text[s.pos].str() + "'")
// s.line_nr--
return s.scan_res(.line_comment, comment)
}
// s.fgenln('// ${s.prev_tok.str()} "$s.line_comment"')
// Skip the comment (return the next token)
@ -748,7 +767,8 @@ pub fn (s mut Scanner) scan() token.Token {
s.pos++
end := s.pos + 1
comment := s.text[start..end]
if s.is_fmt {
// if s.is_fmt {
if false && s.comments_mode == .parse_comments {
s.line_comment = comment
return s.scan_res(.mline_comment, s.line_comment)
}