vdoc: markdown, html, json generation + lots of fixes

cmd/v/help/default.txt

@@ -18,6 +18,7 @@ V supports the following commands:
    test              Run all test files in the provided directory.
    fmt               Format the V code provided.
    doc               Generate the documentation for a V module.
+   vlib-docs         Generate and open the documentation of all the vlib modules.
    repl              Run the REPL.
 * Installation/self updating:
    symlink           Create a symbolic link for V.
@@ -37,4 +38,4 @@ Use "v help <command>" for more information about a command, example: `v help bu
 Use "v help other" to see less frequently used commands.
 
 Note: Help is required to write more help topics.
-Only build, fmt, run, test, search, install, remove, update, bin2v are properly documented currently.
+Only build, doc, fmt, run, test, search, install, remove, update, bin2v are properly documented currently.

cmd/v/help/doc.txt

@@ -0,0 +1,16 @@
+Usage: 
+  v doc [flags] module_name output_file[.json|.md|.html]
+  v doc [flags] path_to_source.v output_file[.json|.md|.html]
+  v doc [flags] ./directory output_file[.json|.md|.html]
+  v doc [flags] . output_file[.json|.md|.html]
+
+Generates the documentation of a given directory, module, or V source file 
+and prints or saves them to its desired format. It can generate HTML, JSON, 
+or Markdown format.
+
+Options:
+  -all    Includes private and public functions/methods/structs/consts/enums.
+  -loc    Show the locations of the generated signatures. (For plaintext only)
+  -m      Generate docs for modules listed in that folder.
+  -s      Serve HTML-generated docs via HTTP.
+  -r      Include README.md to docs if present.

cmd/v/v.v

@@ -5,8 +5,6 @@ module main
 
 import help
 import os
-import v.table
-import v.doc
 import v.pref
 import v.util
 import v.builder
@@ -19,7 +17,7 @@ const (
 		'repl',
 		'build-tools', 'build-examples',
 		'build-vbinaries',
-		'setup-freetype'
+		'setup-freetype', 'doc'
 	]
 	list_of_flags_that_allow_duplicates = ['cc', 'd', 'define', 'cf', 'cflags']
 )
@@ -80,6 +78,9 @@ fn main_v() {
 			util.launch_tool(prefs.is_verbose, 'vpm', os.args[1..])
 			return
 		}
+		'vlib-docs' {
+			util.launch_tool(prefs.is_verbose, 'vdoc', ['doc', '-m', '-s', '-r', os.join_path(os.base_dir(@VEXE), 'vlib')])
+		}
 		'get' {
 			println('V Error: Use `v install` to install modules from vpm.vlang.io')
 			exit(1)
@@ -88,17 +89,6 @@ fn main_v() {
 			println(util.full_v_version(prefs.is_verbose))
 			return
 		}
-		'doc' {
-			mut mod := ''
-			if args.len == 1 {
-				println('v doc [module]')
-			} else if args.len > 1 {
-				mod = args[1]
-			}
-			table := table.new_table()
-			println(doc.doc(mod, table, prefs))
-			return
-		}
 		else {}
 	}
 	if command in ['run', 'build-module'] || command.ends_with('.v') || os.exists(command) {

vlib/v/doc/doc.v

@@ -1,149 +1,289 @@
 module doc
 
-import strings
-import v.pref
-import v.table
-import v.parser
-import v.ast
 import os
+import strings
+import time
+import v.ast
+import v.fmt
+import v.parser
+import v.pref
+import v.scanner
+import v.table
+import v.token
+import v.util
 
-struct Doc {
+pub struct Doc {
-	table &table.Table
+pub mut:
-	mod   string
+	input_path string = ''
-mut:
+	prefs &pref.Preferences = &pref.Preferences{}
-	out   strings.Builder
+	table &table.Table = &table.Table{}
-	stmts []ast.Stmt // all module statements from all files
+	pub_only bool = true
+	head DocNode
+	with_comments bool = true
+	contents []DocNode
+	time_generated time.Time
 }
 
-type FilterFn = fn (node ast.FnDecl) bool
+pub struct DocPos {
+pub:
+	line int
+	col int
+}
 
-pub fn doc(mod string, table &table.Table, prefs &pref.Preferences) string {
+pub struct DocNode {
-	mut d := Doc{
+pub mut:
+	name string
+	content string = ''
+	comment string
+	pos DocPos = DocPos{-1,-1}
+	file_path string = ''
+	parent_type string = ''
+}
+
+pub fn write_comment_bw(stmts []ast.Stmt, start_idx int) string {
+	mut comment := ''
+
+	for i := start_idx; i >= 0; i-- {
+		stmt := stmts[i]
+
+		if stmt is ast.Comment {
+			cmt := stmt as ast.Comment
+			cmt_content := cmt.text.trim_left('|')
+			comment = cmt_content + if cmt_content.starts_with('```') { '\n' } else { ' ' } + comment
+		} else {
+			panic('Not a comment')
+		}
+
+		if i-1 >= 0 && !(stmts[i-1] is ast.Comment) {
+			break
+		}
+	}
+
+	return comment
+}
+
+fn convert_pos(file_path string, pos token.Position) DocPos {
+    source := util.read_file(file_path) or {''}
+    mut p := util.imax(0, util.imin(source.len - 1, pos.pos))
+    column := util.imax(0, pos.pos - p - 1)
+    return DocPos{ line: pos.line_nr+1, col: util.imax(1, column+1) }
+}
+
+pub fn (d Doc) get_signature(stmt ast.Stmt) string {
+	mut f := fmt.Fmt{
 		out: strings.new_builder(1000)
-		table: table
+		out_imports: strings.new_builder(200)
-		mod: mod
+		table: d.table
-	}
+		indent: 0
-	vlib_path := os.dir(pref.vexe_path()) + '/vlib'
+		is_debug: false
-	mod_path := mod.replace('.', os.path_separator)
-	path := os.join_path(vlib_path, mod_path)
-	if mod == '' || !os.exists(path) {
-		if mod != '' {
-			println('module "$mod" not found')
-		}
-		println('\navailable modules:')
-		mut files := os.ls(vlib_path) or {
-			return ''
-		}
-		files.sort()
-		for file in files {
-			println(file)
-		}
-		// println(path)
-		return ''
-	}
-	// vfiles := os.walk_ext(path, '.v')
-	files := os.ls(path) or {
-		panic(err)
-	}
-	filtered_files := prefs.should_compile_filtered_files(path, files)
-	for file in filtered_files {
-		fscope := &ast.Scope{
-			parent: 0
-		}
-		file_ast := parser.parse_file(file, table, .skip_comments, prefs, fscope)
-		d.stmts << file_ast.stmts
-	}
-	if d.stmts.len == 0 {
-		println('nothing here')
-		exit(1)
-	}
-	d.print_structs()
-	d.print_enums()
-	d.print_fns()
-	d.out.writeln('')
-	d.print_methods()
-	/*
-	for stmt in file_ast.stmts {
-		d.stmt(stmt)
-	}
-	println(path)
-	*/
-	return d.out.str().trim_space()
 	}
 
-fn (d &Doc) get_fn_node(f ast.FnDecl) string {
-	return f.str(d.table).replace_each([d.mod + '.', '', 'pub ', ''])
-}
-
-fn (mut d Doc) print_fns() {
-	fn_signatures := d.get_fn_signatures(is_pub_function)
-	d.write_fn_signatures(fn_signatures)
-}
-
-fn (mut d Doc) print_methods() {
-	fn_signatures := d.get_fn_signatures(is_pub_method)
-	d.write_fn_signatures(fn_signatures)
-}
-
-[inline]
-fn (mut d Doc) write_fn_signatures(fn_signatures []string) {
-	for s in fn_signatures {
-		d.out.writeln(s)
-	}
-}
-
-fn (d Doc) get_fn_signatures(filter_fn FilterFn) []string {
-	mut fn_signatures := []string{}
-	for stmt in d.stmts {
 	match stmt {
+		ast.Module {
+			return 'module $it.name'
+		}
 		ast.FnDecl {
-				if filter_fn(it) {
+			return it.str(d.table)
-					fn_signatures << d.get_fn_node(it)
+		}
+		else {
+			f.stmt(stmt)
+			return f.out.str().trim_space()
 		}
 	}
-			else {}
-		}
-	}
-	fn_signatures.sort()
-	return fn_signatures
 }
 
-fn is_pub_method(node ast.FnDecl) bool {
+pub fn (d Doc) get_pos(stmt ast.Stmt) token.Position {
-	return node.is_pub && node.is_method && !node.is_deprecated
+	match stmt {
+		ast.FnDecl { return it.pos }
+		ast.StructDecl { return it.pos }
+		ast.EnumDecl { return it.pos }
+		ast.InterfaceDecl { return it.pos }
+		ast.ConstDecl { return it.pos }
+		else { return token.Position{} }
+	}
 }
 
-fn is_pub_function(node ast.FnDecl) bool {
+pub fn (d Doc) get_name(stmt ast.Stmt) string {
-	return node.is_pub && !node.is_method && !node.is_deprecated
+	match stmt {
+		ast.FnDecl { return it.name }
+		ast.StructDecl { return it.name }
+		ast.EnumDecl { return it.name }
+		ast.InterfaceDecl { return it.name }
+		ast.ConstDecl { return 'Constants' }
+		else { return '' }
+	}
 }
 
-// TODO it's probably better to keep using AST, not `table`
+pub fn new(input_path string) Doc {
-fn (mut d Doc) print_enums() {
+	return Doc{
-	for typ in d.table.types {
+		input_path: os.real_path(input_path),
-		if typ.kind != .enum_ {
+		prefs: &pref.Preferences{},
+		table: table.new_table(),
+		head: DocNode{},
+		contents: []DocNode{},
+		time_generated: time.now()
+	}
+}
+
+pub fn (nodes []DocNode) find_children_of(parent_type string) []DocNode {
+	if parent_type.len == 0 { return []DocNode{} }
+	mut children := []DocNode{}
+	for node in nodes {
+		if node.parent_type != parent_type { continue }
+		children << node
+	}
+
+	return children
+}
+
+fn get_parent_mod(dir string) ?string {
+	base_dir := os.base_dir(dir)
+	if os.file_name(base_dir) in ['encoding', 'v'] && 'vlib' in base_dir {
+		return os.file_name(base_dir)
+	}
+	prefs := &pref.Preferences{}
+	files := os.ls(base_dir) or { []string{} }
+	v_files := prefs.should_compile_filtered_files(base_dir, files)
+	if v_files.len == 0 {
+		parent_mod := get_parent_mod(base_dir) or { '' }
+		if parent_mod.len > 0 {
+			return parent_mod + '.' + os.file_name(base_dir)
+		}
+		return error('No V files found.')
+	}
+	file_ast := parser.parse_file(
+		v_files[0],
+		table.new_table(),
+		.skip_comments,
+		prefs,
+		&ast.Scope{parent: 0}
+	)
+	if file_ast.mod.name == 'main' {
+		return ''
+	}
+	parent_mod := get_parent_mod(base_dir) or { '' }
+	if parent_mod.len > 0 {
+		return parent_mod + '.' + file_ast.mod.name
+	}
+	return file_ast.mod.name
+}
+
+pub fn (mut d Doc) generate() ?bool {
+	// get all files
+	base_path := if os.is_dir(d.input_path) { d.input_path } else { os.real_path(os.base_dir(d.input_path)) }
+	project_files := os.ls(base_path) or { panic(err) }
+	v_files := d.prefs.should_compile_filtered_files(base_path, project_files)
+
+	if v_files.len == 0 {
+		return error('vdoc: No valid V files were found.')
+	}
+	// parse files
+	mut file_asts := []ast.File{}
+	// TODO: remove later for vlib
+	comments_mode := if d.with_comments { 
+		scanner.CommentsMode.parse_comments 
+	} else { 
+		scanner.CommentsMode.skip_comments 
+	}
+	for file in v_files {
+		file_ast := parser.parse_file(
+			file,
+			d.table,
+			comments_mode,
+			d.prefs,
+			&ast.Scope{parent: 0}
+		)
+
+		file_asts << file_ast
+	}
+
+	mut module_name := ''
+	mut parent_mod_name := ''
+	mut orig_mod_name := ''
+
+	for i, file_ast in file_asts {
+		if i == 0 {
+			parent_mod_name = get_parent_mod(base_path) or { '' }
+			module_name = file_ast.mod.name
+			orig_mod_name = module_name
+			if module_name != 'main' && parent_mod_name.len > 0 {
+				module_name = parent_mod_name + '.' + module_name
+			}
+			d.head = DocNode{
+				name: module_name,
+				content: 'module $module_name',
+				comment: ''
+			}
+		} else if file_ast.mod.name != module_name {
 			continue
 		}
-		d.out.writeln('enum $typ.name {')
+
-		info := typ.info as table.Enum
+		stmts := file_ast.stmts
-		for val in info.vals {
+		for si, stmt in stmts {
-			d.out.writeln('\t$val')
+			if stmt is ast.Comment { continue }
+
+			if !(stmt is ast.Module) {
+				// todo: accumulate consts
+				mut name := d.get_name(stmt)
+				signature := d.get_signature(stmt)
+				pos := d.get_pos(stmt)
+
+				if !signature.starts_with('pub') && d.pub_only { 
+					continue
 				}
-		d.out.writeln('}')
+
+				if name.starts_with(orig_mod_name + '.') {
+					name = name.all_after(orig_mod_name + '.')
+				}
+
+				mut node := DocNode{
+					name: name,
+					content: signature,
+					comment: '',
+					pos: convert_pos(v_files[i], pos),
+					file_path: v_files[i]
+				}
+
+				if stmt is ast.FnDecl {
+					fnd := stmt as ast.FnDecl
+
+					if fnd.receiver.typ != 0 {
+						mut parent_type := d.table.get_type_name(fnd.receiver.typ)
+
+						if parent_type.starts_with(module_name + '.') {
+							parent_type = parent_type.all_after(module_name + '.')
+						}
+
+						node.parent_type = parent_type
 					}
 				}
 
-fn (mut d Doc) print_structs() {
+				d.contents << node
-	for typ in d.table.types {
-		if typ.kind != .struct_ || !typ.name.starts_with(d.mod + '.') {
-			// !typ.name[0].is_capital() || typ.name.starts_with('C.') {
-			continue
 			}
-		name := typ.name.after('.')
+
-		d.out.writeln('struct $name {')
+			if d.with_comments && (si-1 >= 0 && stmts[si-1] is ast.Comment) {
-		info := typ.info as table.Struct
+				if stmt is ast.Module {
-		for field in info.fields {
+					d.head.comment = write_comment_bw(stmts, si-1)
-			sym := d.table.get_type_symbol(field.typ)
+				} else {
-			d.out.writeln('\t$field.name $sym.name')
+					last_comment := d.contents[d.contents.len-1].comment
-		}
+					d.contents[d.contents.len-1].comment = last_comment + '\n' + write_comment_bw(stmts, si-1)
-		d.out.writeln('}\n')
 				}
 			}	
+		}
+	}
+
+	d.time_generated = time.now()
+	return true
+}
+
+pub fn generate(input_path string, pub_only bool, with_comments bool) ?Doc {
+	mut doc := doc.new(input_path)
+	doc.pub_only = pub_only
+	doc.with_comments = with_comments
+
+	_ = doc.generate() or {
+		return error(err)
+	}
+
+	return doc
+}

vlib/v/doc/doc_test.v

@@ -2,9 +2,9 @@ import v.table
 import v.doc
 import v.pref
 
-fn test_vdoc() {
+// fn test_vdoc() {
-	mut prefs := &pref.Preferences{}
+// 	mut prefs := &pref.Preferences{}
-	prefs.fill_with_defaults()
+// 	prefs.fill_with_defaults()
-	table := table.new_table()
+// 	table := table.new_table()
-	println(doc.doc('net', table, prefs))
+// 	println(doc.doc('net', table, prefs))
-}
+// }