doc: add comment example support (#7924)

cmd/tools/vdoc/vdoc.v

@@ -121,6 +121,7 @@ mut:
 	is_vlib             bool
 	is_verbose          bool
 	include_readme      bool
+	include_examples    bool = true
 	open_docs           bool
 	server_port         int = 8046
 	inline_assets       bool
@@ -307,7 +308,12 @@ fn escape(str string) string {
 fn (cfg DocConfig) gen_json(idx int) string {
 	dcs := cfg.docs[idx]
 	mut jw := strings.new_builder(200)
-	jw.write('{"module_name":"$dcs.head.name","description":"${escape(dcs.head.comment)}","contents":')
+	comments := if cfg.include_examples {
+		dcs.head.merge_comments()
+	} else {
+		dcs.head.merge_comments_without_examples()
+	}
+	jw.write('{"module_name":"$dcs.head.name","description":"${escape(comments)}","contents":')
 	jw.write(json.encode(dcs.contents.keys().map(dcs.contents[it])))
 	jw.write(',"generator":"vdoc","time_generated":"$dcs.time_generated.str()"}')
 	return jw.str()
@@ -395,11 +401,12 @@ fn html_highlight(code string, tb &table.Table) string {
 	return buf.str()
 }
 
-fn doc_node_html(dd doc.DocNode, link string, head bool, tb &table.Table) string {
+fn doc_node_html(dd doc.DocNode, link string, head bool, include_examples bool, tb &table.Table) string {
 	mut dnw := strings.new_builder(200)
 	link_svg := '<svg xmlns="http://www.w3.org/2000/svg" height="24" viewBox="0 0 24 24" width="24"><path d="M0 0h24v24H0z" fill="none"/><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7c-2.76 0-5 2.24-5 5s2.24 5 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1zM8 13h8v-2H8v2zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4c2.76 0 5-2.24 5-5s-2.24-5-5-5z"/></svg>'
 	head_tag := if head { 'h1' } else { 'h2' }
-	md_content := markdown.to_html(html_tag_escape(dd.comment))
+	comments := dd.merge_comments_without_examples()
+	md_content := markdown.to_html(html_tag_escape(comments))
 	hlighted_code := html_highlight(dd.content, tb)
 	node_class := if dd.kind == .const_group { ' const' } else { '' }
 	sym_name := get_sym_name(dd)
@@ -425,7 +432,19 @@ fn doc_node_html(dd doc.DocNode, link string, head bool, tb &table.Table) string
 		dnw.writeln('<pre class="signature"><code>$hlighted_code</code></pre>')
 	}
 	// do not mess with md_content further, its formatting is important, just output it 1:1 !
-	dnw.writeln('$md_content\n</section>')
+	dnw.writeln('$md_content\n')
+	// Write examples if any found
+	examples := dd.examples()
+	if include_examples && examples.len > 0 {
+		example_title := if examples.len > 1 { 'Examples' } else { 'Example' }
+		dnw.writeln('<section class="doc-node examples"><h4>$example_title</h4>')
+		for example in examples {
+			// hl_example := html_highlight(example, tb)
+			dnw.writeln('<pre><code class="language-v">$example</code></pre>')
+		}
+		dnw.writeln('</section>')
+	}
+	dnw.writeln('</section>')
 	dnw_str := dnw.str()
 	defer {
 		dnw.free()
@@ -499,12 +518,12 @@ fn (cfg DocConfig) write_content(cn &doc.DocNode, dcs &doc.Doc, mut hw strings.B
 	}
 	src_link := get_src_link(cfg.manifest.repo_url, file_path_name, cn.pos.line)
 	if cn.content.len != 0 || (cn.name == 'Constants') {
-		hw.write(doc_node_html(cn, src_link, false, dcs.table))
+		hw.write(doc_node_html(cn, src_link, false, cfg.include_examples, dcs.table))
 	}
 	for child in cn.children {
 		child_file_path_name := child.file_path.replace('$base_dir/', '')
 		child_src_link := get_src_link(cfg.manifest.repo_url, child_file_path_name, child.pos.line)
-		hw.write(doc_node_html(child, child_src_link, false, dcs.table))
+		hw.write(doc_node_html(child, child_src_link, false, cfg.include_examples, dcs.table))
 	}
 }
 
@@ -515,7 +534,7 @@ fn (cfg DocConfig) gen_html(idx int) string {
 	dcs := cfg.docs[idx]
 	dcs_contents := dcs.contents.arr()
 	// generate toc first
-	contents.writeln(doc_node_html(dcs.head, '', true, dcs.table))
+	contents.writeln(doc_node_html(dcs.head, '', true, cfg.include_examples, dcs.table))
 	if is_module_readme(dcs.head) {
 		write_toc(dcs.head, mut symbols_toc)
 	}
@@ -602,8 +621,13 @@ fn (cfg DocConfig) gen_plaintext(idx int) string {
 	dcs := cfg.docs[idx]
 	mut pw := strings.new_builder(200)
 	pw.writeln('$dcs.head.content\n')
-	if dcs.head.comment.trim_space().len > 0 && !cfg.pub_only {
+	comments := if cfg.include_examples {
-		pw.writeln(dcs.head.comment.split_into_lines().map('    ' + it).join('\n'))
+		dcs.head.merge_comments()
+	} else {
+		dcs.head.merge_comments_without_examples()
+	}
+	if comments.trim_space().len > 0 && !cfg.pub_only {
+		pw.writeln(comments.split_into_lines().map('    ' + it).join('\n'))
 	}
 	cfg.write_plaintext_content(dcs.contents.arr(), mut pw)
 	return pw.str()
@@ -613,8 +637,13 @@ fn (cfg DocConfig) write_plaintext_content(contents []doc.DocNode, mut pw string
 	for cn in contents {
 		if cn.content.len > 0 {
 			pw.writeln(cn.content)
-			if cn.comment.len > 0 && !cfg.pub_only {
+			if cn.comments.len > 0 && !cfg.pub_only {
-				pw.writeln(cn.comment.trim_space().split_into_lines().map('    ' + it).join('\n'))
+				comments := if cfg.include_examples {
+					cn.merge_comments()
+				} else {
+					cn.merge_comments_without_examples()
+				}
+				pw.writeln(comments.trim_space().split_into_lines().map('    ' + it).join('\n'))
 			}
 			if cfg.show_loc {
 				pw.writeln('Location: $cn.file_path:$cn.pos.line\n')
@@ -629,8 +658,13 @@ fn (cfg DocConfig) gen_markdown(idx int, with_toc bool) string {
 	mut hw := strings.new_builder(200)
 	mut cw := strings.new_builder(200)
 	hw.writeln('# $dcs.head.content\n')
-	if dcs.head.comment.len > 0 {
+	if dcs.head.comments.len > 0 {
-		hw.writeln('$dcs.head.comment\n')
+		comments := if cfg.include_examples {
+			dcs.head.merge_comments()
+		} else {
+			dcs.head.merge_comments_without_examples()
+		}
+		hw.writeln('$comments\n')
 	}
 	if with_toc {
 		hw.writeln('## Contents')
@@ -648,7 +682,18 @@ fn (cfg DocConfig) write_markdown_content(contents []doc.DocNode, mut cw strings
 			cw.writeln('## $cn.name')
 		}
 		if cn.content.len > 0 {
-			cw.writeln('```v\n$cn.content\n```$cn.comment\n')
+			comments := cn.merge_comments_without_examples()
+			cw.writeln('```v\n$cn.content\n```\n$comments\n')
+			// Write examples if any found
+			examples := cn.examples()
+			if cfg.include_examples && examples.len > 0 {
+				example_title := if examples.len > 1 { 'Examples' } else { 'Example' }
+				cw.writeln('$example_title\n```v\n')
+				for example in examples {
+					cw.writeln('$example\n')
+				}
+				cw.writeln('```\n')
+			}
 			cw.writeln('[\[Return to contents\]](#Contents)\n')
 		}
 		cfg.write_markdown_content(cn.children, mut cw, mut hw, indent + 1, with_toc)
@@ -742,8 +787,13 @@ fn (mut cfg DocConfig) collect_search_index() {
 	for doc in cfg.docs {
 		mod := doc.head.name
 		cfg.search_module_index << mod
+		comments := if cfg.include_examples {
+			doc.head.merge_comments()
+		} else {
+			doc.head.merge_comments_without_examples()
+		}
 		cfg.search_module_data << SearchModuleResult{
-			description: trim_doc_node_description(doc.head.comment)
+			description: trim_doc_node_description(comments)
 			link: cfg.get_file_name(mod)
 		}
 		for _, dn in doc.contents {
@@ -756,7 +806,12 @@ fn (mut cfg DocConfig) create_search_results(mod string, dn doc.DocNode) {
 	if dn.kind == .const_group {
 		return
 	}
-	dn_description := trim_doc_node_description(dn.comment)
+	comments := if cfg.include_examples {
+		dn.merge_comments()
+	} else {
+		dn.merge_comments_without_examples()
+	}
+	dn_description := trim_doc_node_description(comments)
 	cfg.search_index << dn.name
 	cfg.search_data << SearchResult{
 		prefix: if dn.parent_name != '' {
@@ -894,13 +949,16 @@ fn (mut cfg DocConfig) generate_docs_from_file() {
 	}
 	if cfg.include_readme {
 		readme_contents := cfg.get_readme(dir_path)
+		comment := doc.DocComment{
+			text: readme_contents
+		}
 		if cfg.output_type == .stdout {
 			println(markdown.to_plain(readme_contents))
 		} else if cfg.output_type == .html && cfg.is_multi {
 			cfg.docs << doc.Doc{
 				head: doc.DocNode{
 					name: 'README'
-					comment: readme_contents
+					comments: [comment]
 				}
 				time_generated: time.now()
 			}
@@ -932,7 +990,10 @@ fn (mut cfg DocConfig) generate_docs_from_file() {
 		if !is_local_and_single {
 			if cfg.is_multi || (!cfg.is_multi && cfg.include_readme) {
 				readme_contents := cfg.get_readme(dirpath)
-				dcs.head.comment = readme_contents
+				comment := doc.DocComment{
+					text: readme_contents
+				}
+				dcs.head.comments = [comment]
 			}
 			if cfg.pub_only {
 				for name, dc in dcs.contents {
@@ -1183,6 +1244,9 @@ fn main() {
 			'-no-timestamp' {
 				cfg.no_timestamp = true
 			}
+			'-no-examples' {
+				cfg.include_examples = false
+			}
 			'-readme' {
 				cfg.include_readme = true
 			}
@@ -1231,7 +1295,7 @@ fn main() {
 }
 
 fn is_module_readme(dn doc.DocNode) bool {
-	if dn.comment.len > 0 && dn.content == 'module $dn.name' {
+	if dn.comments.len > 0 && dn.content == 'module $dn.name' {
 		return true
 	}
 	return false

vlib/v/doc/comment.v

@@ -0,0 +1,23 @@
+module doc
+
+const (
+	example_pattern = '\x01 Example: '
+)
+
+pub struct DocComment {
+pub mut:
+	text     string // Raw text content of the comment, excluding the comment token chars ('//, /*, */')
+	is_multi bool   // Is a block / multi-line comment
+	pos      DocPos = DocPos{-1, -1, 0}
+}
+
+// is_example returns true if the contents of this comment is a doc example.
+// The current convention is '// Example: <content>'
+pub fn (dc DocComment) is_example() bool {
+	return dc.text.starts_with(example_pattern)
+}
+
+// example returns the content of the example body
+pub fn (dc DocComment) example() string {
+	return dc.text.all_after(example_pattern)
+}

vlib/v/doc/doc.v

@@ -80,7 +80,7 @@ pub struct DocNode {
 pub mut:
 	name        string
 	content     string
-	comment     string
+	comments    []DocComment
 	pos         DocPos = DocPos{-1, -1, 0}
 	file_path   string
 	kind        SymbolKind
@@ -128,7 +128,6 @@ pub fn (mut d Doc) stmt(stmt ast.Stmt, filename string) ?DocNode {
 	mut node := DocNode{
 		name: d.stmt_name(stmt)
 		content: d.stmt_signature(stmt)
-		comment: ''
 		pos: d.convert_pos(filename, stmt.position())
 		file_path: os.join_path(d.base_path, filename)
 		is_pub: d.stmt_pub(stmt)
@@ -139,7 +138,7 @@ pub fn (mut d Doc) stmt(stmt ast.Stmt, filename string) ?DocNode {
 	if node.name.starts_with(d.orig_mod_name + '.') {
 		node.name = node.name.all_after(d.orig_mod_name + '.')
 	}
-	if node.name.len == 0 && node.comment.len == 0 && node.content.len == 0 {
+	if node.name.len == 0 && node.comments.len == 0 && node.content.len == 0 {
 		return error('empty stmt')
 	}
 	match stmt {
@@ -246,12 +245,13 @@ pub fn (mut d Doc) file_ast(file_ast ast.File) map[string]DocNode {
 			last_import_stmt_idx = sidx
 		}
 	}
-	mut prev_comments := []ast.Comment{}
+	mut preceeding_comments := []DocComment{}
 	mut imports_section := true
 	for sidx, stmt in stmts {
 		if stmt is ast.ExprStmt {
+			// Collect comments
 			if stmt.expr is ast.Comment {
-				prev_comments << stmt.expr
+				preceeding_comments << ast_comment_to_doc_comment(stmt.expr)
 				continue
 			}
 		}
@@ -261,37 +261,41 @@ pub fn (mut d Doc) file_ast(file_ast ast.File) map[string]DocNode {
 				continue
 			}
 			// the previous comments were probably a copyright/license one
-			module_comment := get_comment_block_right_before(prev_comments)
+			module_comment := merge_doc_comments(preceeding_comments)
-			prev_comments = []
+			preceeding_comments = []
 			if !d.is_vlib && !module_comment.starts_with('Copyright (c)') {
-				if module_comment in ['', d.head.comment] {
+				if module_comment == '' {
 					continue
 				}
+				/*
 				if d.head.comment != '' {
 					d.head.comment += '\n'
 				}
-				d.head.comment += module_comment
+				*/
+				d.head.comments << preceeding_comments //+= module_comment
 			}
 			continue
 		}
 		if last_import_stmt_idx > 0 && sidx == last_import_stmt_idx {
 			// the accumulated comments were interspersed before/between the imports;
-			// just add them all to the module comment:
+			// just add them all to the module comments:
 			if d.with_head {
-				import_comments := merge_comments(prev_comments)
+				// import_comments := merge_comments(preceeding_comments)
+				/*
 				if d.head.comment != '' {
 					d.head.comment += '\n'
 				}
-				d.head.comment += import_comments
+				*/
+				d.head.comments << preceeding_comments //+= import_comments
 			}
-			prev_comments = []
+			preceeding_comments = []
 			imports_section = false
 		}
 		if stmt is ast.Import {
 			continue
 		}
 		mut node := d.stmt(stmt, os.base(file_ast.path)) or {
-			prev_comments = []
+			preceeding_comments = []
 			continue
 		}
 		if node.parent_name !in contents {
@@ -305,18 +309,20 @@ pub fn (mut d Doc) file_ast(file_ast ast.File) map[string]DocNode {
 				kind: parent_node_kind
 			}
 		}
-		if d.with_comments && (prev_comments.len > 0) {
+		if d.with_comments && (preceeding_comments.len > 0) {
 			// last_comment := contents[contents.len - 1].comment
-			// cmt := last_comment + '\n' + get_comment_block_right_before(prev_comments)
+			// cmt := last_comment + '\n' + merge_doc_comments(preceeding_comments)
-			mut cmt := get_comment_block_right_before(prev_comments)
+			/*
+			mut cmt := merge_doc_comments(preceeding_comments)
 			len := node.name.len
 			// fixed-width symbol name at start of comment
 			if cmt.starts_with(node.name) && cmt.len > len && cmt[len] == ` ` {
 				cmt = '`${cmt[..len]}`' + cmt[len..]
 			}
-			node.comment = cmt
+			*/
+			node.comments << preceeding_comments //= cmt
 		}
-		prev_comments = []
+		preceeding_comments = []
 		if node.parent_name.len > 0 {
 			parent_name := node.parent_name
 			if node.parent_name == 'Constants' {

vlib/v/doc/node.v

@@ -45,3 +45,26 @@ pub fn (cnts map[string]DocNode) arr() []DocNode {
 	contents.sort_by_kind()
 	return contents
 }
+
+// merge_comments returns a `string` with the combined contents of `DocNode.comments`.
+pub fn (dc DocNode) merge_comments() string {
+	return merge_doc_comments(dc.comments)
+}
+
+// merge_comments_without_examples returns a `string` with the
+// combined contents of `DocNode.comments` - excluding any examples.
+pub fn (dc DocNode) merge_comments_without_examples() string {
+	sans_examples := dc.comments.filter(!it.is_example())
+	return merge_doc_comments(sans_examples)
+}
+
+// examples returns a `[]string` containing examples parsed from `DocNode.comments`.
+pub fn (dn DocNode) examples() []string {
+	mut output := []string{}
+	for comment in dn.comments {
+		if comment.is_example() {
+			output << comment.example()
+		}
+	}
+	return output
+}

vlib/v/doc/utils.v

@@ -16,9 +16,33 @@ pub fn merge_comments(comments []ast.Comment) string {
 	return res.join('\n')
 }
 
-// get_comment_block_right_before merges all the comments starting from
+// ast_comment_to_doc_comment converts an `ast.Comment` node type to a `DocComment`
+pub fn ast_comment_to_doc_comment(ast_node ast.Comment) DocComment {
+	text := ast_node.text // TODO .trim_left('\x01') // BUG why are this byte here in the first place?
+	return DocComment{
+		text: text
+		is_multi: ast_node.is_multi
+		pos: DocPos{
+			line: ast_node.pos.line_nr - 1
+			col: 0 // ast_node.pos.pos - ast_node.text.len
+			len: text.len
+		}
+	}
+}
+
+// ast_comments_to_doc_comments converts an array of `ast.Comment` nodes to
+// an array of `DocComment` nodes
+pub fn ast_comments_to_doc_comments(ast_nodes []ast.Comment) []DocComment {
+	mut doc_comments := []DocComment{len: ast_nodes.len}
+	for ast_comment in ast_nodes {
+		doc_comments << ast_comment_to_doc_comment(ast_comment)
+	}
+	return doc_comments
+}
+
+// merge_doc_comments merges all the comments starting from
 // the last up to the first item of the array.
-pub fn get_comment_block_right_before(comments []ast.Comment) string {
+pub fn merge_doc_comments(comments []DocComment) string {
 	if comments.len == 0 {
 		return ''
 	}
@@ -26,7 +50,7 @@ pub fn get_comment_block_right_before(comments []ast.Comment) string {
 	mut last_comment_line_nr := 0
 	for i := comments.len - 1; i >= 0; i-- {
 		cmt := comments[i]
-		if last_comment_line_nr != 0 && cmt.pos.line_nr < last_comment_line_nr - 1 {
+		if last_comment_line_nr != 0 && cmt.pos.line < last_comment_line_nr - 1 {
 			// skip comments that are not part of a continuous block,
 			// located right above the top level statement.
 			// break
@@ -58,7 +82,7 @@ pub fn get_comment_block_right_before(comments []ast.Comment) string {
 		// eprintln('cmt: $cmt')
 		cseparator := if cmt_content.starts_with('```') { '\n' } else { ' ' }
 		comment = cmt_content + cseparator + comment
-		last_comment_line_nr = cmt.pos.line_nr
+		last_comment_line_nr = cmt.pos.line
 	}
 	return comment
 }