vdoc: add `-comments` and new comment merger (#11221)

cmd/tools/vdoc/tests/testdata/basic/main.comments.out

@@ -0,0 +1,7 @@
+module main
+
+const (
+	source_root = 'temp'
+)
+fn funky()
+    funky - comment for function below

cmd/tools/vdoc/tests/testdata/basic/main.out

@@ -0,0 +1,6 @@
+module main
+
+const (
+	source_root = 'temp'
+)
+fn funky()

cmd/tools/vdoc/tests/testdata/basic/main.v

@@ -1,8 +1,8 @@
-const (
+pub const (
 	source_root = 'temp'
 )
 
 // funky - comment for function below
-fn funky() {
+pub fn funky() {
 	println('hi')
 }

cmd/tools/vdoc/tests/testdata/multiline/main.comments.out

@@ -0,0 +1,9 @@
+module main
+
+fn a1()
+    normal comment
+fn a2()
+    this should be merged into the same line
+fn a3()
+    This should be its own parapgraph, because it ends with a dot.  
+    This should be another paragraph.

cmd/tools/vdoc/tests/testdata/multiline/main.out

@@ -0,0 +1,5 @@
+module main
+
+fn a1()
+fn a2()
+fn a3()

cmd/tools/vdoc/tests/testdata/multiline/main.v

@@ -0,0 +1,16 @@
+// normal comment
+pub fn a1() {
+	println('hi')
+}
+
+// this should be merged
+// into the same line
+pub fn a2() {
+	println('hi')
+}
+
+// This should be its own parapgraph, because it ends with a dot.
+// This should be another paragraph.
+pub fn a3() {
+	println('hi')
+}

cmd/tools/vdoc/tests/testdata/newlines/main.comments.out

@@ -0,0 +1,22 @@
+module main
+
+fn funky()
+    hello
+    
+    empty line
+    newline using a full stop.  
+    ```v
+    code
+    ```
+    
+    test
+    ====
+    - foo
+    - bar
+    # test
+    ########### deep test
+    #a shouldnt have a newline test
+    
+    | foo bar   |  yes   |
+    |-----------|--------|
+    |  working  |  yup   |

cmd/tools/vdoc/tests/testdata/newlines/main.out

@@ -0,0 +1,3 @@
+module main
+
+fn funky()

cmd/tools/vdoc/tests/testdata/newlines/main.v

@@ -0,0 +1,23 @@
+// hello
+//
+// empty line
+// newline using a full stop.
+// ```v
+// code
+// ```
+//
+// test
+// ====
+// - foo
+// - bar
+// # test
+// ########### deep test
+// #a shouldnt have a newline
+// test
+//
+// | foo bar   |  yes   |
+// |-----------|--------|
+// |  working  |  yup   |
+pub fn funky() {
+	println('hi')
+}

cmd/tools/vdoc/tests/testdata/project1/main.out

@@ -1 +0,0 @@
-vdoc: No documentation found for /v/vmaster/cmd/tools/vdoc/tests/testdata/project1/main.v

cmd/tools/vdoc/tests/vdoc_file_test.v

@@ -43,25 +43,45 @@ fn check_path(vexe string, dir string, tests []string) int {
 		expected = clean_line_endings(expected)
 		found := clean_line_endings(res.output)
 		if expected != found {
-			println(term.red('FAIL'))
-			println('============')
-			println('expected:')
-			println(expected)
-			println('============')
-			println('found:')
-			println(found)
-			println('============\n')
-			println('diff:')
-			println(diff.color_compare_strings(diff_cmd, rand.ulid(), found, expected))
-			println('============\n')
-			nb_fail++
-		} else {
+			print_compare(expected, found)
+		}
+
+		res_comments := os.execute('$vexe doc -comments $program')
+		if res_comments.exit_code < 0 {
+			panic(res_comments.output)
+		}
+		mut expected_comments := os.read_file(program.replace('main.v', 'main.comments.out')) or {
+			panic(err)
+		}
+		expected_comments = clean_line_endings(expected_comments)
+		found_comments := clean_line_endings(res_comments.output)
+		if expected_comments != found_comments {
+			print_compare(expected_comments, found_comments)
+		}
+
+		if expected == found && expected_comments == found_comments {
 			println(term.green('OK'))
+		} else {
+			nb_fail++
 		}
 	}
 	return nb_fail
 }
 
+fn print_compare(expected string, found string) {
+	println(term.red('FAIL'))
+	println('============')
+	println('expected:')
+	println(expected)
+	println('============')
+	println('found:')
+	println(found)
+	println('============\n')
+	println('diff:')
+	println(diff.color_compare_strings(diff_cmd, rand.ulid(), found, expected))
+	println('============\n')
+}
+
 fn clean_line_endings(s string) string {
 	mut res := s.trim_space()
 	res = res.replace(' \n', '\n')

cmd/tools/vdoc/vdoc.v

@@ -51,6 +51,7 @@ mut:
 	is_verbose       bool
 	include_readme   bool
 	include_examples bool = true
+	include_comments bool // for plaintext
 	inline_assets    bool
 	no_timestamp     bool
 	output_path      string
@@ -95,13 +96,15 @@ fn (vd VDoc) gen_plaintext(d doc.Doc) string {
 	} else {
 		pw.writeln('$d.head.content\n')
 	}
-	comments := if cfg.include_examples {
-		d.head.merge_comments()
-	} else {
-		d.head.merge_comments_without_examples()
-	}
-	if comments.trim_space().len > 0 && !cfg.pub_only {
-		pw.writeln(comments.split_into_lines().map('    ' + it).join('\n'))
+	if cfg.include_comments {
+		comments := if cfg.include_examples {
+			d.head.merge_comments()
+		} else {
+			d.head.merge_comments_without_examples()
+		}
+		if comments.trim_space().len > 0 {
+			pw.writeln(comments.split_into_lines().map('    ' + it).join('\n'))
+		}
 	}
 	vd.write_plaintext_content(d.contents.arr(), mut pw)
 	return pw.str()
@@ -116,7 +119,7 @@ fn (vd VDoc) write_plaintext_content(contents []doc.DocNode, mut pw strings.Buil
 			} else {
 				pw.writeln(cn.content)
 			}
-			if cn.comments.len > 0 && !cfg.pub_only {
+			if cn.comments.len > 0 && cfg.include_comments {
 				comments := if cfg.include_examples {
 					cn.merge_comments()
 				} else {
@@ -414,6 +417,9 @@ fn parse_arguments(args []string) Config {
 			'-l' {
 				cfg.show_loc = true
 			}
+			'-comments' {
+				cfg.include_comments = true
+			}
 			'-m' {
 				cfg.is_multi = true
 			}

cmd/v/help/doc.txt

@@ -21,8 +21,8 @@ Options:
   -o              Specifies the output file/folder path where to store the generated docs.
                   Set it to "stdout" to print the output instead of saving the contents
                   to a file.
-  -color          Force stdout colorize output.
-  -no-color       Force plain text output, without ANSI colors.
+  -color          Forces stdout colorize output.
+  -no-color       Forces plain text output, without ANSI colors.
   -readme         Include README.md to docs if present.
   -v              Enables verbose logging. For debugging purposes.
   -no-timestamp   Omits the timestamp in the output file.
@@ -31,4 +31,5 @@ For HTML mode:
   -inline-assets  Embeds the contents of the CSS and JS assets into the webpage directly.
 
 For plain text mode:
-  -l              Show the locations of the generated signatures.
+  -l              Shows the locations of the generated signatures.
+  -comments       Includes comments in the output.

doc/docs.md

@@ -4111,6 +4111,17 @@ An overview of the module must be placed in the first comment right after the mo
 
 To generate documentation use vdoc, for example `v doc net.http`.
 
+### Newlines in Documentation Comments
+
+Comments spanning multiple lines are merged together using spaces, unless
+
+- the line is empty
+- the line ends with a `.` (end of sentence)
+- the line is contains purely of at least 3 of `-`, `=`, `_`, `*`, `~` (horizontal rule)
+- the line starts with at least one `#` followed by a space (header)
+- the line starts and ends with a `|` (table)
+- the line starts with `- ` (list)
+
 ## Tools
 
 ### v fmt

vlib/v/doc/utils.v

@@ -43,44 +43,71 @@ pub fn merge_doc_comments(comments []DocComment) string {
 	if comments.len == 0 {
 		return ''
 	}
-	mut comment := ''
+	mut commentlines := []string{}
 	mut last_comment_line_nr := 0
 	for i := comments.len - 1; i >= 0; i-- {
 		cmt := comments[i]
-		if last_comment_line_nr != 0 && cmt.pos.line_nr + 1 < last_comment_line_nr - 1 {
+		if (!cmt.is_multi && last_comment_line_nr != 0
+			&& cmt.pos.line_nr + 1 < last_comment_line_nr - 1) || (cmt.is_multi
+			&& cmt.pos.line_nr + cmt.text.count('\n') + 1 < last_comment_line_nr - 1) {
 			// skip comments that are not part of a continuous block,
 			// located right above the top level statement.
-			// break
+			break
 		}
 		mut cmt_content := cmt.text.trim_left('\x01')
 		if cmt.is_multi {
-			// ignore /* */ style comments for now
+			// /**/ comments are deliberately NOT supported as vdoc comments,
+			// so just ignore them:
 			continue
-			// if cmt_content.len == 0 {
-			// continue
-			// }
-			// mut new_cmt_content := ''
-			// mut is_codeblock := false
-			// // println(cmt_content)
-			// lines := cmt_content.split_into_lines()
-			// for j, line in lines {
-			// trimmed := line.trim_space().trim_left(cmt_prefix)
-			// if trimmed.starts_with('- ') || (trimmed.len >= 2 && trimmed[0].is_digit() && trimmed[1] == `.`) || is_codeblock {
-			// new_cmt_content += line + '\n'
-			// } else if line.starts_with('```') {
-			// is_codeblock = !is_codeblock
-			// new_cmt_content += line + '\n'
-			// } else {
-			// new_cmt_content += trimmed + '\n'
-			// }
-			// }
-			// return new_cmt_content
+		} else {
+			if cmt_content.starts_with(' ') {
+				cmt_content = cmt_content[1..]
+			}
+			commentlines << cmt_content
 		}
-		// eprintln('cmt: $cmt')
-		cseparator := if cmt_content.starts_with('```') { '\n' } else { ' ' }
-		comment = cmt_content + cseparator + comment
 		last_comment_line_nr = cmt.pos.line_nr + 1
 	}
+	commentlines = commentlines.reverse()
+	mut is_codeblock := false
+	mut previously_newline := true
+	mut comment := ''
+	for line in commentlines {
+		if line.starts_with('```') {
+			is_codeblock = !is_codeblock
+			comment = comment + '\n' + line
+			continue
+		} else if is_codeblock {
+			comment = comment + '\n' + line
+			continue
+		}
+
+		line_trimmed := line.trim(' ')
+
+		mut is_horizontalrule := false
+		line_no_spaces := line_trimmed.replace(' ', '')
+		for char in ['-', '=', '*', '_', '~'] {
+			if line_no_spaces.starts_with(char.repeat(3))
+				&& line_no_spaces.count(char) == line_no_spaces.len {
+				is_horizontalrule = true
+				break
+			}
+		}
+
+		if line_trimmed == '' || is_horizontalrule
+			|| (line.starts_with('#') && line.before(' ').count('#') == line.before(' ').len)
+			|| (line_trimmed.starts_with('|') && line_trimmed.ends_with('|'))
+			|| line_trimmed.starts_with('- ') {
+			comment = comment + '\n' + line
+			previously_newline = true
+		} else if line.ends_with('.') {
+			comment = comment + '\n' + line + '  '
+			previously_newline = true
+		} else {
+			sep := if previously_newline { '\n' } else { ' ' }
+			comment = comment + sep + line
+			previously_newline = false
+		}
+	}
 	return comment
 }