From b299fb1e92549efd297cf3b124c2e0c1a9d20289 Mon Sep 17 00:00:00 2001 From: Larpon Date: Fri, 8 Jan 2021 11:25:22 +0100 Subject: [PATCH] vdoc: heavy refactor, immutable config (#7945) --- cmd/tools/modules/testing/common.v | 60 +- cmd/tools/vbuild-tools.v | 50 +- cmd/tools/vdoc/html.v | 522 +++++++++++++ cmd/tools/vdoc/httpserver.v | 95 +++ cmd/tools/vdoc/markdown.v | 50 ++ cmd/tools/vdoc/utils.v | 145 ++++ cmd/tools/vdoc/vdoc.v | 1100 +++++----------------------- cmd/tools/vtest-cleancode.v | 2 +- cmd/tools/vtest-compiler.v | 2 +- 9 files changed, 1055 insertions(+), 971 deletions(-) create mode 100644 cmd/tools/vdoc/html.v create mode 100644 cmd/tools/vdoc/httpserver.v create mode 100644 cmd/tools/vdoc/markdown.v create mode 100644 cmd/tools/vdoc/utils.v diff --git a/cmd/tools/modules/testing/common.v b/cmd/tools/modules/testing/common.v index dedb30964b..c7a2414208 100644 --- a/cmd/tools/modules/testing/common.v +++ b/cmd/tools/modules/testing/common.v @@ -143,6 +143,10 @@ pub fn (mut ts TestSession) init() { ts.benchmark = benchmark.new_benchmark_no_cstep() } +pub fn (mut ts TestSession) add(file string) { + ts.files << file +} + pub fn (mut ts TestSession) test() { // Ensure that .tmp.c files generated from compiling _test.v files, // are easy to delete at the end, *without* affecting the existing ones. @@ -307,32 +311,38 @@ pub fn prepare_test_session(zargs string, folder string, oskipped []string, main files := os.walk_ext(os.join_path(parent_dir, folder), '.v') mut mains := []string{} mut skipped := oskipped.clone() - for f in files { - if !f.contains('modules') && !f.contains('preludes') { - // $if !linux { - // run pg example only on linux - if f.contains('/pg/') { - continue - } - // } - if f.contains('life_gg') || f.contains('/graph.v') || f.contains('rune.v') { - continue - } - $if windows { - // skip pico example on windows - if f.ends_with('examples\\pico\\pico.v') { - continue - } - } - c := os.read_file(f) or { panic(err) } - maxc := if c.len > 300 { 300 } else { c.len } - start := c[0..maxc] - if start.contains('module ') && !start.contains('module main') { - skipped_f := f.replace(os.join_path(parent_dir, ''), '') - skipped << skipped_f - } - mains << f + next_file: for f in files { + if f.contains('modules') || f.contains('preludes') { + continue } + // $if !linux { + // run pg example only on linux + if f.contains('/pg/') { + continue + } + // } + if f.contains('life_gg') || f.contains('/graph.v') || f.contains('rune.v') { + continue + } + $if windows { + // skip pico example on windows + if f.ends_with('examples\\pico\\pico.v') { + continue + } + } + c := os.read_file(f) or { panic(err) } + maxc := if c.len > 300 { 300 } else { c.len } + start := c[0..maxc] + if start.contains('module ') && !start.contains('module main') { + skipped_f := f.replace(os.join_path(parent_dir, ''), '') + skipped << skipped_f + } + for skip_prefix in oskipped { + if f.starts_with(skip_prefix) { + continue next_file + } + } + mains << f } session.files << mains session.skip_files << skipped diff --git a/cmd/tools/vbuild-tools.v b/cmd/tools/vbuild-tools.v index c7d7585032..8c487cfd0a 100644 --- a/cmd/tools/vbuild-tools.v +++ b/cmd/tools/vbuild-tools.v @@ -4,23 +4,41 @@ import os import testing import v.util -fn p(s string) string { - println(s) - return s -} +// NB: tools like vdoc are compiled in their own subfolder +// => cmd/tools/vdoc/vdoc.exe +// Usually, they have several top level .v files in the subfolder, +// that cannot be compiled separately, but instead, the whole folder, +// should be compiled (v folder). +// To implement that, these folders are initially skipped, then added +// as a whole *after the testing.prepare_test_session call*. +const tools_in_subfolders = ['vdoc'] + +// non_packaged_tools are tools that should not be packaged with +// prebuild versions of V, to keep the size smaller. +// They are mainly usefull for the V project itself, not to end users. +const non_packaged_tools = ['gen1m', 'gen_vc', 'fast', 'wyhash'] fn main() { + util.ensure_modules_for_all_tools_are_installed('-v' in os.args) args_string := os.args[1..].join(' ') - skips := []string{} vexe := os.getenv('VEXE') vroot := os.dir(vexe) - util.ensure_modules_for_all_tools_are_installed('-v' in os.args) + os.chdir(vroot) folder := 'cmd/tools' + tfolder := os.join_path(vroot, 'cmd', 'tools') main_label := 'Building $folder ...' finish_label := 'building $folder' - mut session := testing.prepare_test_session(args_string.all_before('build-tools'), - folder, skips, main_label) + // + mut skips := []string{} + for stool in tools_in_subfolders { + skips << os.join_path(tfolder, stool) + } + buildopts := args_string.all_before('build-tools') + mut session := testing.prepare_test_session(buildopts, folder, skips, main_label) session.rm_binaries = false + for stool in tools_in_subfolders { + session.add(os.join_path(tfolder, stool)) + } session.test() eprintln(session.benchmark.total_message(finish_label)) if session.failed { @@ -29,9 +47,17 @@ fn main() { // mut executables := os.ls(session.vtmp_dir) ? executables.sort() - executables = executables.filter(it !in ['gen1m', 'gen_vc', 'fast', 'wyhash']) - for exe in executables { - os.mv_by_cp(os.join_path(session.vtmp_dir, exe), os.join_path(vroot, 'cmd', 'tools', - exe)) + for texe in executables { + tname := texe.replace(os.file_ext(texe), '') + if tname in non_packaged_tools { + continue + } + // + tpath := os.join_path(session.vtmp_dir, texe) + if tname in tools_in_subfolders { + os.mv_by_cp(tpath, os.join_path(tfolder, tname, texe)) + continue + } + os.mv_by_cp(tpath, os.join_path(tfolder, texe)) } } diff --git a/cmd/tools/vdoc/html.v b/cmd/tools/vdoc/html.v new file mode 100644 index 0000000000..0c3f4798a6 --- /dev/null +++ b/cmd/tools/vdoc/html.v @@ -0,0 +1,522 @@ +module main + +import os +import net.urllib +import strings +import markdown +import v.scanner +import v.table +import v.token +import v.doc +import v.pref + +const ( + css_js_assets = ['doc.css', 'normalize.css', 'doc.js', 'dark-mode.js'] + res_path = os.resource_abs_path('resources') + favicons_path = os.join_path(res_path, 'favicons') + html_content = ' + + + + + + {{ title }} | vdoc + + + + + + + + + + + {{ head_assets }} + + +
+ +
+
+
+{{ contents }} +
+ {{ footer_content }} +
+
+ {{ right_content }} +
+
+
+ {{ footer_assets }} + + +' +) + +enum HighlightTokenTyp { + unone + boolean + builtin + char + comment + function + keyword + name + number + operator + punctuation + string + symbol +} + +struct SearchModuleResult { + description string + link string +} + +struct SearchResult { + prefix string + badge string + description string + link string +} + +fn (vd VDoc) render_search_index(out Output) { + mut js_search_index := strings.new_builder(200) + mut js_search_data := strings.new_builder(200) + js_search_index.write('var searchModuleIndex = [') + js_search_data.write('var searchModuleData = [') + for i, title in vd.search_module_index { + data := vd.search_module_data[i] + js_search_index.write('"$title",') + js_search_data.write('["$data.description","$data.link"],') + } + js_search_index.writeln('];') + js_search_index.write('var searchIndex = [') + js_search_data.writeln('];') + js_search_data.write('var searchData = [') + for i, title in vd.search_index { + data := vd.search_data[i] + js_search_index.write('"$title",') + // array instead of object to reduce file size + js_search_data.write('["$data.badge","$data.description","$data.link","$data.prefix"],') + } + js_search_index.writeln('];') + js_search_data.writeln('];') + out_file_path := os.join_path(out.path, 'search_index.js') + os.write_file(out_file_path, js_search_index.str() + js_search_data.str()) +} + +fn (mut vd VDoc) render_static_html(serve_via_http bool, out Output) { + vd.assets = { + 'doc_css': vd.get_resource(css_js_assets[0], true, out) + 'normalize_css': vd.get_resource(css_js_assets[1], true, out) + 'doc_js': vd.get_resource(css_js_assets[2], !serve_via_http, out) + 'dark_mode_js': vd.get_resource(css_js_assets[3], !serve_via_http, out) + 'light_icon': vd.get_resource('light.svg', true, out) + 'dark_icon': vd.get_resource('dark.svg', true, out) + 'menu_icon': vd.get_resource('menu.svg', true, out) + 'arrow_icon': vd.get_resource('arrow.svg', true, out) + } +} + +fn (vd VDoc) get_resource(name string, minify bool, out Output) string { + cfg := vd.cfg + path := os.join_path(res_path, name) + mut res := os.read_file(path) or { panic('vdoc: could not read $path') } + if minify { + if name.ends_with('.js') { + res = js_compress(res) + } else { + res = res.split_into_lines().map(it.trim_space()).join('') + } + } + // TODO: Make SVG inline for now + if cfg.inline_assets || path.ends_with('.svg') { + return res + } else { + output_path := os.join_path(out.path, name) + if !os.exists(output_path) { + println('Generating $out.typ in "$output_path"') + os.write_file(output_path, res) + } + return name + } +} + +fn (mut vd VDoc) collect_search_index(out Output) { + cfg := vd.cfg + for doc in vd.docs { + mod := doc.head.name + vd.search_module_index << mod + comments := if cfg.include_examples { + doc.head.merge_comments() + } else { + doc.head.merge_comments_without_examples() + } + vd.search_module_data << SearchModuleResult{ + description: trim_doc_node_description(comments) + link: vd.get_file_name(mod, out) + } + for _, dn in doc.contents { + vd.create_search_results(mod, dn, out) + } + } +} + +fn (mut vd VDoc) create_search_results(mod string, dn doc.DocNode, out Output) { + cfg := vd.cfg + if dn.kind == .const_group { + return + } + comments := if cfg.include_examples { + dn.merge_comments() + } else { + dn.merge_comments_without_examples() + } + dn_description := trim_doc_node_description(comments) + vd.search_index << dn.name + vd.search_data << SearchResult{ + prefix: if dn.parent_name != '' { + '$dn.kind ($dn.parent_name)' + } else { + '$dn.kind ' + } + description: dn_description + badge: mod + link: vd.get_file_name(mod, out) + '#' + get_node_id(dn) + } + for child in dn.children { + vd.create_search_results(mod, child, out) + } +} + +fn (vd VDoc) write_content(cn &doc.DocNode, d &doc.Doc, mut hw strings.Builder) { + cfg := vd.cfg + base_dir := os.dir(os.real_path(cfg.input_path)) + file_path_name := if cfg.is_multi { + cn.file_path.replace('$base_dir/', '') + } else { + os.file_name(cn.file_path) + } + src_link := get_src_link(vd.manifest.repo_url, file_path_name, cn.pos.line) + if cn.content.len != 0 || (cn.name == 'Constants') { + hw.write(doc_node_html(cn, src_link, false, cfg.include_examples, d.table)) + } + for child in cn.children { + child_file_path_name := child.file_path.replace('$base_dir/', '') + child_src_link := get_src_link(vd.manifest.repo_url, child_file_path_name, child.pos.line) + hw.write(doc_node_html(child, child_src_link, false, cfg.include_examples, d.table)) + } +} + +fn (vd VDoc) gen_html(d doc.Doc) string { + cfg := vd.cfg + mut symbols_toc := strings.new_builder(200) + mut modules_toc := strings.new_builder(200) + mut contents := strings.new_builder(200) + dcs_contents := d.contents.arr() + // generate toc first + contents.writeln(doc_node_html(d.head, '', true, cfg.include_examples, d.table)) + if is_module_readme(d.head) { + write_toc(d.head, mut symbols_toc) + } + for cn in dcs_contents { + vd.write_content(&cn, &d, mut contents) + write_toc(cn, mut symbols_toc) + } // write head + // write css + version := if vd.manifest.version.len != 0 { vd.manifest.version } else { '' } + header_name := if cfg.is_multi && vd.docs.len > 1 { + os.file_name(os.real_path(cfg.input_path)) + } else { + d.head.name + } + // write nav1 + if cfg.is_multi || vd.docs.len > 1 { + mut submod_prefix := '' + for i, dc in vd.docs { + if i - 1 >= 0 && dc.head.name.starts_with(submod_prefix + '.') { + continue + } + names := dc.head.name.split('.') + submod_prefix = if names.len > 1 { names[0] } else { dc.head.name } + mut href_name := './${dc.head.name}.html' + if (cfg.is_vlib && dc.head.name == 'builtin' && !cfg.include_readme) || + dc.head.name == 'README' { + href_name = './index.html' + } else if submod_prefix !in vd.docs.map(it.head.name) { + href_name = '#' + } + submodules := vd.docs.filter(it.head.name.starts_with(submod_prefix + '.')) + dropdown := if submodules.len > 0 { vd.assets['arrow_icon'] } else { '' } + active_class := if dc.head.name == d.head.name { ' active' } else { '' } + modules_toc.write('
  • $dropdown$submod_prefix
    ') + for j, cdoc in submodules { + if j == 0 { + modules_toc.write('') + } + } + modules_toc.write('
    • ') + } + } + modules_toc_str := modules_toc.str() + symbols_toc_str := symbols_toc.str() + modules_toc.free() + symbols_toc.free() + return html_content.replace('{{ title }}', d.head.name).replace('{{ head_name }}', + header_name).replace('{{ version }}', version).replace('{{ light_icon }}', vd.assets['light_icon']).replace('{{ dark_icon }}', + vd.assets['dark_icon']).replace('{{ menu_icon }}', vd.assets['menu_icon']).replace('{{ head_assets }}', + if cfg.inline_assets { + '\n${tabs[0]}\n${tabs[0]}\n${tabs[0]}' + } else { + '\n${tabs[0]}\n${tabs[0]}\n${tabs[0]}' + }).replace('{{ toc_links }}', if cfg.is_multi || vd.docs.len > 1 { + modules_toc_str + } else { + symbols_toc_str + }).replace('{{ contents }}', contents.str()).replace('{{ right_content }}', if cfg.is_multi && + vd.docs.len > 1 && d.head.name != 'README' { + '
    ' + } else { + '' + }).replace('{{ footer_content }}', gen_footer_text(d, !cfg.no_timestamp)).replace('{{ footer_assets }}', + if cfg.inline_assets { + '' + } else { + '' + }) +} + +fn get_src_link(repo_url string, file_name string, line_nr int) string { + mut url := urllib.parse(repo_url) or { return '' } + if url.path.len <= 1 || file_name.len == 0 { + return '' + } + url.path = url.path.trim_right('/') + match url.host { + 'github.com' { '/blob/master/$file_name' } + 'gitlab.com' { '/-/blob/master/$file_name' } + 'git.sir.ht' { '/tree/master/$file_name' } + else { '' } + } + if url.path == '/' { + return '' + } + url.fragment = 'L$line_nr' + return url.str() +} + +fn html_highlight(code string, tb &table.Table) string { + builtin := ['bool', 'string', 'i8', 'i16', 'int', 'i64', 'i128', 'byte', 'u16', 'u32', 'u64', + 'u128', 'rune', 'f32', 'f64', 'any_int', 'any_float', 'byteptr', 'voidptr', 'any'] + highlight_code := fn (tok token.Token, typ HighlightTokenTyp) string { + lit := if typ in [.unone, .operator, .punctuation] { + tok.kind.str() + } else if typ == .string { + "'$tok.lit'" + } else if typ == .char { + '`$tok.lit`' + } else { + tok.lit + } + return if typ in [.unone, .name] { + lit + } else { + '$lit' + } + } + mut s := scanner.new_scanner(code, .parse_comments, &pref.Preferences{}) + mut tok := s.scan() + mut next_tok := s.scan() + mut buf := strings.new_builder(200) + mut i := 0 + for i < code.len { + if i == tok.pos { + mut tok_typ := HighlightTokenTyp.unone + match tok.kind { + .name { + if tok.lit in builtin || tb.known_type(tok.lit) { + tok_typ = .builtin + } else if next_tok.kind == .lcbr { + tok_typ = .symbol + } else if next_tok.kind == .lpar { + tok_typ = .function + } else { + tok_typ = .name + } + } + .comment { + tok_typ = .comment + } + .chartoken { + tok_typ = .char + } + .string { + tok_typ = .string + } + .number { + tok_typ = .number + } + .key_true, .key_false { + tok_typ = .boolean + } + .lpar, .lcbr, .rpar, .rcbr, .lsbr, .rsbr, .semicolon, .colon, .comma, .dot { + tok_typ = .punctuation + } + else { + if token.is_key(tok.lit) || token.is_decl(tok.kind) { + tok_typ = .keyword + } else if tok.kind == .decl_assign || tok.kind.is_assign() || tok.is_unary() || + tok.kind.is_relational() || tok.kind.is_infix() { + tok_typ = .operator + } + } + } + buf.write(highlight_code(tok, tok_typ)) + if next_tok.kind != .eof { + i = tok.pos + tok.len + tok = next_tok + next_tok = s.scan() + } else { + break + } + } else { + buf.write_b(code[i]) + i++ + } + } + return buf.str() +} + +fn doc_node_html(dn doc.DocNode, link string, head bool, include_examples bool, tb &table.Table) string { + mut dnw := strings.new_builder(200) + link_svg := '' + head_tag := if head { 'h1' } else { 'h2' } + comments := dn.merge_comments_without_examples() + md_content := markdown.to_html(html_tag_escape(comments)) + hlighted_code := html_highlight(dn.content, tb) + node_class := if dn.kind == .const_group { ' const' } else { '' } + sym_name := get_sym_name(dn) + mut node_id := get_node_id(dn) + mut hash_link := if !head { ' #' } else { '' } + if head && is_module_readme(dn) { + node_id = 'readme_$node_id' + hash_link = ' #' + } + dnw.writeln('${tabs[1]}
    ') + if dn.name.len > 0 { + if dn.kind == .const_group { + dnw.write('${tabs[2]}
    <$head_tag>$sym_name$hash_link') + } else { + dnw.write('${tabs[2]}
    <$head_tag>$dn.kind $sym_name$hash_link') + } + if link.len != 0 { + dnw.write('$link_svg') + } + dnw.write('
    ') + } + if !head && dn.content.len > 0 { + dnw.writeln('
    $hlighted_code
    ') + } + // do not mess with md_content further, its formatting is important, just output it 1:1 ! + dnw.writeln('$md_content\n') + // Write examples if any found + examples := dn.examples() + if include_examples && examples.len > 0 { + example_title := if examples.len > 1 { 'Examples' } else { 'Example' } + dnw.writeln('

    $example_title

    ') + for example in examples { + // hl_example := html_highlight(example, tb) + dnw.writeln('
    $example
    ') + } + dnw.writeln('
    ') + } + dnw.writeln('
    ') + dnw_str := dnw.str() + defer { + dnw.free() + } + return dnw_str +} + +fn html_tag_escape(str string) string { + return str.replace_each(['<', '<', '>', '>']) +} + +fn js_compress(str string) string { + mut js := strings.new_builder(200) + lines := str.split_into_lines() + rules := [') {', ' = ', ', ', '{ ', ' }', ' (', '; ', ' + ', ' < ', ' - ', ' || ', ' var', + ': ', ' >= ', ' && ', ' else if', ' === ', ' !== ', ' else '] + clean := ['){', '=', ',', '{', '}', '(', ';', '+', '<', '-', '||', 'var', ':', '>=', '&&', + 'else if', '===', '!==', 'else'] + for line in lines { + mut trimmed := line.trim_space() + if trimmed.starts_with('//') || (trimmed.starts_with('/*') && trimmed.ends_with('*/')) { + continue + } + for i in 0 .. rules.len - 1 { + trimmed = trimmed.replace(rules[i], clean[i]) + } + js.write(trimmed) + } + js_str := js.str() + js.free() + return js_str +} + +fn write_toc(dn doc.DocNode, mut toc strings.Builder) { + mut toc_slug := if dn.name.len == 0 || dn.content.len == 0 { '' } else { slug(dn.name) } + if toc_slug == '' && dn.children.len > 0 { + if dn.children[0].name == '' { + toc_slug = slug(dn.name) + } else { + toc_slug = slug(dn.name + '.' + dn.children[0].name) + } + } + if is_module_readme(dn) { + toc.write('
  • README') + } else if dn.name != 'Constants' { + toc.write('
  • $dn.kind $dn.name') + toc.writeln(' ') + } else { + toc.write('
  • $dn.name') + } + toc.writeln('
    • ') +} diff --git a/cmd/tools/vdoc/httpserver.v b/cmd/tools/vdoc/httpserver.v new file mode 100644 index 0000000000..f67dc82a18 --- /dev/null +++ b/cmd/tools/vdoc/httpserver.v @@ -0,0 +1,95 @@ +module main + +import io +import net +import strings + +fn (mut vd VDoc) serve_html(out Output) { + cfg := vd.cfg + if out.typ == .html { + vd.render_static_html(true, out) + } + docs := vd.render(out) + dkeys := docs.keys() + if dkeys.len < 1 { + eprintln('no documentation created, the module has no `pub` functions') + exit(1) + } + def_name := docs.keys()[0] + server_url := 'http://localhost:' + cfg.server_port.str() + server := net.listen_tcp(cfg.server_port) or { panic(err) } + println('Serving docs on: $server_url') + if cfg.open_docs { + open_url(server_url) + } + content_type := match out.typ { + .html { 'text/html' } + .markdown { 'text/markdown' } + .json { 'application/json' } + else { 'text/plain' } + } + server_context := VdocHttpServerContext{ + docs: docs + content_type: content_type + default_filename: def_name + } + for { + mut conn := server.accept() or { + server.close() or { } + panic(err) + } + handle_http_connection(mut conn, server_context) + conn.close() or { eprintln('error closing the connection: $err') } + } +} + +struct VdocHttpServerContext { + docs map[string]string + content_type string + default_filename string +} + +fn handle_http_connection(mut con net.TcpConn, ctx &VdocHttpServerContext) { + mut reader := io.new_buffered_reader(reader: io.make_reader(con)) + first_line := reader.read_line() or { + send_http_response(mut con, 501, ctx.content_type, 'bad request') + return + } + request_parts := first_line.split(' ') + if request_parts.len != 3 { + send_http_response(mut con, 501, ctx.content_type, 'bad request') + return + } + urlpath := request_parts[1] + filename := if urlpath == '/' { + ctx.default_filename.trim_left('/') + } else { + urlpath.trim_left('/') + } + if ctx.docs[filename].len == 0 { + send_http_response(mut con, 404, ctx.content_type, 'file not found') + return + } + send_http_response(mut con, 200, ctx.content_type, ctx.docs[filename]) +} + +fn send_http_response(mut con net.TcpConn, http_code int, content_type string, html string) { + content_length := html.len.str() + shttp_code := http_code.str() + mut http_response := strings.new_builder(20000) + http_response.write('HTTP/1.1 ') + http_response.write(shttp_code) + http_response.write(' OK\r\n') + http_response.write('Server: VDoc\r\n') + http_response.write('Content-Type: ') + http_response.write(content_type) + http_response.write('\r\n') + http_response.write('Content-Length: ') + http_response.write(content_length) + http_response.write('\r\n') + http_response.write('Connection: close\r\n') + http_response.write('\r\n') + http_response.write(html) + sresponse := http_response.str() + con.write_str(sresponse) or { eprintln('error sending http response: $err') } +} diff --git a/cmd/tools/vdoc/markdown.v b/cmd/tools/vdoc/markdown.v new file mode 100644 index 0000000000..01000a86e1 --- /dev/null +++ b/cmd/tools/vdoc/markdown.v @@ -0,0 +1,50 @@ +module main + +import strings +import v.doc + +fn (vd VDoc) gen_markdown(d doc.Doc, with_toc bool) string { + mut hw := strings.new_builder(200) + mut cw := strings.new_builder(200) + hw.writeln('# $d.head.content\n') + if d.head.comments.len > 0 { + comments := if vd.cfg.include_examples { + d.head.merge_comments() + } else { + d.head.merge_comments_without_examples() + } + hw.writeln('$comments\n') + } + if with_toc { + hw.writeln('## Contents') + } + vd.write_markdown_content(d.contents.arr(), mut cw, mut hw, 0, with_toc) + footer_text := gen_footer_text(d, !vd.cfg.no_timestamp) + cw.writeln('#### $footer_text') + return hw.str() + '\n' + cw.str() +} + +fn (vd VDoc) write_markdown_content(contents []doc.DocNode, mut cw strings.Builder, mut hw strings.Builder, indent int, with_toc bool) { + for cn in contents { + if with_toc && cn.name.len > 0 { + hw.writeln(' '.repeat(2 * indent) + '- [#$cn.name](${slug(cn.name)})') + cw.writeln('## $cn.name') + } + if cn.content.len > 0 { + comments := cn.merge_comments_without_examples() + cw.writeln('```v\n$cn.content\n```\n$comments\n') + // Write examples if any found + examples := cn.examples() + if vd.cfg.include_examples && examples.len > 0 { + example_title := if examples.len > 1 { 'Examples' } else { 'Example' } + cw.writeln('$example_title\n```v\n') + for example in examples { + cw.writeln('$example\n') + } + cw.writeln('```\n') + } + cw.writeln('[\[Return to contents\]](#Contents)\n') + } + vd.write_markdown_content(cn.children, mut cw, mut hw, indent + 1, with_toc) + } +} diff --git a/cmd/tools/vdoc/utils.v b/cmd/tools/vdoc/utils.v new file mode 100644 index 0000000000..4f0f488bc9 --- /dev/null +++ b/cmd/tools/vdoc/utils.v @@ -0,0 +1,145 @@ +module main + +import os +import v.doc + +[inline] +fn slug(title string) string { + return title.replace(' ', '-') +} + +[inline] +fn open_url(url string) { + $if windows { + os.system('start $url') + } + $if macos { + os.system('open $url') + } + $if linux { + os.system('xdg-open $url') + } +} + +fn escape(str string) string { + return str.replace_each(['"', '\\"', '\r\n', '\\n', '\n', '\\n', '\t', '\\t']) +} + +fn get_sym_name(dn doc.DocNode) string { + sym_name := if dn.parent_name.len > 0 && dn.parent_name != 'void' { + '($dn.parent_name) $dn.name' + } else { + dn.name + } + return sym_name +} + +fn get_node_id(dn doc.DocNode) string { + tag := if dn.parent_name.len > 0 && dn.parent_name != 'void' { + '${dn.parent_name}.$dn.name' + } else { + dn.name + } + return slug(tag) +} + +fn is_module_readme(dn doc.DocNode) bool { + if dn.comments.len > 0 && dn.content == 'module $dn.name' { + return true + } + return false +} + +fn trim_doc_node_description(description string) string { + mut dn_description := description.replace_each(['\r\n', '\n', '"', '\\"']) + // 80 is enough to fill one line + if dn_description.len > 80 { + dn_description = dn_description[..80] + } + if '\n' in dn_description { + dn_description = dn_description.split('\n')[0] + } + // if \ is last character, it ends with \" which leads to a JS error + if dn_description.ends_with('\\') { + dn_description = dn_description.trim_right('\\') + } + return dn_description +} + +fn set_output_type_from_str(format string) OutputType { + output_type := match format { + 'htm', 'html' { OutputType.html } + 'md', 'markdown' { OutputType.markdown } + 'json' { OutputType.json } + 'stdout' { OutputType.stdout } + else { OutputType.plaintext } + } + return output_type +} + +fn get_ignore_paths(path string) ?[]string { + ignore_file_path := os.join_path(path, '.vdocignore') + ignore_content := os.read_file(ignore_file_path) or { + return error_with_code('ignore file not found.', 1) + } + mut res := []string{} + if ignore_content.trim_space().len > 0 { + rules := ignore_content.split_into_lines().map(it.trim_space()) + mut final := []string{} + for rule in rules { + if rule.contains('*.') || rule.contains('**') { + println('vdoc: Wildcards in ignore rules are not allowed for now.') + continue + } + final << rule + } + res = final.map(os.join_path(path, it.trim_right('/'))) + } else { + mut dirs := os.ls(path) or { return []string{} } + res = dirs.map(os.join_path(path, it)).filter(os.is_dir(it)) + } + return res.map(it.replace('/', os.path_separator)) +} + +fn is_included(path string, ignore_paths []string) bool { + if path.len == 0 { + return true + } + for ignore_path in ignore_paths { + if ignore_path !in path { + continue + } + return false + } + return true +} + +fn get_modules_list(path string, ignore_paths2 []string) []string { + files := os.ls(path) or { return []string{} } + mut ignore_paths := get_ignore_paths(path) or { []string{} } + ignore_paths << ignore_paths2 + mut dirs := []string{} + for file in files { + fpath := os.join_path(path, file) + if os.is_dir(fpath) && is_included(fpath, ignore_paths) && !os.is_link(path) { + dirs << get_modules_list(fpath, ignore_paths.filter(it.starts_with(fpath))) + } else if fpath.ends_with('.v') && !fpath.ends_with('_test.v') { + if path in dirs { + continue + } + dirs << path + } + } + dirs.sort() + return dirs +} + +fn gen_footer_text(d &doc.Doc, include_timestamp bool) string { + footer_text := 'Powered by vdoc.' + if !include_timestamp { + return footer_text + } + generated_time := d.time_generated + time_str := '$generated_time.day $generated_time.smonth() $generated_time.year $generated_time.hhmmss()' + return '$footer_text Generated on: $time_str' +} diff --git a/cmd/tools/vdoc/vdoc.v b/cmd/tools/vdoc/vdoc.v index 4f96abd6fc..67db6fa709 100644 --- a/cmd/tools/vdoc/vdoc.v +++ b/cmd/tools/vdoc/vdoc.v @@ -1,8 +1,6 @@ module main import markdown -import net -import net.urllib import os import os.cmdline import time @@ -10,93 +8,14 @@ import strings import sync import runtime import v.doc -import v.scanner -import v.table -import v.token -import v.vmod import v.pref +import v.vmod import json -import io - -enum HighlightTokenTyp { - unone - boolean - builtin - char - comment - function - keyword - name - number - operator - punctuation - string - symbol -} const ( - css_js_assets = ['doc.css', 'normalize.css', 'doc.js', 'dark-mode.js'] allowed_formats = ['md', 'markdown', 'json', 'text', 'stdout', 'html', 'htm'] - res_path = os.resource_abs_path('resources') - favicons_path = os.join_path(res_path, 'favicons') vexe = pref.vexe_path() vroot = os.dir(vexe) - html_content = ' - - - - - - {{ title }} | vdoc - - - - - - - - - - - {{ head_assets }} - - -
    - -
    -
    -
    -{{ contents }} -
    - {{ footer_content }} -
    -
    - {{ right_content }} -
    -
    -
    - {{ footer_assets }} - - -' tabs = ['\t\t', '\t\t\t\t\t\t', '\t\t\t\t\t\t\t'] ) @@ -109,531 +28,85 @@ enum OutputType { stdout } -struct DocConfig { +struct VDoc { + cfg Config [required] mut: - is_local bool - local_filename string - local_pos int - pub_only bool = true - show_loc bool // for plaintext - serve_http bool // for html - is_multi bool - is_vlib bool - is_verbose bool - include_readme bool - include_examples bool = true - open_docs bool - server_port int = 8046 - inline_assets bool - no_timestamp bool - output_path string - input_path string - symbol_name string - output_type OutputType = .unset docs []doc.Doc - manifest vmod.Manifest assets map[string]string + manifest vmod.Manifest search_index []string search_data []SearchResult search_module_index []string // search results are split into a module part and the rest search_module_data []SearchModuleResult } -struct SearchModuleResult { - description string - link string +struct Config { +mut: + pub_only bool = true + is_local bool + local_filename string + local_pos int + show_loc bool // for plaintext + serve_http bool // for html + is_multi bool + is_vlib bool + is_verbose bool + include_readme bool + include_examples bool = true + open_docs bool + server_port int = 8046 + inline_assets bool + no_timestamp bool + output_path string + output_type OutputType = .unset + input_path string + symbol_name string } -struct SearchResult { - prefix string - badge string - description string - link string +// +struct Output { +mut: + path string + typ OutputType = .unset } struct ParallelDoc { - d doc.Doc - i int + d doc.Doc + out Output } -[inline] -fn slug(title string) string { - return title.replace(' ', '-') -} - -[inline] -fn open_url(url string) { - $if windows { - os.system('start $url') - } - $if macos { - os.system('open $url') - } - $if linux { - os.system('xdg-open $url') - } -} - -fn (mut cfg DocConfig) serve_html() { - cfg.render_static() - docs := cfg.render() - dkeys := docs.keys() - if dkeys.len < 1 { - eprintln('no documentation created, the module has no `pub` functions') - exit(1) - } - def_name := docs.keys()[0] - server_url := 'http://localhost:' + cfg.server_port.str() - server := net.listen_tcp(cfg.server_port) or { panic(err) } - println('Serving docs on: $server_url') - if cfg.open_docs { - open_url(server_url) - } - content_type := match cfg.output_type { - .html { 'text/html' } - .markdown { 'text/markdown' } - .json { 'application/json' } - else { 'text/plain' } - } - server_context := VdocHttpServerContext{ - docs: docs - content_type: content_type - default_filename: def_name - } - for { - mut conn := server.accept() or { - server.close() or { } - panic(err) - } - handle_http_connection(mut conn, server_context) - conn.close() or { eprintln('error closing the connection: $err') } - } -} - -struct VdocHttpServerContext { - docs map[string]string - content_type string - default_filename string -} - -fn handle_http_connection(mut con net.TcpConn, ctx &VdocHttpServerContext) { - mut reader := io.new_buffered_reader(reader: io.make_reader(con)) - first_line := reader.read_line() or { - send_http_response(mut con, 501, ctx.content_type, 'bad request') - return - } - request_parts := first_line.split(' ') - if request_parts.len != 3 { - send_http_response(mut con, 501, ctx.content_type, 'bad request') - return - } - urlpath := request_parts[1] - filename := if urlpath == '/' { - ctx.default_filename.trim_left('/') - } else { - urlpath.trim_left('/') - } - if ctx.docs[filename].len == 0 { - send_http_response(mut con, 404, ctx.content_type, 'file not found') - return - } - send_http_response(mut con, 200, ctx.content_type, ctx.docs[filename]) -} - -fn send_http_response(mut con net.TcpConn, http_code int, content_type string, html string) { - content_length := html.len.str() - shttp_code := http_code.str() - mut http_response := strings.new_builder(20000) - http_response.write('HTTP/1.1 ') - http_response.write(shttp_code) - http_response.write(' OK\r\n') - http_response.write('Server: VDoc\r\n') - http_response.write('Content-Type: ') - http_response.write(content_type) - http_response.write('\r\n') - http_response.write('Content-Length: ') - http_response.write(content_length) - http_response.write('\r\n') - http_response.write('Connection: close\r\n') - http_response.write('\r\n') - http_response.write(html) - sresponse := http_response.str() - con.write_str(sresponse) or { eprintln('error sending http response: $err') } -} - -fn get_src_link(repo_url string, file_name string, line_nr int) string { - mut url := urllib.parse(repo_url) or { return '' } - if url.path.len <= 1 || file_name.len == 0 { - return '' - } - url.path = url.path.trim_right('/') + match url.host { - 'github.com' { '/blob/master/$file_name' } - 'gitlab.com' { '/-/blob/master/$file_name' } - 'git.sir.ht' { '/tree/master/$file_name' } - else { '' } - } - if url.path == '/' { - return '' - } - url.fragment = 'L$line_nr' - return url.str() -} - -fn js_compress(str string) string { - mut js := strings.new_builder(200) - lines := str.split_into_lines() - rules := [') {', ' = ', ', ', '{ ', ' }', ' (', '; ', ' + ', ' < ', ' - ', ' || ', ' var', - ': ', ' >= ', ' && ', ' else if', ' === ', ' !== ', ' else '] - clean := ['){', '=', ',', '{', '}', '(', ';', '+', '<', '-', '||', 'var', ':', '>=', '&&', - 'else if', '===', '!==', 'else'] - for line in lines { - mut trimmed := line.trim_space() - if trimmed.starts_with('//') || (trimmed.starts_with('/*') && trimmed.ends_with('*/')) { - continue - } - for i in 0 .. rules.len - 1 { - trimmed = trimmed.replace(rules[i], clean[i]) - } - js.write(trimmed) - } - js_str := js.str() - js.free() - return js_str -} - -fn escape(str string) string { - return str.replace_each(['"', '\\"', '\r\n', '\\n', '\n', '\\n', '\t', '\\t']) -} - -fn (cfg DocConfig) gen_json(idx int) string { - dcs := cfg.docs[idx] +fn (vd VDoc) gen_json(d doc.Doc) string { + cfg := vd.cfg mut jw := strings.new_builder(200) comments := if cfg.include_examples { - dcs.head.merge_comments() + d.head.merge_comments() } else { - dcs.head.merge_comments_without_examples() + d.head.merge_comments_without_examples() } - jw.write('{"module_name":"$dcs.head.name","description":"${escape(comments)}","contents":') - jw.write(json.encode(dcs.contents.keys().map(dcs.contents[it]))) - jw.write(',"generator":"vdoc","time_generated":"$dcs.time_generated.str()"}') + jw.write('{"module_name":"$d.head.name","description":"${escape(comments)}","contents":') + jw.write(json.encode(d.contents.keys().map(d.contents[it]))) + jw.write(',"generator":"vdoc","time_generated":"$d.time_generated.str()"}') return jw.str() } -fn html_highlight(code string, tb &table.Table) string { - builtin := ['bool', 'string', 'i8', 'i16', 'int', 'i64', 'i128', 'byte', 'u16', 'u32', 'u64', - 'u128', 'rune', 'f32', 'f64', 'any_int', 'any_float', 'byteptr', 'voidptr', 'any'] - highlight_code := fn (tok token.Token, typ HighlightTokenTyp) string { - lit := if typ in [.unone, .operator, .punctuation] { - tok.kind.str() - } else if typ == .string { - "'$tok.lit'" - } else if typ == .char { - '`$tok.lit`' - } else { - tok.lit - } - return if typ in [.unone, .name] { - lit - } else { - '$lit' - } - } - mut s := scanner.new_scanner(code, .parse_comments, &pref.Preferences{}) - mut tok := s.scan() - mut next_tok := s.scan() - mut buf := strings.new_builder(200) - mut i := 0 - for i < code.len { - if i == tok.pos { - mut tok_typ := HighlightTokenTyp.unone - match tok.kind { - .name { - if tok.lit in builtin || tb.known_type(tok.lit) { - tok_typ = .builtin - } else if next_tok.kind == .lcbr { - tok_typ = .symbol - } else if next_tok.kind == .lpar { - tok_typ = .function - } else { - tok_typ = .name - } - } - .comment { - tok_typ = .comment - } - .chartoken { - tok_typ = .char - } - .string { - tok_typ = .string - } - .number { - tok_typ = .number - } - .key_true, .key_false { - tok_typ = .boolean - } - .lpar, .lcbr, .rpar, .rcbr, .lsbr, .rsbr, .semicolon, .colon, .comma, .dot { - tok_typ = .punctuation - } - else { - if token.is_key(tok.lit) || token.is_decl(tok.kind) { - tok_typ = .keyword - } else if tok.kind == .decl_assign || tok.kind.is_assign() || tok.is_unary() || - tok.kind.is_relational() || tok.kind.is_infix() { - tok_typ = .operator - } - } - } - buf.write(highlight_code(tok, tok_typ)) - if next_tok.kind != .eof { - i = tok.pos + tok.len - tok = next_tok - next_tok = s.scan() - } else { - break - } - } else { - buf.write_b(code[i]) - i++ - } - } - return buf.str() -} - -fn doc_node_html(dd doc.DocNode, link string, head bool, include_examples bool, tb &table.Table) string { - mut dnw := strings.new_builder(200) - link_svg := '' - head_tag := if head { 'h1' } else { 'h2' } - comments := dd.merge_comments_without_examples() - md_content := markdown.to_html(html_tag_escape(comments)) - hlighted_code := html_highlight(dd.content, tb) - node_class := if dd.kind == .const_group { ' const' } else { '' } - sym_name := get_sym_name(dd) - mut node_id := get_node_id(dd) - mut hash_link := if !head { ' #' } else { '' } - if head && is_module_readme(dd) { - node_id = 'readme_$node_id' - hash_link = ' #' - } - dnw.writeln('${tabs[1]}
    ') - if dd.name.len > 0 { - if dd.kind == .const_group { - dnw.write('${tabs[2]}
    <$head_tag>$sym_name$hash_link') - } else { - dnw.write('${tabs[2]}
    <$head_tag>$dd.kind $sym_name$hash_link') - } - if link.len != 0 { - dnw.write('$link_svg') - } - dnw.write('
    ') - } - if !head && dd.content.len > 0 { - dnw.writeln('
    $hlighted_code
    ') - } - // do not mess with md_content further, its formatting is important, just output it 1:1 ! - dnw.writeln('$md_content\n') - // Write examples if any found - examples := dd.examples() - if include_examples && examples.len > 0 { - example_title := if examples.len > 1 { 'Examples' } else { 'Example' } - dnw.writeln('

    $example_title

    ') - for example in examples { - // hl_example := html_highlight(example, tb) - dnw.writeln('
    $example
    ') - } - dnw.writeln('
    ') - } - dnw.writeln('
    ') - dnw_str := dnw.str() - defer { - dnw.free() - } - return dnw_str -} - -fn html_tag_escape(str string) string { - return str.replace_each(['<', '<', '>', '>']) -} - -fn get_sym_name(dn doc.DocNode) string { - sym_name := if dn.parent_name.len > 0 && dn.parent_name != 'void' { - '($dn.parent_name) $dn.name' - } else { - dn.name - } - return sym_name -} - -fn get_node_id(dn doc.DocNode) string { - tag := if dn.parent_name.len > 0 && dn.parent_name != 'void' { - '${dn.parent_name}.$dn.name' - } else { - dn.name - } - return slug(tag) -} - -fn (cfg DocConfig) readme_idx() int { - for i, dc in cfg.docs { - if dc.head.name != 'README' { - continue - } - return i - } - return -1 -} - -fn write_toc(dn doc.DocNode, mut toc strings.Builder) { - mut toc_slug := if dn.name.len == 0 || dn.content.len == 0 { '' } else { slug(dn.name) } - if toc_slug == '' && dn.children.len > 0 { - if dn.children[0].name == '' { - toc_slug = slug(dn.name) - } else { - toc_slug = slug(dn.name + '.' + dn.children[0].name) - } - } - if is_module_readme(dn) { - toc.write('
  • README') - } else if dn.name != 'Constants' { - toc.write('
  • $dn.kind $dn.name') - toc.writeln(' ') - } else { - toc.write('
  • $dn.name') - } - toc.writeln('
    • ') -} - -fn (cfg DocConfig) write_content(cn &doc.DocNode, dcs &doc.Doc, mut hw strings.Builder) { - base_dir := os.dir(os.real_path(cfg.input_path)) - file_path_name := if cfg.is_multi { - cn.file_path.replace('$base_dir/', '') - } else { - os.file_name(cn.file_path) - } - src_link := get_src_link(cfg.manifest.repo_url, file_path_name, cn.pos.line) - if cn.content.len != 0 || (cn.name == 'Constants') { - hw.write(doc_node_html(cn, src_link, false, cfg.include_examples, dcs.table)) - } - for child in cn.children { - child_file_path_name := child.file_path.replace('$base_dir/', '') - child_src_link := get_src_link(cfg.manifest.repo_url, child_file_path_name, child.pos.line) - hw.write(doc_node_html(child, child_src_link, false, cfg.include_examples, dcs.table)) - } -} - -fn (cfg DocConfig) gen_html(idx int) string { - mut symbols_toc := strings.new_builder(200) - mut modules_toc := strings.new_builder(200) - mut contents := strings.new_builder(200) - dcs := cfg.docs[idx] - dcs_contents := dcs.contents.arr() - // generate toc first - contents.writeln(doc_node_html(dcs.head, '', true, cfg.include_examples, dcs.table)) - if is_module_readme(dcs.head) { - write_toc(dcs.head, mut symbols_toc) - } - for cn in dcs_contents { - cfg.write_content(&cn, &dcs, mut contents) - write_toc(cn, mut symbols_toc) - } // write head - // write css - version := if cfg.manifest.version.len != 0 { cfg.manifest.version } else { '' } - header_name := if cfg.is_multi && cfg.docs.len > 1 { - os.file_name(os.real_path(cfg.input_path)) - } else { - dcs.head.name - } - // write nav1 - if cfg.is_multi || cfg.docs.len > 1 { - mut submod_prefix := '' - for i, doc in cfg.docs { - if i - 1 >= 0 && doc.head.name.starts_with(submod_prefix + '.') { - continue - } - names := doc.head.name.split('.') - submod_prefix = if names.len > 1 { names[0] } else { doc.head.name } - mut href_name := './${doc.head.name}.html' - if (cfg.is_vlib && doc.head.name == 'builtin' && !cfg.include_readme) || - doc.head.name == 'README' { - href_name = './index.html' - } else if submod_prefix !in cfg.docs.map(it.head.name) { - href_name = '#' - } - submodules := cfg.docs.filter(it.head.name.starts_with(submod_prefix + '.')) - dropdown := if submodules.len > 0 { cfg.assets['arrow_icon'] } else { '' } - active_class := if doc.head.name == dcs.head.name { ' active' } else { '' } - modules_toc.write('
  • $dropdown$submod_prefix
    ') - for j, cdoc in submodules { - if j == 0 { - modules_toc.write('') - } - } - modules_toc.write('
    • ') - } - } - modules_toc_str := modules_toc.str() - symbols_toc_str := symbols_toc.str() - modules_toc.free() - symbols_toc.free() - return html_content.replace('{{ title }}', dcs.head.name).replace('{{ head_name }}', - header_name).replace('{{ version }}', version).replace('{{ light_icon }}', cfg.assets['light_icon']).replace('{{ dark_icon }}', - cfg.assets['dark_icon']).replace('{{ menu_icon }}', cfg.assets['menu_icon']).replace('{{ head_assets }}', - if cfg.inline_assets { - '\n${tabs[0]}\n${tabs[0]}\n${tabs[0]}' - } else { - '\n${tabs[0]}\n${tabs[0]}\n${tabs[0]}' - }).replace('{{ toc_links }}', if cfg.is_multi || cfg.docs.len > 1 { - modules_toc_str - } else { - symbols_toc_str - }).replace('{{ contents }}', contents.str()).replace('{{ right_content }}', if cfg.is_multi && - cfg.docs.len > 1 && dcs.head.name != 'README' { - '
    ' - } else { - '' - }).replace('{{ footer_content }}', cfg.gen_footer_text(idx)).replace('{{ footer_assets }}', - if cfg.inline_assets { - '' - } else { - '' - }) -} - -fn (cfg DocConfig) gen_plaintext(idx int) string { - dcs := cfg.docs[idx] +fn (vd VDoc) gen_plaintext(d doc.Doc) string { + cfg := vd.cfg mut pw := strings.new_builder(200) - pw.writeln('$dcs.head.content\n') + pw.writeln('$d.head.content\n') comments := if cfg.include_examples { - dcs.head.merge_comments() + d.head.merge_comments() } else { - dcs.head.merge_comments_without_examples() + d.head.merge_comments_without_examples() } if comments.trim_space().len > 0 && !cfg.pub_only { pw.writeln(comments.split_into_lines().map(' ' + it).join('\n')) } - cfg.write_plaintext_content(dcs.contents.arr(), mut pw) + vd.write_plaintext_content(d.contents.arr(), mut pw) return pw.str() } -fn (cfg DocConfig) write_plaintext_content(contents []doc.DocNode, mut pw strings.Builder) { +fn (vd VDoc) write_plaintext_content(contents []doc.DocNode, mut pw strings.Builder) { + cfg := vd.cfg for cn in contents { if cn.content.len > 0 { pw.writeln(cn.content) @@ -649,89 +122,32 @@ fn (cfg DocConfig) write_plaintext_content(contents []doc.DocNode, mut pw string pw.writeln('Location: $cn.file_path:$cn.pos.line\n') } } - cfg.write_plaintext_content(cn.children, mut pw) + vd.write_plaintext_content(cn.children, mut pw) } } -fn (cfg DocConfig) gen_markdown(idx int, with_toc bool) string { - dcs := cfg.docs[idx] - mut hw := strings.new_builder(200) - mut cw := strings.new_builder(200) - hw.writeln('# $dcs.head.content\n') - if dcs.head.comments.len > 0 { - comments := if cfg.include_examples { - dcs.head.merge_comments() - } else { - dcs.head.merge_comments_without_examples() - } - hw.writeln('$comments\n') - } - if with_toc { - hw.writeln('## Contents') - } - cfg.write_markdown_content(dcs.contents.arr(), mut cw, mut hw, 0, with_toc) - footer_text := cfg.gen_footer_text(idx) - cw.writeln('#### $footer_text') - return hw.str() + '\n' + cw.str() -} - -fn (cfg DocConfig) write_markdown_content(contents []doc.DocNode, mut cw strings.Builder, mut hw strings.Builder, indent int, with_toc bool) { - for cn in contents { - if with_toc && cn.name.len > 0 { - hw.writeln(' '.repeat(2 * indent) + '- [#$cn.name](${slug(cn.name)})') - cw.writeln('## $cn.name') - } - if cn.content.len > 0 { - comments := cn.merge_comments_without_examples() - cw.writeln('```v\n$cn.content\n```\n$comments\n') - // Write examples if any found - examples := cn.examples() - if cfg.include_examples && examples.len > 0 { - example_title := if examples.len > 1 { 'Examples' } else { 'Example' } - cw.writeln('$example_title\n```v\n') - for example in examples { - cw.writeln('$example\n') - } - cw.writeln('```\n') - } - cw.writeln('[\[Return to contents\]](#Contents)\n') - } - cfg.write_markdown_content(cn.children, mut cw, mut hw, indent + 1, with_toc) - } -} - -fn (cfg DocConfig) gen_footer_text(idx int) string { - current_doc := cfg.docs[idx] - footer_text := 'Powered by vdoc.' - if cfg.no_timestamp { - return footer_text - } - generated_time := current_doc.time_generated - time_str := '$generated_time.day $generated_time.smonth() $generated_time.year $generated_time.hhmmss()' - return '$footer_text Generated on: $time_str' -} - -fn (cfg DocConfig) render_doc(doc doc.Doc, i int) (string, string) { - name := cfg.get_file_name(doc.head.name) - output := match cfg.output_type { - .html { cfg.gen_html(i) } - .markdown { cfg.gen_markdown(i, true) } - .json { cfg.gen_json(i) } - else { cfg.gen_plaintext(i) } +fn (vd VDoc) render_doc(d doc.Doc, out Output) (string, string) { + name := vd.get_file_name(d.head.name, out) + output := match out.typ { + .html { vd.gen_html(d) } + .markdown { vd.gen_markdown(d, true) } + .json { vd.gen_json(d) } + else { vd.gen_plaintext(d) } } return name, output } // get_file_name returns the final file name from a module name -fn (cfg DocConfig) get_file_name(mod string) string { +fn (vd VDoc) get_file_name(mod string, out Output) string { + cfg := vd.cfg mut name := mod // since builtin is generated first, ignore it if (cfg.is_vlib && mod == 'builtin' && !cfg.include_readme) || mod == 'README' { name = 'index' - } else if !cfg.is_multi && !os.is_dir(cfg.output_path) { - name = os.file_name(cfg.output_path) + } else if !cfg.is_multi && !os.is_dir(out.path) { + name = os.file_name(out.path) } - name = name + match cfg.output_type { + name = name + match out.typ { .html { '.html' } .markdown { '.md' } .json { '.json' } @@ -740,153 +156,47 @@ fn (cfg DocConfig) get_file_name(mod string) string { return name } -fn (cfg DocConfig) work_processor(mut work sync.Channel, mut wg sync.WaitGroup) { +fn (vd VDoc) work_processor(mut work sync.Channel, mut wg sync.WaitGroup) { for { mut pdoc := ParallelDoc{} if !work.pop(&pdoc) { break } - file_name, content := cfg.render_doc(pdoc.d, pdoc.i) - output_path := os.join_path(cfg.output_path, file_name) - println('Generating $output_path') + file_name, content := vd.render_doc(pdoc.d, pdoc.out) + output_path := os.join_path(pdoc.out.path, file_name) + println('Generating $pdoc.out.typ in "$output_path"') os.write_file(output_path, content) } wg.done() } -fn (cfg DocConfig) render_parallel() { +fn (vd VDoc) render_parallel(out Output) { vjobs := runtime.nr_jobs() - mut work := sync.new_channel(cfg.docs.len) + mut work := sync.new_channel(vd.docs.len) mut wg := sync.new_waitgroup() - for i in 0 .. cfg.docs.len { - p_doc := ParallelDoc{cfg.docs[i], i} + for i in 0 .. vd.docs.len { + p_doc := ParallelDoc{vd.docs[i], out} work.push(&p_doc) } work.close() wg.add(vjobs) for _ in 0 .. vjobs { - go cfg.work_processor(mut work, mut wg) + go vd.work_processor(mut work, mut wg) } wg.wait() } -fn (cfg DocConfig) render() map[string]string { +fn (vd VDoc) render(out Output) map[string]string { mut docs := map[string]string{} - for i, doc in cfg.docs { - name, output := cfg.render_doc(doc, i) + for doc in vd.docs { + name, output := vd.render_doc(doc, out) docs[name] = output.trim_space() } - cfg.vprintln('Rendered: ' + docs.keys().str()) + vd.vprintln('Rendered: ' + docs.keys().str()) return docs } -fn (mut cfg DocConfig) collect_search_index() { - if cfg.output_type != .html { - return - } - for doc in cfg.docs { - mod := doc.head.name - cfg.search_module_index << mod - comments := if cfg.include_examples { - doc.head.merge_comments() - } else { - doc.head.merge_comments_without_examples() - } - cfg.search_module_data << SearchModuleResult{ - description: trim_doc_node_description(comments) - link: cfg.get_file_name(mod) - } - for _, dn in doc.contents { - cfg.create_search_results(mod, dn) - } - } -} - -fn (mut cfg DocConfig) create_search_results(mod string, dn doc.DocNode) { - if dn.kind == .const_group { - return - } - comments := if cfg.include_examples { - dn.merge_comments() - } else { - dn.merge_comments_without_examples() - } - dn_description := trim_doc_node_description(comments) - cfg.search_index << dn.name - cfg.search_data << SearchResult{ - prefix: if dn.parent_name != '' { - '$dn.kind ($dn.parent_name)' - } else { - '$dn.kind ' - } - description: dn_description - badge: mod - link: cfg.get_file_name(mod) + '#' + get_node_id(dn) - } - for child in dn.children { - cfg.create_search_results(mod, child) - } -} - -fn trim_doc_node_description(description string) string { - mut dn_description := description.replace_each(['\r\n', '\n', '"', '\\"']) - // 80 is enough to fill one line - if dn_description.len > 80 { - dn_description = dn_description[..80] - } - if '\n' in dn_description { - dn_description = dn_description.split('\n')[0] - } - // if \ is last character, it ends with \" which leads to a JS error - if dn_description.ends_with('\\') { - dn_description = dn_description.trim_right('\\') - } - return dn_description -} - -fn (cfg DocConfig) render_search_index() { - mut js_search_index := strings.new_builder(200) - mut js_search_data := strings.new_builder(200) - js_search_index.write('var searchModuleIndex = [') - js_search_data.write('var searchModuleData = [') - for i, title in cfg.search_module_index { - data := cfg.search_module_data[i] - js_search_index.write('"$title",') - js_search_data.write('["$data.description","$data.link"],') - } - js_search_index.writeln('];') - js_search_index.write('var searchIndex = [') - js_search_data.writeln('];') - js_search_data.write('var searchData = [') - for i, title in cfg.search_index { - data := cfg.search_data[i] - js_search_index.write('"$title",') - // array instead of object to reduce file size - js_search_data.write('["$data.badge","$data.description","$data.link","$data.prefix"],') - } - js_search_index.writeln('];') - js_search_data.writeln('];') - out_file_path := os.join_path(cfg.output_path, 'search_index.js') - os.write_file(out_file_path, js_search_index.str() + js_search_data.str()) -} - -fn (mut cfg DocConfig) render_static() { - if cfg.output_type != .html { - return - } - cfg.assets = { - 'doc_css': cfg.get_resource(css_js_assets[0], true) - 'normalize_css': cfg.get_resource(css_js_assets[1], true) - 'doc_js': cfg.get_resource(css_js_assets[2], !cfg.serve_http) - 'dark_mode_js': cfg.get_resource(css_js_assets[3], !cfg.serve_http) - 'light_icon': cfg.get_resource('light.svg', true) - 'dark_icon': cfg.get_resource('dark.svg', true) - 'menu_icon': cfg.get_resource('menu.svg', true) - 'arrow_icon': cfg.get_resource('arrow.svg', true) - } -} - -fn (cfg DocConfig) get_readme(path string) string { +fn (vd VDoc) get_readme(path string) string { mut fname := '' for name in ['readme', 'README'] { if os.exists(os.join_path(path, '${name}.md')) { @@ -898,12 +208,13 @@ fn (cfg DocConfig) get_readme(path string) string { return '' } readme_path := os.join_path(path, '${fname}.md') - cfg.vprintln('Reading README file from $readme_path') + vd.vprintln('Reading README file from $readme_path') readme_contents := os.read_file(readme_path) or { '' } return readme_contents } -fn (cfg DocConfig) emit_generate_err(err string, errcode int) { +fn (vd VDoc) emit_generate_err(err string, errcode int) { + cfg := vd.cfg mut err_msg := err if errcode == 1 { mod_list := get_modules_list(cfg.input_path, []string{}) @@ -916,20 +227,25 @@ fn (cfg DocConfig) emit_generate_err(err string, errcode int) { eprintln(err_msg) } -fn (mut cfg DocConfig) generate_docs_from_file() { - if cfg.output_path.len == 0 { - if cfg.output_type == .unset { - cfg.output_type = .stdout - } else { - cfg.vprintln('No output path has detected. Using input path instead.') - cfg.output_path = cfg.input_path - } - } else if cfg.output_type == .unset { - cfg.vprintln('Output path detected. Identifying output type..') - ext := os.file_ext(cfg.output_path) - cfg.set_output_type_from_str(ext.all_after('.')) +fn (mut vd VDoc) generate_docs_from_file() { + cfg := vd.cfg + mut out := Output{ + path: cfg.output_path + typ: cfg.output_type } - if cfg.include_readme && cfg.output_type !in [.html, .stdout] { + if out.path.len == 0 { + if cfg.output_type == .unset { + out.typ = .stdout + } else { + vd.vprintln('No output path has detected. Using input path instead.') + out.path = cfg.input_path + } + } else if out.typ == .unset { + vd.vprintln('Output path detected. Identifying output type..') + ext := os.file_ext(out.path) + out.typ = set_output_type_from_str(ext.all_after('.')) + } + if cfg.include_readme && out.typ !in [.html, .stdout] { eprintln('vdoc: Including README.md for doc generation is supported on HTML output, or when running directly in the terminal.') exit(1) } @@ -942,20 +258,20 @@ fn (mut cfg DocConfig) generate_docs_from_file() { } manifest_path := os.join_path(dir_path, 'v.mod') if os.exists(manifest_path) { - cfg.vprintln('Reading v.mod info from $manifest_path') + vd.vprintln('Reading v.mod info from $manifest_path') if manifest := vmod.from_file(manifest_path) { - cfg.manifest = manifest + vd.manifest = manifest } } if cfg.include_readme { - readme_contents := cfg.get_readme(dir_path) + readme_contents := vd.get_readme(dir_path) comment := doc.DocComment{ text: readme_contents } - if cfg.output_type == .stdout { + if out.typ == .stdout { println(markdown.to_plain(readme_contents)) - } else if cfg.output_type == .html && cfg.is_multi { - cfg.docs << doc.Doc{ + } else if out.typ == .html && cfg.is_multi { + vd.docs << doc.Doc{ head: doc.DocNode{ name: 'README' comments: [comment] @@ -972,15 +288,15 @@ fn (mut cfg DocConfig) generate_docs_from_file() { is_local_and_single := cfg.is_local && !cfg.is_multi for dirpath in dirs { mut dcs := doc.Doc{} - cfg.vprintln('Generating docs for $dirpath') + vd.vprintln('Generating $out.typ docs for "$dirpath"') if is_local_and_single { dcs = doc.generate_with_pos(dirpath, cfg.local_filename, cfg.local_pos) or { - cfg.emit_generate_err(err, errcode) + vd.emit_generate_err(err, errcode) exit(1) } } else { dcs = doc.generate(dirpath, cfg.pub_only, true) or { - cfg.emit_generate_err(err, errcode) + vd.emit_generate_err(err, errcode) exit(1) } } @@ -989,7 +305,7 @@ fn (mut cfg DocConfig) generate_docs_from_file() { } if !is_local_and_single { if cfg.is_multi || (!cfg.is_multi && cfg.include_readme) { - readme_contents := cfg.get_readme(dirpath) + readme_contents := vd.get_readme(dirpath) comment := doc.DocComment{ text: readme_contents } @@ -1011,21 +327,25 @@ fn (mut cfg DocConfig) generate_docs_from_file() { } } } - cfg.docs << dcs + vd.docs << dcs } + // Important. Let builtin be in the top of the module list + // if we are generating docs for vlib. if cfg.is_vlib { - mut docs := cfg.docs.filter(it.head.name == 'builtin') - docs << cfg.docs.filter(it.head.name != 'builtin') - cfg.docs = docs + mut docs := vd.docs.filter(it.head.name == 'builtin') + docs << vd.docs.filter(it.head.name != 'builtin') + vd.docs = docs } if cfg.serve_http { - cfg.serve_html() + vd.serve_html(out) return } - cfg.vprintln('Rendering docs...') - if cfg.output_path.len == 0 || cfg.output_path == 'stdout' { - cfg.render_static() - outputs := cfg.render() + vd.vprintln('Rendering docs...') + if out.path.len == 0 || out.path == 'stdout' { + if out.typ == .html { + vd.render_static_html(cfg.serve_http, out) + } + outputs := vd.render(out) if outputs.len == 0 { println('No documentation for $dirs') } else { @@ -1033,146 +353,50 @@ fn (mut cfg DocConfig) generate_docs_from_file() { println(outputs[first]) } } else { - if !os.is_dir(cfg.output_path) { - cfg.output_path = os.real_path('.') + if !os.is_dir(out.path) { + out.path = os.real_path('.') } - if !os.exists(cfg.output_path) { - os.mkdir(cfg.output_path) or { panic(err) } + if !os.exists(out.path) { + os.mkdir(out.path) or { panic(err) } } if cfg.is_multi { - cfg.output_path = os.join_path(cfg.output_path, '_docs') - if !os.exists(cfg.output_path) { - os.mkdir(cfg.output_path) or { panic(err) } + out.path = os.join_path(out.path, '_docs') + if !os.exists(out.path) { + os.mkdir(out.path) or { panic(err) } } else { for fname in css_js_assets { - os.rm(os.join_path(cfg.output_path, fname)) + os.rm(os.join_path(out.path, fname)) } } } - cfg.render_static() - cfg.render_parallel() + if out.typ == .html { + vd.render_static_html(cfg.serve_http, out) + } + vd.render_parallel(out) println('Creating search index...') - cfg.collect_search_index() - cfg.render_search_index() - // move favicons to target directory - println('Copying favicons...') - favicons := os.ls(favicons_path) or { panic(err) } - for favicon in favicons { - favicon_path := os.join_path(favicons_path, favicon) - destination_path := os.join_path(cfg.output_path, favicon) - os.cp(favicon_path, destination_path) + if out.typ == .html { + vd.collect_search_index(out) + vd.render_search_index(out) + // move favicons to target directory + println('Copying favicons...') + favicons := os.ls(favicons_path) or { panic(err) } + for favicon in favicons { + favicon_path := os.join_path(favicons_path, favicon) + destination_path := os.join_path(out.path, favicon) + os.cp(favicon_path, destination_path) + } } } } -fn (mut cfg DocConfig) set_output_type_from_str(format string) { - match format { - 'htm', 'html' { cfg.output_type = .html } - 'md', 'markdown' { cfg.output_type = .markdown } - 'json' { cfg.output_type = .json } - 'stdout' { cfg.output_type = .stdout } - else { cfg.output_type = .plaintext } - } - cfg.vprintln('Setting output type to "$cfg.output_type"') -} - -fn (cfg DocConfig) vprintln(str string) { - if cfg.is_verbose { +fn (vd VDoc) vprintln(str string) { + if vd.cfg.is_verbose { println('vdoc: $str') } } -fn get_ignore_paths(path string) ?[]string { - ignore_file_path := os.join_path(path, '.vdocignore') - ignore_content := os.read_file(ignore_file_path) or { - return error_with_code('ignore file not found.', 1) - } - mut res := []string{} - if ignore_content.trim_space().len > 0 { - rules := ignore_content.split_into_lines().map(it.trim_space()) - mut final := []string{} - for rule in rules { - if rule.contains('*.') || rule.contains('**') { - println('vdoc: Wildcards in ignore rules are not allowed for now.') - continue - } - final << rule - } - res = final.map(os.join_path(path, it.trim_right('/'))) - } else { - mut dirs := os.ls(path) or { return []string{} } - res = dirs.map(os.join_path(path, it)).filter(os.is_dir(it)) - } - return res.map(it.replace('/', os.path_separator)) -} - -fn is_included(path string, ignore_paths []string) bool { - if path.len == 0 { - return true - } - for ignore_path in ignore_paths { - if ignore_path !in path { - continue - } - return false - } - return true -} - -fn get_modules_list(path string, ignore_paths2 []string) []string { - files := os.ls(path) or { return []string{} } - mut ignore_paths := get_ignore_paths(path) or { []string{} } - ignore_paths << ignore_paths2 - mut dirs := []string{} - for file in files { - fpath := os.join_path(path, file) - if os.is_dir(fpath) && is_included(fpath, ignore_paths) && !os.is_link(path) { - dirs << get_modules_list(fpath, ignore_paths.filter(it.starts_with(fpath))) - } else if fpath.ends_with('.v') && !fpath.ends_with('_test.v') { - if path in dirs { - continue - } - dirs << path - } - } - dirs.sort() - return dirs -} - -fn (cfg DocConfig) get_resource(name string, minify bool) string { - path := os.join_path(res_path, name) - mut res := os.read_file(path) or { panic('vdoc: could not read $path') } - if minify { - if name.ends_with('.js') { - res = js_compress(res) - } else { - res = res.split_into_lines().map(it.trim_space()).join('') - } - } - // TODO: Make SVG inline for now - if cfg.inline_assets || path.ends_with('.svg') { - return res - } else { - output_path := os.join_path(cfg.output_path, name) - if !os.exists(output_path) { - println('Generating $output_path') - os.write_file(output_path, res) - } - return name - } -} - -fn main() { - if os.args.len < 2 || '-h' in os.args || '--help' in os.args || os.args[1..] == ['doc', 'help'] { - os.system('$vexe help doc') - exit(0) - } - args := os.args[2..].clone() - mut cfg := DocConfig{ - manifest: vmod.Manifest{ - repo_url: '' - } - } +fn parse_arguments(args []string) Config { + mut cfg := Config{} for i := 0; i < args.len; i++ { arg := args[i] current_args := args[i..] @@ -1192,7 +416,7 @@ fn main() { eprintln('vdoc: "$format" is not a valid format. Only $allowed_str are allowed.') exit(1) } - cfg.set_output_type_from_str(format) + cfg.output_type = set_output_type_from_str(format) i++ } '-inline-assets' { @@ -1265,10 +489,7 @@ fn main() { } } } - if cfg.input_path.len == 0 { - eprintln('vdoc: No input path found.') - exit(1) - } + // Correct from configuration from user input if cfg.output_path == 'stdout' && cfg.output_type == .html { cfg.inline_assets = true } @@ -1284,19 +505,34 @@ fn main() { cfg.is_multi = true cfg.input_path = os.join_path(vroot, 'vlib') } else if !is_path { - cfg.vprintln('Input "$cfg.input_path" is not a valid path. Looking for modules named "$cfg.input_path"...') + // TODO vd.vprintln('Input "$cfg.input_path" is not a valid path. Looking for modules named "$cfg.input_path"...') mod_path := doc.lookup_module(cfg.input_path) or { eprintln('vdoc: $err') exit(1) } cfg.input_path = mod_path } - cfg.generate_docs_from_file() + return cfg } -fn is_module_readme(dn doc.DocNode) bool { - if dn.comments.len > 0 && dn.content == 'module $dn.name' { - return true +fn main() { + if os.args.len < 2 || '-h' in os.args || '--help' in os.args || os.args[1..] == ['doc', 'help'] { + os.system('$vexe help doc') + exit(0) } - return false + args := os.args[2..].clone() + cfg := parse_arguments(args) + if cfg.input_path.len == 0 { + eprintln('vdoc: No input path found.') + exit(1) + } + // Config is immutable from this point on + mut vd := VDoc{ + cfg: cfg + manifest: vmod.Manifest{ + repo_url: '' + } + } + vd.vprintln('Setting output type to "$cfg.output_type"') + vd.generate_docs_from_file() } diff --git a/cmd/tools/vtest-cleancode.v b/cmd/tools/vtest-cleancode.v index f7e0f19e9e..d50c9443f9 100644 --- a/cmd/tools/vtest-cleancode.v +++ b/cmd/tools/vtest-cleancode.v @@ -18,8 +18,8 @@ const ( 'nonexistant', ] vfmt_verify_list = [ - 'cmd/tools/vdoc/vdoc.v', 'cmd/v/v.v', + 'cmd/tools/vdoc/', 'vlib/arrays/', 'vlib/benchmark/', 'vlib/bitfield/', diff --git a/cmd/tools/vtest-compiler.v b/cmd/tools/vtest-compiler.v index 1fb10fa2ef..325b1ad474 100644 --- a/cmd/tools/vtest-compiler.v +++ b/cmd/tools/vtest-compiler.v @@ -34,7 +34,7 @@ fn v_test_compiler(vargs string) { eprintln('v.c can be compiled without warnings. This is good :)') } } - building_tools_failed := testing.v_build_failing(vargs, 'cmd/tools') + building_tools_failed := os.system('"$vexe" build-tools') != 0 eprintln('') testing.eheader('Testing all _test.v files...') mut compiler_test_session := testing.new_test_session(vargs)