doc: detailed documentation of the compiler pipeline (#7043)

pull/7059/head
Daniel Däschle 2020-12-01 03:58:39 +01:00 committed by GitHub
parent 83137cbe84
commit 6af082e70e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 132 additions and 5 deletions

109
vlib/v/README.md 100644
View File

@ -0,0 +1,109 @@
# Compiler pipeline
A simple high level explanation
how the compiler pipeline (`parser` -> `checker` -> `generator`) works.
## Reading files
### Getting builtin files
To load all builtin files,
a preference `Preferences.lookup_path` for the path where to look for exists.
See `Builder.get_builtin_files` as example.
If the file is a `.vsh` file and the backend is C, `vlib/os` will also be loaded as builtin.
### Getting project files
Either there is a specific file: `my_file.v` or a directory containing V files.
In the last case it scans that directory for all files.
See `Builder.v_files_from_dir` as the helper method.
This list of files needs to be filtered so that only `*.v` files exist.
Skips the following file types:
- `*_test.v`
- either `*.c.v` or `*.c.js` depending on the backend
- all files that doesn't end with `.v`
- Files that are not defined in `Preferences.compile_defines`
or `Preferences.compile_defines_all` **if any file is defined**.
## Parsing files
To parse something a new template is created as the first step:
```v
import v.table
table := table.new_table()
```
a new preference is created:
```v
import v.pref
pref := pref.Preferences{}
```
and a new scope is created:
```v
import v.ast
scope := ast.Scope{
parent: 0
}
```
after that, you can parse your files.
## Parse text
If you want to parse only text which isn't saved on the disk you can use this function.
```v oksyntax
import v.parser
code := ''
// table, pref and scope needs to be passed as reference
parsed_file := parser.parse_text(code, table, .parse_comments, &pref, &scope)
```
## Parse a single file
For parsing files on disk, a path needs to be provided.
The paths are collected one step earlier.
```v oksyntax
import v.parser
path := ''
// table, pref and scope needs to be passed as reference
parsed_file := parser.parse_file(path, table, .parse_comments, &pref, &scope)
```
## Parse a set of files
If you have a batch of paths available which should be parsed,
there is also a function which does all the work.
```v oksyntax
import v.parser
paths := ['']
// table, pref and scope needs to be passed as reference
parsed_files := parser.parse_files(paths, table, &pref, &scope)
```
## Parse imports
A file often contains imports. These imports might need to be parsed as well.
The builder contains a method which does this: `Builder.parse_imports`.
If the module which is imported isn't parsed already,
you have to collect it relatively from the main file.
For this the `ast.File` contains a list of imports.
Those imports needs to be found on disk.
`.` is just replaced with seperators in the relative location of the main file.
Then all files from that directory are collected and parsed again like the previous steps explained.
## Checking AST
A new checker is created:
```v oksyntax
import v.checker
mut checker := checker.new_checker(table, &pref)
```
After checking your files in `checker.errors` and `checker.warnings` you can see the results.
### Check `ast.File`
```v oksyntax
checker.check(parsed_file)
```
### Check a list of `ast.File`
```v oksyntax
checker.check_files(parsed_files)
```
## Generate target from AST
Generating C code works just as this:
```v oksyntax
import v.gen
res := gen.cgen(parsed_files, table, &pref)
```

View File

@ -113,7 +113,7 @@ fn (mut p Parser) vweb() ast.ComptimeCall {
println('>>> end of vweb template END') println('>>> end of vweb template END')
println('\n\n') println('\n\n')
} }
mut file := parse_text(v_code, p.table, p.pref, scope, p.global_scope) mut file := parse_comptime(v_code, p.table, p.pref, scope, p.global_scope)
file = { file = {
file | file |
path: tmpl_path path: tmpl_path

View File

@ -81,11 +81,11 @@ pub fn parse_stmt(text string, table &table.Table, scope &ast.Scope) ast.Stmt {
return p.stmt(false) return p.stmt(false)
} }
pub fn parse_text(text string, b_table &table.Table, pref &pref.Preferences, scope &ast.Scope, global_scope &ast.Scope) ast.File { pub fn parse_comptime(text string, table &table.Table, pref &pref.Preferences, scope &ast.Scope, global_scope &ast.Scope) ast.File {
s := scanner.new_scanner(text, .skip_comments, pref) s := scanner.new_scanner(text, .skip_comments, pref)
mut p := Parser{ mut p := Parser{
scanner: s scanner: s
table: b_table table: table
pref: pref pref: pref
scope: scope scope: scope
errors: []errors.Error{} errors: []errors.Error{}
@ -95,7 +95,25 @@ pub fn parse_text(text string, b_table &table.Table, pref &pref.Preferences, sco
return p.parse() return p.parse()
} }
pub fn parse_file(path string, b_table &table.Table, comments_mode scanner.CommentsMode, pref &pref.Preferences, global_scope &ast.Scope) ast.File { pub fn parse_text(text string, table &table.Table, comments_mode scanner.CommentsMode, pref &pref.Preferences, global_scope &ast.Scope) ast.File {
s := scanner.new_scanner(text, comments_mode, pref)
mut p := Parser{
scanner: s
comments_mode: comments_mode
table: table
pref: pref
scope: &ast.Scope{
start_pos: 0
parent: global_scope
}
errors: []errors.Error{}
warnings: []errors.Warning{}
global_scope: global_scope
}
return p.parse()
}
pub fn parse_file(path string, table &table.Table, comments_mode scanner.CommentsMode, pref &pref.Preferences, global_scope &ast.Scope) ast.File {
// NB: when comments_mode == .toplevel_comments, // NB: when comments_mode == .toplevel_comments,
// the parser gives feedback to the scanner about toplevel statements, so that the scanner can skip // the parser gives feedback to the scanner about toplevel statements, so that the scanner can skip
// all the tricky inner comments. This is needed because we do not have a good general solution // all the tricky inner comments. This is needed because we do not have a good general solution
@ -107,7 +125,7 @@ pub fn parse_file(path string, b_table &table.Table, comments_mode scanner.Comme
mut p := Parser{ mut p := Parser{
scanner: scanner.new_scanner_file(path, comments_mode, pref) scanner: scanner.new_scanner_file(path, comments_mode, pref)
comments_mode: comments_mode comments_mode: comments_mode
table: b_table table: table
file_name: path file_name: path
file_base: os.base(path) file_base: os.base(path)
file_name_dir: os.dir(path) file_name_dir: os.dir(path)