regex: add split (#14329)
David 'Epper' Marshall
parent
0bd8d872d1
commit
3fd63264d7
GPG Key ID:
B75D4F293C7052DB
|
|
@ -174,6 +174,7 @@ struct TestItemRe {
|
|||
rep string
|
||||
r string
|
||||
}
|
||||
|
||||
const (
|
||||
match_test_suite_replace = [
|
||||
// replace tests
|
||||
|
|
@ -234,6 +235,7 @@ struct TestItemCGroup {
|
|||
cg []int // [number of items (3*# item), id_group_0, start_0, end_0, id_group_1, start1, start2,... ]
|
||||
cgn map[string]int
|
||||
}
|
||||
|
||||
const (
|
||||
cgroups_test_suite = [
|
||||
TestItemCGroup{
|
||||
|
|
@ -276,13 +278,13 @@ cgroups_test_suite = [
|
|||
]
|
||||
)
|
||||
|
||||
|
||||
struct Test_find_all {
|
||||
src string
|
||||
q string
|
||||
res []int // [0,4,5,6...]
|
||||
res_str []string // ['find0','find1'...]
|
||||
}
|
||||
|
||||
const (
|
||||
find_all_test_suite = [
|
||||
Test_find_all{
|
||||
|
|
@ -361,6 +363,42 @@ find_all_test_suite = [
|
|||
]
|
||||
)
|
||||
|
||||
struct Test_split {
|
||||
src string
|
||||
q string
|
||||
res []string // ['abc','def',...]
|
||||
}
|
||||
|
||||
const (
|
||||
split_test_suite = [
|
||||
Test_split{'abcd 1234 efgh 1234 ghkl1234 ab34546df', r'\d+', ['abcd ', ' efgh ', ' ghkl',
|
||||
' ab', 'df']},
|
||||
Test_split{'abcd 1234 efgh 1234 ghkl1234 ab34546df', r'\a+', [' 1234 ', ' 1234 ', '1234 ',
|
||||
'34546']},
|
||||
Test_split{'oggi pippo è andato a casa di pluto ed ha trovato pippo', r'p[iplut]+o', [
|
||||
'oggi ', ' è andato a casa di ', ' ed ha trovato ']},
|
||||
Test_split{'oggi pibao è andato a casa di pbababao ed ha trovato pibabababao', r'(pi?(ba)+o)', [
|
||||
'oggi ', ' è andato a casa di ', ' ed ha trovato ']},
|
||||
Test_split{'Today is a good day and tomorrow will be for sure.', r'[Tt]o\w+', [
|
||||
' is a good day and ', ' will be for sure.']},
|
||||
Test_split{'pera\nurl = https://github.com/dario/pig.html\npippo', r'url *= *https?://[\w./]+', [
|
||||
'pera\n', '\npippo']},
|
||||
Test_split{'pera\nurl = https://github.com/dario/pig.html\npippo', r'url *= *https?://.*' +
|
||||
'\n', ['pera\n', 'pippo']},
|
||||
Test_split{'#.#......##.#..#..##........##....###...##...######.......#.....#..#......#...#........###.#..#.', r'#[.#]{4}##[.#]{4}##[.#]{4}###', [
|
||||
'#.#......##.#..#..##........#', '##.......#.....#..#......#...#........###.#..#.']},
|
||||
Test_split{'#.#......##.#..#..##........##....###...##...######.......#.....#..#......#...#........###.#..#.', r'.*#[.#]{4}##[.#]{4}##[.#]{4}###', [
|
||||
'##.......#.....#..#......#...#........###.#..#.']},
|
||||
Test_split{'1234 Aa dddd Aaf 12334 Aa opopo Aaf', r'Aa.+Aaf', ['1234 ', ' 12334 ']},
|
||||
Test_split{'@for something @endfor @for something else @endfor altro testo @for body @endfor uno due @for senza dire più @endfor pippo', r'@for.+@endfor', [
|
||||
' ', ' altro testo ', ' uno due ', ' pippo']},
|
||||
Test_split{'+++pippo+++\n elvo +++ pippo2 +++ +++ oggi+++', r'\+{3}.*\+{3}', [
|
||||
'\n elvo ', ' ']},
|
||||
Test_split{'foobar', r'\d', ['foobar']},
|
||||
Test_split{'1234', r'\d+', []},
|
||||
]
|
||||
)
|
||||
|
||||
const (
|
||||
debug = true // true for debug println
|
||||
)
|
||||
|
|
@ -370,7 +408,7 @@ fn test_regex(){
|
|||
for c, to in cgroups_test_suite {
|
||||
// debug print
|
||||
if debug {
|
||||
println("$c [${to.src}] [q${to.q}] (${to.s}, ${to.e})")
|
||||
println('$c [$to.src] [q$to.q] ($to.s, $to.e)')
|
||||
}
|
||||
|
||||
mut re := regex.regex_opt(to.q) or {
|
||||
|
|
@ -382,21 +420,25 @@ fn test_regex(){
|
|||
if to.cgn.len > 0 {
|
||||
re.group_csave_flag = true
|
||||
// re.group_csave = [-1].repeat(3*20+1)
|
||||
if debug { println("continuous save")}
|
||||
if debug {
|
||||
println('continuous save')
|
||||
}
|
||||
} else {
|
||||
if debug { println("NO continuous save")}
|
||||
if debug {
|
||||
println('NO continuous save')
|
||||
}
|
||||
}
|
||||
|
||||
start, end := re.match_string(to.src)
|
||||
|
||||
mut tmp_str := ""
|
||||
mut tmp_str := ''
|
||||
if start >= 0 && end > start {
|
||||
tmp_str = to.src[start..end]
|
||||
}
|
||||
|
||||
if start != to.s || end != to.e {
|
||||
println("#$c [$to.src] q[$to.q] res[$tmp_str] base:[${to.s},${to.e}] $start, $end")
|
||||
eprintln("ERROR!")
|
||||
println('#$c [$to.src] q[$to.q] res[$tmp_str] base:[$to.s,$to.e] $start, $end')
|
||||
eprintln('ERROR!')
|
||||
assert false
|
||||
continue
|
||||
}
|
||||
|
|
@ -404,7 +446,7 @@ fn test_regex(){
|
|||
// check cgroups
|
||||
if to.cgn.len > 0 {
|
||||
if re.group_csave.len == 0 || re.group_csave[0] != to.cg[0] {
|
||||
eprintln("Capturing group len error! found: ${re.group_csave[0]} true ground: ${to.cg[0]}")
|
||||
eprintln('Capturing group len error! found: ${re.group_csave[0]} true ground: ${to.cg[0]}')
|
||||
assert false
|
||||
continue
|
||||
}
|
||||
|
|
@ -413,7 +455,7 @@ fn test_regex(){
|
|||
mut ln := re.group_csave[0] * 3
|
||||
for ln > 0 {
|
||||
if re.group_csave[ln] != to.cg[ln] {
|
||||
eprintln("Capturing group failed on $ln item!")
|
||||
eprintln('Capturing group failed on $ln item!')
|
||||
assert false
|
||||
}
|
||||
ln--
|
||||
|
|
@ -422,7 +464,7 @@ fn test_regex(){
|
|||
// check named captured groups
|
||||
for k in to.cgn.keys() {
|
||||
if to.cgn[k] != (re.group_map[k] - 1) { // we have -1 because the map not found is 0, in groups we start from 0 and we store using +1
|
||||
eprintln("Named capturing group error! [$k]")
|
||||
eprintln('Named capturing group error! [$k]')
|
||||
assert false
|
||||
continue
|
||||
}
|
||||
|
|
@ -435,8 +477,8 @@ fn test_regex(){
|
|||
for ln := 0; ln < re.groups.len; ln++ {
|
||||
if re.groups[ln] != to.cg[ln] {
|
||||
eprintln("Capture group doesn't match:")
|
||||
eprintln("true ground: ${to.cg}")
|
||||
eprintln("elaborated : ${re.groups}")
|
||||
eprintln('true ground: $to.cg')
|
||||
eprintln('elaborated : $re.groups')
|
||||
assert false
|
||||
}
|
||||
}
|
||||
|
|
@ -446,7 +488,9 @@ fn test_regex(){
|
|||
// check find_all
|
||||
for c, to in find_all_test_suite {
|
||||
// debug print
|
||||
if debug { println("#$c [$to.src] q[$to.q] ($to.res, $to.res_str)") }
|
||||
if debug {
|
||||
println('#$c [$to.src] q[$to.q] ($to.res, $to.res_str)')
|
||||
}
|
||||
|
||||
mut re := regex.regex_opt(to.q) or {
|
||||
eprintln('err: $err')
|
||||
|
|
@ -458,14 +502,42 @@ fn test_regex(){
|
|||
res := re.find_all(to.src)
|
||||
if res != to.res {
|
||||
eprintln('err: find_all !!')
|
||||
if debug { println("#$c exp: $to.res calculated: $res") }
|
||||
if debug {
|
||||
println('#$c exp: $to.res calculated: $res')
|
||||
}
|
||||
assert false
|
||||
}
|
||||
|
||||
res_str := re.find_all_str(to.src)
|
||||
if res_str != to.res_str {
|
||||
eprintln('err: find_all_str !!')
|
||||
if debug { println("#$c exp: $to.res_str calculated: $res_str") }
|
||||
if debug {
|
||||
println('#$c exp: $to.res_str calculated: $res_str')
|
||||
}
|
||||
assert false
|
||||
}
|
||||
}
|
||||
|
||||
// check split
|
||||
for c, to in split_test_suite {
|
||||
// debug print
|
||||
if debug {
|
||||
println('#$c [$to.src] q[$to.q] ($to.res)')
|
||||
}
|
||||
|
||||
mut re := regex.regex_opt(to.q) or {
|
||||
eprintln('err: $err')
|
||||
assert false
|
||||
continue
|
||||
}
|
||||
|
||||
re.reset()
|
||||
res := re.split(to.src)
|
||||
if res != to.res {
|
||||
eprintln('err: split !!')
|
||||
if debug {
|
||||
println('#$c exp: $to.res calculated: $res')
|
||||
}
|
||||
assert false
|
||||
}
|
||||
}
|
||||
|
|
@ -473,7 +545,9 @@ fn test_regex(){
|
|||
// check replace
|
||||
for c, to in match_test_suite_replace {
|
||||
// debug print
|
||||
if debug { println("#$c [$to.src] q[$to.q] $to.r") }
|
||||
if debug {
|
||||
println('#$c [$to.src] q[$to.q] $to.r')
|
||||
}
|
||||
|
||||
mut re := regex.regex_opt(to.q) or {
|
||||
eprintln('err: $err')
|
||||
|
|
@ -483,7 +557,7 @@ fn test_regex(){
|
|||
|
||||
res := re.replace(to.src, to.rep)
|
||||
if res != to.r {
|
||||
eprintln("ERROR: replace.")
|
||||
eprintln('ERROR: replace.')
|
||||
assert false
|
||||
continue
|
||||
}
|
||||
|
|
@ -492,7 +566,7 @@ fn test_regex(){
|
|||
// check replace simple
|
||||
for c, to in match_test_suite_replace_simple {
|
||||
// debug print
|
||||
if debug { println("#$c [$to.src] q[$to.q] $to.r") }
|
||||
if debug { println('#$c [$to.src] q[$to.q] $to.r') }
|
||||
|
||||
mut re := regex.regex_opt(to.q) or {
|
||||
eprintln('err: $err')
|
||||
|
|
@ -502,7 +576,7 @@ fn test_regex(){
|
|||
|
||||
res := re.replace_simple(to.src, to.rep)
|
||||
if res != to.r {
|
||||
eprintln("ERROR: replace.")
|
||||
eprintln('ERROR: replace.')
|
||||
assert false
|
||||
continue
|
||||
}
|
||||
|
|
@ -511,7 +585,7 @@ fn test_regex(){
|
|||
// check match and find
|
||||
for c, to in match_test_suite {
|
||||
// debug print
|
||||
if debug { println("#$c [$to.src] q[$to.q] $to.s $to.e") }
|
||||
if debug { println('#$c [$to.src] q[$to.q] $to.s $to.e') }
|
||||
|
||||
// test the find
|
||||
if to.s > 0 {
|
||||
|
|
@ -526,7 +600,7 @@ fn test_regex(){
|
|||
|
||||
if start != to.s || end != to.e {
|
||||
err_str := re.get_parse_error_string(start)
|
||||
eprintln("ERROR : $err_str start: ${start} end: ${end}")
|
||||
eprintln('ERROR : $err_str start: $start end: $end')
|
||||
assert false
|
||||
} else {
|
||||
// tmp_str := text[start..end]
|
||||
|
|
@ -548,14 +622,14 @@ fn test_regex(){
|
|||
// println("#$c [$to.src] q[$to.q]")
|
||||
start, end := re.match_string(to.src)
|
||||
|
||||
mut tmp_str := ""
|
||||
mut tmp_str := ''
|
||||
if start >= 0 && end > start {
|
||||
tmp_str = to.src[start..end]
|
||||
}
|
||||
|
||||
if start != to.s || end != to.e {
|
||||
eprintln("#$c [$to.src] q[$to.q] res[$tmp_str] $start, $end")
|
||||
eprintln("ERROR!")
|
||||
eprintln('#$c [$to.src] q[$to.q] res[$tmp_str] $start, $end')
|
||||
eprintln('ERROR!')
|
||||
assert false
|
||||
continue
|
||||
}
|
||||
|
|
@ -571,51 +645,50 @@ fn test_regex(){
|
|||
tmp_str1 := to.src.clone()
|
||||
start1, end1 := re.match_string(tmp_str1)
|
||||
if start1 != start || end1 != end {
|
||||
eprintln("two run ERROR!!")
|
||||
eprintln('two run ERROR!!')
|
||||
assert false
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if debug { println("DONE!") }
|
||||
if debug { println('DONE!') }
|
||||
}
|
||||
|
||||
// test regex_base function
|
||||
fn test_regex_func() {
|
||||
query := r"\d\dabcd"
|
||||
test_str := "78abcd"
|
||||
query := r'\d\dabcd'
|
||||
test_str := '78abcd'
|
||||
mut re, re_err, err_pos := regex.regex_base(query)
|
||||
if re_err == regex.compile_ok {
|
||||
start, end := re.match_string(test_str)
|
||||
assert (start == 0) && (end == 6)
|
||||
} else {
|
||||
eprintln("Error in query string in pos ${err_pos}")
|
||||
eprintln("Error: ${re.get_parse_error_string(re_err)}")
|
||||
eprintln('Error in query string in pos $err_pos')
|
||||
eprintln('Error: ${re.get_parse_error_string(re_err)}')
|
||||
assert false
|
||||
}
|
||||
}
|
||||
|
||||
fn my_repl_1(re regex.RE, in_txt string, start int, end int) string {
|
||||
s0 := re.get_group_by_id(in_txt, 0)
|
||||
println("[$start, $end] => ${s0}")
|
||||
return "a" + s0.to_upper()
|
||||
println('[$start, $end] => $s0')
|
||||
return 'a' + s0.to_upper()
|
||||
}
|
||||
|
||||
fn test_regex_func_replace1() {
|
||||
txt := "abbabbbabbbbaabba"
|
||||
query := r"a(b+)"
|
||||
txt := 'abbabbbabbbbaabba'
|
||||
query := r'a(b+)'
|
||||
mut re := regex.regex_opt(query) or { panic(err) }
|
||||
result := re.replace_by_fn(txt, my_repl_1)
|
||||
|
||||
assert result == "aBBaBBBaBBBBaaBBa"
|
||||
assert result == 'aBBaBBBaBBBBaaBBa'
|
||||
}
|
||||
|
||||
fn my_repl(re regex.RE, in_txt string, start int, end int) string {
|
||||
s0 := re.get_group_by_id(in_txt,0)[0..1] + "X"
|
||||
s1 := re.get_group_by_id(in_txt,1)[0..1] + "X"
|
||||
s2 := re.get_group_by_id(in_txt,2)[0..1] + "X"
|
||||
return "${s0}${s1}${s2}"
|
||||
s0 := re.get_group_by_id(in_txt, 0)[0..1] + 'X'
|
||||
s1 := re.get_group_by_id(in_txt, 1)[0..1] + 'X'
|
||||
s2 := re.get_group_by_id(in_txt, 2)[0..1] + 'X'
|
||||
return '$s0$s1$s2'
|
||||
}
|
||||
|
||||
// test regex replace function
|
||||
|
|
@ -625,13 +698,13 @@ fn test_regex_func_replace(){
|
|||
query := r'"(content":\s+")(.*)(, "channel_id")'
|
||||
mut re := regex.regex_opt(query) or { panic(err) }
|
||||
|
||||
mut txt1 := ""
|
||||
mut txt2 := ""
|
||||
mut txt1 := ''
|
||||
mut txt2 := ''
|
||||
|
||||
for _ in 0 .. 3 {
|
||||
rnd := int(10 + rand.u32() % 20)
|
||||
txt1 += txt + filler[0..rnd] + "\n"
|
||||
txt2 += "cXTX,X" + filler[0..rnd] + "\n"
|
||||
txt1 += txt + filler[0..rnd] + '\n'
|
||||
txt2 += 'cXTX,X' + filler[0..rnd] + '\n'
|
||||
}
|
||||
|
||||
result := re.replace_by_fn(txt1, my_repl)
|
||||
|
|
@ -643,18 +716,18 @@ fn test_regex_func_replace(){
|
|||
}
|
||||
|
||||
fn rest_regex_replace_n() {
|
||||
s := "dario 1234 pepep 23454 pera"
|
||||
query := r"\d+"
|
||||
s := 'dario 1234 pepep 23454 pera'
|
||||
query := r'\d+'
|
||||
|
||||
mut re := regex.regex_opt(query) or { panic(err) }
|
||||
|
||||
assert re.replace_n(s, "[repl]", 0) == "dario 1234 pepep 23454 pera"
|
||||
assert re.replace_n(s, "[repl]", -1) == "dario 1234 pepep [repl] pera"
|
||||
assert re.replace_n(s, "[repl]", 1) == "dario [repl] pepep 23454 pera"
|
||||
assert re.replace_n(s, "[repl]", 2) == "dario [repl] pepep [repl] pera"
|
||||
assert re.replace_n(s, "[repl]", -2) == "dario [repl] pepep [repl] pera"
|
||||
assert re.replace_n(s, "[repl]", 3) == "dario [repl] pepep [repl] pera"
|
||||
assert re.replace_n(s, "[repl]", -3) == "dario [repl] pepep [repl] pera"
|
||||
assert re.replace_n(s, '[repl]', 0) == 'dario 1234 pepep 23454 pera'
|
||||
assert re.replace_n(s, '[repl]', -1) == 'dario 1234 pepep [repl] pera'
|
||||
assert re.replace_n(s, '[repl]', 1) == 'dario [repl] pepep 23454 pera'
|
||||
assert re.replace_n(s, '[repl]', 2) == 'dario [repl] pepep [repl] pera'
|
||||
assert re.replace_n(s, '[repl]', -2) == 'dario [repl] pepep [repl] pera'
|
||||
assert re.replace_n(s, '[repl]', 3) == 'dario [repl] pepep [repl] pera'
|
||||
assert re.replace_n(s, '[repl]', -3) == 'dario [repl] pepep [repl] pera'
|
||||
|
||||
// mut res := re.replace_n(s, "[repl]", -1)
|
||||
// println("source: ${s}")
|
||||
|
|
@ -670,14 +743,15 @@ const(
|
|||
r'+{3}.*\+{3}*',
|
||||
r'+{3}.*\+{3}+',
|
||||
r'+{3}.*\+{3}??',
|
||||
r'+{3}.*\+{3}{4}'
|
||||
r'+{3}.*\+{3}{4}',
|
||||
]
|
||||
)
|
||||
|
||||
fn test_quantifier_sequences() {
|
||||
for pattern in test_quantifier_sequences_list {
|
||||
re, re_err, err_pos := regex.regex_base(pattern)
|
||||
if re_err != regex.err_syntax_error {
|
||||
eprintln("pattern: $pattern => $re_err")
|
||||
eprintln('pattern: $pattern => $re_err')
|
||||
}
|
||||
assert re_err == regex.err_syntax_error
|
||||
}
|
||||
|
|
@ -691,6 +765,7 @@ struct Test_find_groups {
|
|||
e int // end index
|
||||
res []int // groups indexes
|
||||
}
|
||||
|
||||
const (
|
||||
find_groups_test_suite = [
|
||||
Test_find_groups{
|
||||
|
|
@ -716,6 +791,7 @@ find_groups_test_suite = [
|
|||
},
|
||||
]
|
||||
)
|
||||
|
||||
fn test_groups_in_find() {
|
||||
for test_obj in find_groups_test_suite {
|
||||
src_text := test_obj.src
|
||||
|
|
@ -746,9 +822,10 @@ fn test_groups_in_find(){
|
|||
|
||||
const (
|
||||
err_query_list = [
|
||||
r'([a]|[b])*'
|
||||
r'([a]|[b])*',
|
||||
]
|
||||
)
|
||||
|
||||
fn test_errors() {
|
||||
mut count := 0
|
||||
for query in err_query_list {
|
||||
|
|
@ -760,7 +837,6 @@ fn test_errors(){
|
|||
assert count == err_query_list.len
|
||||
}
|
||||
|
||||
|
||||
fn test_long_query() {
|
||||
test_len := 32768
|
||||
mut buf := strings.new_builder(test_len * 3)
|
||||
|
|
@ -799,11 +875,11 @@ fn test_long_query() {
|
|||
assert start >= 0 && end == base_string.len
|
||||
}
|
||||
|
||||
|
||||
struct Test_negation_group {
|
||||
src string
|
||||
res bool
|
||||
}
|
||||
|
||||
const (
|
||||
negation_groups = [
|
||||
Test_negation_group{'automobile', false},
|
||||
|
|
@ -814,8 +890,9 @@ const(
|
|||
Test_negation_group{'boring test', false},
|
||||
]
|
||||
)
|
||||
|
||||
fn test_negation_groups() {
|
||||
mut query := r"(?!auto)\w+le"
|
||||
mut query := r'(?!auto)\w+le'
|
||||
mut re := regex.regex_opt(query) or { panic(err) }
|
||||
for test in negation_groups {
|
||||
start, end := re.match_string(test.src)
|
||||
|
|
|
|||
|
|
@ -293,6 +293,38 @@ pub fn (mut re RE) find_all(in_txt string) []int {
|
|||
return res
|
||||
}
|
||||
|
||||
// split returns the sections of string around the regex
|
||||
//
|
||||
// Usage:
|
||||
// ```v
|
||||
// blurb := 'foobar boo steelbar toolbox foot tooooot'
|
||||
// mut re := regex.regex_opt('f|t[eo]+')?
|
||||
// res := re.split(blurb) // ['bar boo s', 'lbar ', 'lbox ', 't ', 't']
|
||||
// ```
|
||||
pub fn (mut re RE) split(in_txt string) []string {
|
||||
pos := re.find_all(in_txt)
|
||||
|
||||
mut sections := []string{cap: pos.len / 2 + 1}
|
||||
|
||||
if pos.len == 0 {
|
||||
return [in_txt]
|
||||
}
|
||||
for i := 0; i < pos.len; i += 2 {
|
||||
if pos[i] == 0 {
|
||||
continue
|
||||
}
|
||||
if i == 0 {
|
||||
sections << in_txt[..pos[i]]
|
||||
} else {
|
||||
sections << in_txt[pos[i - 1]..pos[i]]
|
||||
}
|
||||
}
|
||||
if pos[pos.len - 1] != in_txt.len {
|
||||
sections << in_txt[pos[pos.len - 1]..]
|
||||
}
|
||||
return sections
|
||||
}
|
||||
|
||||
// find_all_str find all the non overlapping occurrences of the match pattern, return a string list
|
||||
[direct_array_access]
|
||||
pub fn (mut re RE) find_all_str(in_txt string) []string {
|
||||
|
|
|
|||
Loading…
Reference in New Issue