vieter-v
/
v
2
0
Fork 0
Code Issues Pull Requests Releases Activity

depgraph/map: fix memory bug

pull/4254/head
ka-weihe 2020-04-05 22:09:52 +02:00 committed by GitHub
parent ffe8452f02
commit 307daacf67
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 47 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
vlib/builtin/map.v
View File

@ -58,7 +58,7 @@ const (
// Number of bits from the hash stored for each entry // Number of bits from the hash stored for each entry
hashbits = 24 hashbits = 24
// Number of bits from the hash stored for rehashing // Number of bits from the hash stored for rehashing
cached_hashbits = 16 max_cached_hashbits = 16
// Initial log-number of buckets in the hashtable // Initial log-number of buckets in the hashtable
init_log_capicity = 5 init_log_capicity = 5
// Initial number of buckets in the hashtable // Initial number of buckets in the hashtable
@ -139,10 +139,10 @@ pub struct map {
// Byte size of value // Byte size of value
value_bytes int value_bytes int
mut: mut:
// Index of the highest index in the hashtable // highest even index in the hashtable
cap u32 cap u32
// Number of cached hashbits left for rehasing // Number of cached hashbits left for rehasing
window byte cached_hashbits byte
// Used for right-shifting out used hashbits // Used for right-shifting out used hashbits
shift byte shift byte
// Array storing key-values (ordered) // Array storing key-values (ordered)
@ -163,7 +163,7 @@ fn new_map(n, value_bytes int) map {
return map{ return map{
value_bytes: value_bytes value_bytes: value_bytes
cap: init_cap cap: init_cap
window: cached_hashbits cached_hashbits: max_cached_hashbits
shift: init_log_capicity shift: init_log_capicity
key_values: new_dense_array() key_values: new_dense_array()
metas: &u32(vcalloc(sizeof(u32) * (init_capicity + extra_metas_inc))) metas: &u32(vcalloc(sizeof(u32) * (init_capicity + extra_metas_inc)))
@ -182,17 +182,17 @@ fn new_map_init(n, value_bytes int, keys &string, values voidptr) map {
[inline] [inline]
fn (m map) key_to_index(key string) (u32,u32) { fn (m map) key_to_index(key string) (u32,u32) {
hash := u32(wyhash.wyhash_c(key.str, u64(key.len), 0)) hash := wyhash.wyhash_c(key.str, u64(key.len), 0)
index := hash & m.cap index := hash & m.cap
meta := ((hash>>m.shift) & hash_mask) | probe_inc meta := ((hash>>m.shift) & hash_mask) | probe_inc
return index,meta return u32(index),u32(meta)
} }
[inline] [inline]
fn meta_less(metas &u32, i u32, m u32) (u32,u32) { fn (m map) meta_less(_index u32, _metas u32) (u32,u32) {
mut index := i mut index := _index
mut meta := m mut meta := _metas
for meta < metas[index] { for meta < m.metas[index] {
index += 2 index += 2
meta += probe_inc meta += probe_inc
} }
@ -200,37 +200,35 @@ fn meta_less(metas &u32, i u32, m u32) (u32,u32) {
} }
[inline] [inline]
fn (m mut map) meta_greater(ms &u32, i u32, me u32, kvi u32) &u32 { fn (m mut map) meta_greater(_index u32, _metas u32, kvi u32) {
mut metas := ms mut meta := _metas
mut meta := me mut index := _index
mut index := i
mut kv_index := kvi mut kv_index := kvi
for metas[index] != 0 { for m.metas[index] != 0 {
if meta > metas[index] { if meta > m.metas[index] {
tmp_meta := metas[index] tmp_meta := m.metas[index]
metas[index] = meta m.metas[index] = meta
meta = tmp_meta meta = tmp_meta
tmp_index := metas[index + 1] tmp_index := m.metas[index + 1]
metas[index + 1] = kv_index m.metas[index + 1] = kv_index
kv_index = tmp_index kv_index = tmp_index
} }
index += 2 index += 2
meta += probe_inc meta += probe_inc
} }
metas[index] = meta m.metas[index] = meta
metas[index + 1] = kv_index m.metas[index + 1] = kv_index
probe_count := (meta>>hashbits) - 1 probe_count := (meta>>hashbits) - 1
if (probe_count<<1) == m.extra_metas { if (probe_count<<1) == m.extra_metas {
m.extra_metas += extra_metas_inc m.extra_metas += extra_metas_inc
mem_size := (m.cap + 2 + m.extra_metas) mem_size := (m.cap + 2 + m.extra_metas)
metas = &u32(C.realloc(metas, sizeof(u32) * mem_size)) m.metas = &u32(C.realloc(m.metas, sizeof(u32) * mem_size))
C.memset(metas + mem_size - extra_metas_inc, 0, sizeof(u32) * extra_metas_inc) C.memset(m.metas + mem_size - extra_metas_inc, 0, sizeof(u32) * extra_metas_inc)
// Should almost never happen // Should almost never happen
if probe_count == 252 { if probe_count == 252 {
panic('Probe overflow') panic('Probe overflow')
} }
} }
return metas
} }
fn (m mut map) set(key string, value voidptr) { fn (m mut map) set(key string, value voidptr) {
@ -239,7 +237,7 @@ fn (m mut map) set(key string, value voidptr) {
m.expand() m.expand()
} }
mut index,mut meta := m.key_to_index(key) mut index,mut meta := m.key_to_index(key)
index,meta = meta_less(m.metas, index, meta) index,meta = m.meta_less(index, meta)
// While we might have a match // While we might have a match
for meta == m.metas[index] { for meta == m.metas[index] {
kv_index := m.metas[index + 1] kv_index := m.metas[index + 1]
@ -257,7 +255,7 @@ fn (m mut map) set(key string, value voidptr) {
} }
C.memcpy(kv.value, value, m.value_bytes) C.memcpy(kv.value, value, m.value_bytes)
kv_index := m.key_values.push(kv) kv_index := m.key_values.push(kv)
m.metas = m.meta_greater(m.metas, index, meta, kv_index) m.meta_greater(index, meta, kv_index)
m.size++ m.size++
} }
@ -266,15 +264,15 @@ fn (m mut map) expand() {
old_cap := m.cap old_cap := m.cap
m.cap = ((m.cap + 2)<<1) - 2 m.cap = ((m.cap + 2)<<1) - 2
// Check if any hashbits are left // Check if any hashbits are left
if m.window == 0 { if m.cached_hashbits == 0 {
m.shift += cached_hashbits m.shift += max_cached_hashbits
m.window = cached_hashbits m.cached_hashbits = max_cached_hashbits
m.rehash() m.rehash()
} }
else { else {
m.cached_rehash(old_cap) m.cached_rehash(old_cap)
} }
m.window-- m.cached_hashbits--
} }
fn (m mut map) rehash() { fn (m mut map) rehash() {
@ -287,36 +285,36 @@ fn (m mut map) rehash() {
} }
kv := m.key_values.data[i] kv := m.key_values.data[i]
mut index,mut meta := m.key_to_index(kv.key) mut index,mut meta := m.key_to_index(kv.key)
index,meta = meta_less(m.metas, index, meta) index,meta = m.meta_less(index, meta)
m.metas = m.meta_greater(m.metas, index, meta, i) m.meta_greater(index, meta, i)
} }
} }
fn (m mut map) cached_rehash(old_cap u32) { fn (m mut map) cached_rehash(old_cap u32) {
mut new_meta := &u32(vcalloc(sizeof(u32) * (m.cap + 2 + m.extra_metas))) old_metas := m.metas
m.metas = &u32(vcalloc(sizeof(u32) * (m.cap + 2 + m.extra_metas)))
old_extra_metas := m.extra_metas old_extra_metas := m.extra_metas
for i := u32(0); i <= old_cap + old_extra_metas; i += 2 { for i := u32(0); i <= old_cap + old_extra_metas; i += 2 {
if m.metas[i] == 0 { if old_metas[i] == 0 {
continue continue
} }
old_meta := m.metas[i] old_meta := old_metas[i]
old_probe_count := ((old_meta>>hashbits) - 1)<<1 old_probe_count := ((old_meta>>hashbits) - 1)<<1
old_index := (i - old_probe_count) & (m.cap>>1) old_index := (i - old_probe_count) & (m.cap>>1)
mut index := (old_index | (old_meta<<m.shift)) & m.cap mut index := (old_index | (old_meta<<m.shift)) & m.cap
mut meta := (old_meta & hash_mask) | probe_inc mut meta := (old_meta & hash_mask) | probe_inc
index,meta = meta_less(new_meta, index, meta) index,meta = m.meta_less(index, meta)
kv_index := m.metas[i + 1] kv_index := old_metas[i + 1]
new_meta = m.meta_greater(new_meta, index, meta, kv_index) m.meta_greater(index, meta, kv_index)
} }
unsafe{ unsafe{
free(m.metas) free(old_metas)
} }
m.metas = new_meta
} }
fn (m map) get3(key string, zero voidptr) voidptr { fn (m map) get3(key string, zero voidptr) voidptr {
mut index,mut meta := m.key_to_index(key) mut index,mut meta := m.key_to_index(key)
index,meta = meta_less(m.metas, index, meta) index,meta = m.meta_less(index, meta)
for meta == m.metas[index] { for meta == m.metas[index] {
kv_index := m.metas[index + 1] kv_index := m.metas[index + 1]
if key == m.key_values.data[kv_index].key { if key == m.key_values.data[kv_index].key {
@ -328,7 +326,6 @@ fn (m map) get3(key string, zero voidptr) voidptr {
meta += probe_inc meta += probe_inc
} }
return zero return zero
//return voidptr(0)
} }
fn (m map) exists(key string) bool { fn (m map) exists(key string) bool {
@ -336,7 +333,7 @@ fn (m map) exists(key string) bool {
return false return false
} }
mut index,mut meta := m.key_to_index(key) mut index,mut meta := m.key_to_index(key)
index,meta = meta_less(m.metas, index, meta) index,meta = m.meta_less(index, meta)
for meta == m.metas[index] { for meta == m.metas[index] {
kv_index := m.metas[index + 1] kv_index := m.metas[index + 1]
if key == m.key_values.data[kv_index].key { if key == m.key_values.data[kv_index].key {
@ -350,7 +347,7 @@ fn (m map) exists(key string) bool {
pub fn (m mut map) delete(key string) { pub fn (m mut map) delete(key string) {
mut index,mut meta := m.key_to_index(key) mut index,mut meta := m.key_to_index(key)
index,meta = meta_less(m.metas, index, meta) index,meta = m.meta_less(index, meta)
// Perform backwards shifting // Perform backwards shifting
for meta == m.metas[index] { for meta == m.metas[index] {
kv_index := m.metas[index + 1] kv_index := m.metas[index + 1]
@ -367,9 +364,11 @@ pub fn (m mut map) delete(key string) {
if m.key_values.size <= 32 { if m.key_values.size <= 32 {
return return
} }
if (f32(m.key_values.size) / f32(m.key_values.deletes)) < 1 { // Clean up key_values if too many have been deleted
if m.key_values.deletes >= (m.key_values.size >> 1) {
m.key_values.zeros_to_end() m.key_values.zeros_to_end()
m.rehash() m.rehash()
m.key_values.deletes = 0
} }
return return
} }

3
vlib/v/depgraph/depgraph.v
View File

@ -86,10 +86,11 @@ pub fn (graph mut DepGraph) add(mod string, deps []string) {
pub fn (graph &DepGraph) resolve() &DepGraph { pub fn (graph &DepGraph) resolve() &DepGraph {
mut node_names := OrderedDepMap{} mut node_names := OrderedDepMap{}
mut node_deps := OrderedDepMap{}
for node in graph.nodes { for node in graph.nodes {
node_names.add(node.name, node.deps) node_names.add(node.name, node.deps)
node_deps.add(node.name, node.deps)
} }
mut node_deps := node_names
mut resolved := new_dep_graph() mut resolved := new_dep_graph()
for node_deps.size() != 0 { for node_deps.size() != 0 {
mut ready_set := []string mut ready_set := []string