slab allocator: micro-optimize and refactor; remove code (major)

Signed-off-by: Alex Aizman <[email protected]>

ext/dsort/shard/recm.go

@@ -362,7 +362,6 @@ func (recm *RecordManager) Cleanup() {
 
 	// NOTE: may call oom.FreeToOS
 	core.T.PageMM().FreeSpec(memsys.FreeSpec{
-		Totally: true,
 		ToOS:    true,
 		MinSize: 1, // force toGC to free all (even small) memory to system
 	})

memsys/a_test.go

@@ -16,7 +16,6 @@ package memsys_test
 
 import (
 	"flag"
-	"fmt"
 	"os"
 	"sync"
 	"testing"
@@ -74,7 +73,7 @@ func Test_Sleep(*testing.T) {
 	go printMaxRingLen(mem, c)
 	for range 7 {
 		time.Sleep(duration / 8)
-		mem.FreeSpec(memsys.FreeSpec{IdleDuration: 1, MinSize: cos.MiB})
+		mem.FreeSpec(memsys.FreeSpec{MinSize: cos.MiB})
 	}
 	wg.Wait()
 	close(c)
@@ -88,7 +87,6 @@ func Test_NoSleep(*testing.T) {
 
 	mem := &memsys.MMSA{Name: "bmem", TimeIval: time.Second * 20, MinPctTotal: 5}
 	mem.Init(0)
-	go printStats(mem)
 
 	wg := &sync.WaitGroup{}
 	random := cos.NowRand()
@@ -102,7 +100,7 @@ func Test_NoSleep(*testing.T) {
 	go printMaxRingLen(mem, c)
 	for range 7 {
 		time.Sleep(duration / 8)
-		mem.FreeSpec(memsys.FreeSpec{Totally: true, ToOS: true, MinSize: cos.MiB * 10})
+		mem.FreeSpec(memsys.FreeSpec{ToOS: true, MinSize: cos.MiB * 10})
 	}
 	wg.Wait()
 	close(c)
@@ -154,20 +152,3 @@ func memstress(mem *memsys.MMSA, id int, ttl time.Duration, siz, tot int64, wg *
 		tlog.Logf("%4d: done\n", id)
 	}
 }
-
-func printStats(mem *memsys.MMSA) {
-	for {
-		time.Sleep(mem.TimeIval)
-		stats := mem.GetStats()
-		for i := range memsys.NumPageSlabs {
-			slab, err := mem.GetSlab(int64(i+1) * memsys.PageSize)
-			cos.AssertNoErr(err)
-			x := ""
-			idle := stats.Idle[i]
-			if idle > 0 {
-				x = fmt.Sprintf(", idle=%v", idle)
-			}
-			fmt.Printf("%s: hits %d%s\n", slab.Tag(), stats.Hits[i], x)
-		}
-	}
-}

memsys/gc.go

@@ -13,19 +13,15 @@ import (
 )
 
 func (r *MMSA) freeMemToOS(mingc int64, p int, forces ...bool) {
-	var (
-		togc  = r.toGC.Load()
-		force bool
-	)
+	var force bool
 	if len(forces) > 0 {
 		force = forces[0]
 	}
 	if p >= PressureExtreme {
 		force = true
 	}
 	if !force && p <= PressureLow {
-		togc := r.toGC.Load()
-		if togc < mingc {
+		if togc := r.toGC.Load(); togc < mingc {
 			return // too little to bother
 		}
 	}
@@ -50,6 +46,6 @@ func (r *MMSA) freeMemToOS(mingc int64, p int, forces ...bool) {
 	if started := oom.FreeToOS(force); !started {
 		return
 	}
-	nlog.Warningln(r.String(), "free mem to OS [", r.pressure2S(p), force, cos.ToSizeIEC(togc, 1), load, "]")
-	r.toGC.Store(0)
+	togc := r.toGC.Swap(0)
+	nlog.Warningln(r.String(), "free mem to OS(pressure,force,togc,load) [", p, force, cos.ToSizeIEC(togc, 1), load, "]")
 }

memsys/housekeep_mm.go

@@ -6,14 +6,13 @@
 package memsys
 
 import (
-	"sort"
 	"time"
 
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
-	"github.com/NVIDIA/aistore/cmn/mono"
 	"github.com/NVIDIA/aistore/cmn/nlog"
+	"github.com/NVIDIA/aistore/sys"
 )
 
 const (
@@ -30,26 +29,8 @@ var (
 // API: on-demand memory freeing to the user-provided specification
 func (r *MMSA) FreeSpec(spec FreeSpec) {
 	var freed int64
-	if spec.Totally {
-		for _, s := range r.rings {
-			freed += s.cleanup()
-		}
-	} else {
-		if spec.IdleDuration == 0 {
-			spec.IdleDuration = freeIdleMinDur // using the default
-		}
-		stats := r.GetStats()
-		for _, s := range r.rings {
-			if idle := s.idleDur(stats); idle > spec.IdleDuration {
-				x := s.cleanup()
-				if x > 0 {
-					freed += x
-					if cmn.Rom.FastV(5, cos.SmoduleMemsys) {
-						nlog.Infof("%s: idle for %v - cleanup", s.tag, idle)
-					}
-				}
-			}
-		}
+	for _, s := range r.rings {
+		freed += s.cleanup()
 	}
 	if freed > 0 {
 		r.toGC.Add(freed)
@@ -61,29 +42,32 @@ func (r *MMSA) FreeSpec(spec FreeSpec) {
 	}
 }
 
-// copies part of the internal stats into user-visible Stats
-func (r *MMSA) GetStats() (stats *Stats) {
-	stats = &Stats{}
-	r._snap(stats, mono.NanoTime())
-	return
-}
-
 //
 // private
 //
 
 func (r *MMSA) hkcb(now int64) time.Duration {
-	// 1. refresh and clone stats
-	r.refreshStats(now)
-
-	// 2. update swapping state and compute mem-pressure ranking
+	// update swapping state and compute mem-pressure ranking
 	err := r.mem.Get()
-	debug.AssertNoErr(err)
+	if err != nil {
+		// (unlikely)
+		nlog.Errorln(err)
+		return max(r.TimeIval, time.Minute)
+	}
 	r.updSwap(&r.mem)
 	pressure := r.Pressure(&r.mem)
 
-	// 3. memory is enough, free only those that are idle for a while
+	// memory is enough: update idle times and free idle slabs, unless out of cpu
 	if pressure == PressureLow {
+		var (
+			load     = sys.MaxLoad()
+			highLoad = sys.HighLoadWM()
+		)
+		// too busy and not too "pressured"
+		if load >= float64(highLoad) {
+			return r.hkIval(pressure)
+		}
+		r.refreshStats(now)
 		r.optDepth.Store(optDepth)
 		if freed := r.freeIdle(); freed > 0 {
 			r.toGC.Add(freed)
@@ -92,7 +76,7 @@ func (r *MMSA) hkcb(now int64) time.Duration {
 		return r.hkIval(pressure)
 	}
 
-	// 4. calibrate and mem-free accordingly
+	// calibrate and mem-free accordingly
 	var (
 		mingc = sizeToGC // minimum accumulated size that triggers GC
 		depth int        // => current ring depth tbd
@@ -112,14 +96,8 @@ func (r *MMSA) hkcb(now int64) time.Duration {
 		depth = optDepth / 2
 	}
 
-	// 5.
-	// - sort (idle < less-idle < busy) taking into account durations and ring hits (in that order)
-	// - _reduce_
-	sort.Slice(r.sorted, r.idleLess)
-	for _, s := range r.sorted { // idle first
-		if idle := r.statsSnapshot.Idle[s.ringIdx()]; idle > freeIdleMinDur/2 {
-			depth = minDepth
-		}
+	// 5. reduce
+	for _, s := range r.rings {
 		freed := s.reduce(depth)
 		r.toGC.Add(freed)
 	}
@@ -143,44 +121,27 @@ func (r *MMSA) hkIval(pressure int) time.Duration {
 // refresh and clone internal hits/idle stats
 func (r *MMSA) refreshStats(now int64) {
 	for i := range r.numSlabs {
-		hits, prev := r.slabStats.hits[i].Load(), r.slabStats.prev[i]
-		hinc := hits - prev
-		if hinc == 0 {
-			r.slabStats.idleTs[i].CAS(0, now)
+		hits := r.hits[i].Swap(0)
+		if hits == 0 {
+			if !r.idleTs[i].CAS(0, now) {
+				r.idleDur[i] = time.Duration(now - r.idleTs[i].Load())
+			}
 		} else {
-			r.slabStats.idleTs[i].Store(0)
-		}
-		r.slabStats.hinc[i], r.slabStats.prev[i] = hinc, hits
-	}
-
-	r._snap(r.statsSnapshot, now)
-}
-
-func (r *MMSA) idleLess(i, j int) bool {
-	var (
-		ii = r.sorted[i].ringIdx()
-		jj = r.sorted[j].ringIdx()
-	)
-	if r.statsSnapshot.Idle[ii] > 0 {
-		if r.statsSnapshot.Idle[jj] > 0 {
-			return r.statsSnapshot.Idle[ii] > r.statsSnapshot.Idle[jj]
+			r.idleTs[i].Store(0)
+			r.idleDur[i] = 0
 		}
-		return true
-	}
-	if r.slabStats.idleTs[jj].Load() != 0 {
-		return false
 	}
-	return r.slabStats.hinc[ii] < r.slabStats.hinc[jj]
 }
 
 // freeIdle traverses and deallocates idle slabs- those that were not used for at
 // least the specified duration; returns freed size
 func (r *MMSA) freeIdle() (total int64) {
-	for _, s := range r.rings {
+	for i, s := range r.rings {
 		var (
 			freed int64
-			idle  = r.statsSnapshot.Idle[s.ringIdx()]
+			idle  = r.idleDur[i]
 		)
+		debug.Assert(s.ringIdx() == i)
 		switch {
 		case idle > freeIdleZero:
 			freed = s.cleanup()
@@ -198,13 +159,3 @@ func (r *MMSA) freeIdle() (total int64) {
 	}
 	return
 }
-
-func (r *MMSA) _snap(stats *Stats, now int64) {
-	for i := range r.rings {
-		stats.Hits[i] = r.slabStats.hits[i].Load()
-		stats.Idle[i] = 0
-		if since := r.slabStats.idleTs[i].Load(); since != 0 {
-			stats.Idle[i] = time.Duration(now - since)
-		}
-	}
-}

memsys/init.go

@@ -196,22 +196,19 @@ func (r *MMSA) Init(maxUse int64) (err error) {
 	if !r.isPage() {
 		r.maxSlabSize, r.numSlabs = MaxSmallSlabSize, NumSmallSlabs
 	}
-	r.slabStats = &slabStats{}
-	r.statsSnapshot = &Stats{}
 	r.rings = make([]*Slab, r.numSlabs)
-	r.sorted = make([]*Slab, r.numSlabs)
 	for i := range r.numSlabs {
 		bufSize := r.slabIncStep * int64(i+1)
 		slab := &Slab{
 			m:       r,
 			tag:     r.Name + "." + cos.ToSizeIEC(bufSize, 0),
 			bufSize: bufSize,
+			idx:     i,
 			get:     make([][]byte, 0, optDepth),
 			put:     make([][]byte, 0, optDepth),
 		}
 		slab.pMinDepth = &r.optDepth
 		r.rings[i] = slab
-		r.sorted[i] = slab
 	}
 	return
 }