Skip to content

Commit

Permalink
scrub: dynamic table (UI works)
Browse files Browse the repository at this point in the history
* part six, prev. commit: 0634d83
* remove static template
* hide zeros
* refactor a bunch

Signed-off-by: Alex Aizman <[email protected]>
  • Loading branch information
alex-aizman committed Dec 19, 2024
1 parent bafb910 commit 882d0ad
Show file tree
Hide file tree
Showing 24 changed files with 187 additions and 68 deletions.
2 changes: 1 addition & 1 deletion cmd/aisnode/main.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Package main for the AIS node executable.
/*
* Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
*/
package main

Expand Down
84 changes: 46 additions & 38 deletions cmd/cli/cli/scrub.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,25 @@ import (
// - async execution
// - pretty print

type (
_log struct {
fh *os.File
tag string
fn string
cnt int
mu sync.Mutex
}
// fields exported => teb/template
scrubOne teb.ScrubOne
)

type (
scrubCtx struct {
c *cli.Context
scrubs []*scrubOne
qbck cmn.QueryBcks
pref string
tmpl string
units string
// sizing
small int64
large int64
Expand All @@ -57,25 +69,6 @@ type (
}
_many bool
}
_log struct {
fh *os.File
tag string
fn string
cnt int
mu sync.Mutex
}
// fields exported => teb/template
scrubOne struct {
parent *scrubCtx
Bck cmn.Bck
Listed uint64
Stats struct {
Misplaced uint64
MissingCp uint64
SmallSz uint64
LargeSz uint64
}
}
)

func scrubHandler(c *cli.Context) (err error) {
Expand All @@ -102,10 +95,6 @@ func scrubHandler(c *cli.Context) (err error) {
}

ctx.last.Store(mono.NanoTime()) // pace interim results
ctx.tmpl = teb.ScrubTmpl
if flagIsSet(ctx.c, noHeaderFlag) {
ctx.tmpl = teb.ScrubBody
}

ctx.ival = listObjectsWaitTime
if flagIsSet(c, refreshFlag) {
Expand Down Expand Up @@ -140,6 +129,11 @@ func scrubHandler(c *cli.Context) (err error) {
return err
}

ctx.units, err = parseUnitsFlag(ctx.c, unitsFlag)
if err != nil {
return err
}

if ctx.qbck.IsBucket() {
err = waitForFunc(ctx.one, ctx.ival)
} else {
Expand Down Expand Up @@ -218,7 +212,19 @@ func (ctx *scrubCtx) many() error {
}
wg.Wait()

return teb.Print(ctx.scrubs, ctx.tmpl)
return ctx.prnt()
}

// print and be done
func (ctx *scrubCtx) prnt() error {
out := make([]*teb.ScrubOne, len(ctx.scrubs))
for i, scr := range ctx.scrubs {
out[i] = (*teb.ScrubOne)(scr)
}
all := teb.ScrubHelper{All: out}
tab := all.MakeTab(ctx.units)

return teb.Print(out, tab.Template(flagIsSet(ctx.c, noHeaderFlag)))
}

func (ctx *scrubCtx) gols(bck cmn.Bck, wg cos.WG, mu *sync.Mutex) {
Expand All @@ -239,7 +245,9 @@ func (ctx *scrubCtx) one() error {
if err != nil {
return err
}
return teb.Print([]*scrubOne{scr}, ctx.tmpl)

ctx.scrubs = []*scrubOne{scr}
return ctx.prnt()
}

func (ctx *scrubCtx) ls(bck cmn.Bck) (*scrubOne, error) {
Expand All @@ -250,7 +258,7 @@ func (ctx *scrubCtx) ls(bck cmn.Bck) (*scrubOne, error) {
bck.Props = bprops
var (
lsargs api.ListArgs
scr = &scrubOne{parent: ctx, Bck: bck}
scr = &scrubOne{Bck: bck, Prefix: ctx.pref}
lsmsg = &apc.LsoMsg{Prefix: ctx.pref, Flags: apc.LsObjCached | apc.LsMissing}
)
lsmsg.AddProps(apc.GetPropsName, apc.GetPropsSize)
Expand Down Expand Up @@ -279,7 +287,7 @@ func (ctx *scrubCtx) ls(bck cmn.Bck) (*scrubOne, error) {
continue
}
debug.Assert(en.IsPresent(), bck.Cname(en.Name), " must be present") // (LsObjCached)
scr.upd(en, bprops)
scr.upd(ctx, en, bprops)
}

if lsmsg.ContinuationToken == "" {
Expand Down Expand Up @@ -337,33 +345,33 @@ func (ctx *scrubCtx) ls(bck cmn.Bck) (*scrubOne, error) {
// scrubOne //
//////////////

func (scr *scrubOne) upd(en *cmn.LsoEnt, bprops *cmn.Bprops) {
func (scr *scrubOne) upd(parent *scrubCtx, en *cmn.LsoEnt, bprops *cmn.Bprops) {
scr.Listed++
if !en.IsStatusOK() {
scr.Stats.Misplaced++
scr.log(&scr.parent.log.misplaced, scr.Bck.Cname(en.Name))
scr.log(&parent.log.misplaced, scr.Bck.Cname(en.Name), parent._many)
return
}
if bprops.Mirror.Enabled && en.Copies < int16(bprops.Mirror.Copies) {
scr.Stats.MissingCp++
scr.log(&scr.parent.log.missing, scr.Bck.Cname(en.Name))
scr.log(&parent.log.missing, scr.Bck.Cname(en.Name), parent._many)
}
if en.Size <= scr.parent.small {
if en.Size <= parent.small {
scr.Stats.SmallSz++
scr.log(&scr.parent.log.small, scr.Bck.Cname(en.Name))
} else if en.Size >= scr.parent.large {
scr.log(&parent.log.small, scr.Bck.Cname(en.Name), parent._many)
} else if en.Size >= parent.large {
scr.Stats.LargeSz++
scr.log(&scr.parent.log.large, scr.Bck.Cname(en.Name))
scr.log(&parent.log.large, scr.Bck.Cname(en.Name), parent._many)
}
}

func (scr *scrubOne) log(to *_log, s string) {
if scr.parent._many {
func (*scrubOne) log(to *_log, s string, lock bool) {
if lock {
to.mu.Lock()
}
fmt.Fprintln(to.fh, s)
to.cnt++
if scr.parent._many {
if lock {
to.mu.Unlock()
}
}
Expand Down
8 changes: 4 additions & 4 deletions cmd/cli/teb/performance.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func (c *PerfTabCtx) MakeTab(st StstMap) (*Table, int, error) {

// 2. exclude zero columns unless requested specific match or (--all)
if c.Regex == nil {
cols = _zerout(cols, st)
cols = st._zerout(cols)
}

// 3. sort (remaining) columns and shift `err-*` columns to the right
Expand All @@ -91,7 +91,7 @@ func (c *PerfTabCtx) MakeTab(st StstMap) (*Table, int, error) {
})

// 4. add STATUS column unless all nodes are online (`NodeOnline`)
cols = _addStatus(cols, st)
cols = st._addStatus(cols)

// 5. add regex-specified (ie, user-requested) metrics that are missing
// ------ api.GetStatsAndStatus() does not return zero counters -------
Expand Down Expand Up @@ -214,7 +214,7 @@ func (c *PerfTabCtx) MakeTab(st StstMap) (*Table, int, error) {
//

// remove all-zeros columns
func _zerout(cols []*header, st StstMap) []*header {
func (st StstMap) _zerout(cols []*header) []*header {
for i := 0; i < len(cols); i++ {
var found bool
h := cols[i]
Expand All @@ -240,7 +240,7 @@ func _zerout(cols []*header, st StstMap) []*header {
}

// (aternatively, could always add, conditionally hide)
func _addStatus(cols []*header, st StstMap) []*header {
func (st StstMap) _addStatus(cols []*header) []*header {
for _, ds := range st {
if ds.Status != NodeOnline {
cols = append(cols, &header{name: colStatus, hide: false})
Expand Down
117 changes: 117 additions & 0 deletions cmd/cli/teb/scrub.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Package teb contains templates and (templated) tables to format CLI output.
/*
* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
*/
package teb

import (
"strconv"

"github.com/NVIDIA/aistore/cmn"
)

const (
colBucket = "BUCKET" // + [/PREFIX]
colObjects = "OBJECTS"
colMisplaced = "MISPLACED"
colMissing = "MISSING COPIES"
colSmall = "SMALL"
colLarge = "LARGE"
)

type (
ScrubOne struct {
Bck cmn.Bck
Prefix string
Listed uint64
Stats struct {
Misplaced uint64
MissingCp uint64
SmallSz uint64
LargeSz uint64
}
}
ScrubHelper struct {
All []*ScrubOne
}
)

func (h *ScrubHelper) colFirst() string {
var num int
for _, scr := range h.All {
if scr.Prefix != "" {
num++
}
}
switch {
case num == len(h.All):
return colBucket + "/PREFIX"
case num > 0:
return colBucket + "[/PREFIX]"
default:
return colBucket
}
}

func (h *ScrubHelper) MakeTab(units string) *Table {
var (
cols = []*header{
{name: h.colFirst()},
{name: colObjects},
{name: colMisplaced},
{name: colMissing},
{name: colSmall},
{name: colLarge},
}
table = newTable(cols...)
)

_ = units // TODO -- FIXME: add total size; use units

h.hideMisplaced(cols, colMisplaced)
h.hideMissing(cols, colMissing)

for _, scr := range h.All {
row := []string{
scr.Bck.Cname(scr.Prefix),
strconv.FormatUint(scr.Listed, 10),
strconv.FormatUint(scr.Stats.Misplaced, 10),
strconv.FormatUint(scr.Stats.MissingCp, 10),
strconv.FormatUint(scr.Stats.SmallSz, 10),
strconv.FormatUint(scr.Stats.LargeSz, 10),
}
table.addRow(row)
}

return table
}

//
// remove/hide a few named all-zero columns
//

func (h *ScrubHelper) hideMisplaced(cols []*header, col string) {
for _, scr := range h.All {
if scr.Stats.Misplaced != 0 {
return
}
}
h._hideCol(cols, col)
}

func (h *ScrubHelper) hideMissing(cols []*header, col string) {
for _, scr := range h.All {
if scr.Stats.MissingCp != 0 {
return
}
}
h._hideCol(cols, col)
}

func (*ScrubHelper) _hideCol(cols []*header, name string) {
for _, col := range cols {
if col.name == name {
col.hide = true
}
}
}
6 changes: 0 additions & 6 deletions cmd/cli/teb/templates.go
Original file line number Diff line number Diff line change
Expand Up @@ -224,12 +224,6 @@ const (
"{{FormatBytesUns $v.TotalSize.PresentObjs 2}} {{FormatBytesUns $v.TotalSize.RemoteObjs 2}}\t {{$v.UsedPct}}%\n" +
"{{end}}"

scrubHdr = "BUCKET\t OBJECTS\t MISPLACED\t MISSING COPIES\t SMALL\t LARGE\n"
ScrubBody = "{{range $v := . }}" +
"{{FormatBckName $v.Bck}}\t {{$v.Listed}}\t {{$v.Stats.Misplaced}}\t {{$v.Stats.MissingCp}}\t {{$v.Stats.SmallSz}}\t {{$v.Stats.LargeSz}}\n" +
"{{end}}"
ScrubTmpl = scrubHdr + ScrubBody

// For `object put` mass uploader. A caller adds to the template
// total count and size. That is why the template ends with \t
MultiPutTmpl = "Files to upload:\nEXTENSION\t COUNT\t SIZE\n" +
Expand Down
2 changes: 1 addition & 1 deletion core/lcopy.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Package core provides core metadata and in-cluster API
/*
* Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
*/
package core

Expand Down
2 changes: 1 addition & 1 deletion core/meta/jsp.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Package meta: cluster-level metadata
/*
* Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
*/
package meta

Expand Down
2 changes: 1 addition & 1 deletion fs/err.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
/*
* Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
*/
package fs

Expand Down
2 changes: 1 addition & 1 deletion fs/utils.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
/*
* Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
*/
package fs

Expand Down
2 changes: 1 addition & 1 deletion fs/xattr_unix.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
/*
* Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
*/
package fs

Expand Down
2 changes: 1 addition & 1 deletion ios/diskstats_linux.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Package ios is a collection of interfaces to the local storage subsystem;
// the package includes OS-dependent implementations for those interfaces.
/*
* Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
*/
package ios

Expand Down
Loading

0 comments on commit 882d0ad

Please sign in to comment.