balloons: expose balloons and optionally containers with affinity in NRT

cmd/plugins/balloons/policy/balloons-policy.go

@@ -29,10 +29,12 @@ import (
 	"github.com/containers/nri-plugins/pkg/resmgr/events"
 	libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory"
 	policy "github.com/containers/nri-plugins/pkg/resmgr/policy"
+	policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy"
 	"github.com/containers/nri-plugins/pkg/utils"
 	"github.com/containers/nri-plugins/pkg/utils/cpuset"
 	idset "github.com/intel/goresctrl/pkg/utils"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -73,6 +75,7 @@ type balloons struct {
 	allowed   cpuset.CPUSet          // bounding set of CPUs we're allowed to use
 	reserved  cpuset.CPUSet          // system-/kube-reserved CPUs
 	freeCpus  cpuset.CPUSet          // CPUs to be included in growing or new ballons
+	ifreeCpus cpuset.CPUSet          // initially free CPUs before assigning any containers
 	cpuTree   *cpuTreeNode           // system CPU topology
 
 	reservedBalloonDef *BalloonDef // reserved balloon definition, pointer to bpoptions.BalloonDefs[x]
@@ -326,8 +329,182 @@ func (p *balloons) ExportResourceData(c cache.Container) map[string]string {
 }
 
 // GetTopologyZones returns the policy/pool data for 'topology zone' CRDs.
-func (b *balloons) GetTopologyZones() []*policy.TopologyZone {
-	return nil
+func (p *balloons) GetTopologyZones() []*policy.TopologyZone {
+	showContainers := false
+	if p.bpoptions.ShowContainersInNrt != nil {
+		showContainers = *p.bpoptions.ShowContainersInNrt
+	}
+
+	zones := []*policyapi.TopologyZone{}
+	sysmCpu := 1000 * p.cpuTree.cpus.Size()
+	for _, bln := range p.balloons {
+		// Expose every balloon as a separate zone.
+		zone := &policyapi.TopologyZone{
+			Name: bln.PrettyName(),
+			Type: "balloon",
+		}
+
+		cpu := &policyapi.ZoneResource{
+			Name: policyapi.CPUResource,
+		}
+
+		// "Capacity" is the total number of CPUs available in
+		// the system, including CPUs not allowed to be used
+		// by the policy.
+		cpu.Capacity = *resource.NewMilliQuantity(
+			int64(sysmCpu),
+			resource.DecimalSI)
+
+		// "Allocatable" is the largest CPU request of a
+		// container that can be fit into the balloon, given
+		// that this or other balloons do not include any
+		// containers.
+		maxBlnSize := p.ifreeCpus.Size()
+		if bln.Def.MinBalloons > 0 {
+			// If this is a pre-created balloon, then
+			// ifreeCpus is missing CPUs pre-allocated for
+			// it.
+			maxBlnSize += bln.Def.MinCpus
+		}
+		if bln.Def.MaxCpus == NoLimit || bln.Def.MaxCpus > maxBlnSize {
+			cpu.Allocatable = *resource.NewMilliQuantity(
+				1000*int64(maxBlnSize),
+				resource.DecimalSI)
+		} else {
+			cpu.Allocatable = *resource.NewMilliQuantity(
+				1000*int64(bln.Def.MaxCpus),
+				resource.DecimalSI)
+		}
+
+		// "Available" is the largest CPU request of a
+		// container that currently fits into the
+		// balloon. This takes into account containers already
+		// in the balloon, balloon's CPU limit (maxCPUs),
+		// policy's allowed CPUs and already allocated CPUs to
+		// other balloons (freeCpus) as the balloon may be
+		// inflated to fit the container.
+		blnReqmCpu := p.requestedMilliCpus(bln)
+		cpu.Available = *resource.NewMilliQuantity(
+			int64(bln.MaxAvailMilliCpus(p.freeCpus)-blnReqmCpu),
+			resource.DecimalSI)
+
+		zone.Resources = append(zone.Resources, cpu)
+
+		attributes := []*policyapi.ZoneAttribute{
+			{
+				// "cpuset" are CPUs allowed only to
+				// containers in this balloon.
+				Name:  policyapi.CPUsAttribute,
+				Value: bln.Cpus.String(),
+			},
+			{
+				// "shared cpuset" are CPUs allowed to
+				// containers in this and other
+				// balloons that shareIdleCPUsInSame
+				// scope.
+				Name:  policyapi.SharedCPUsAttribute,
+				Value: bln.SharedIdleCpus.String(),
+			},
+			{
+				// "excess cpus" is the largest CPU
+				// request of a container that fits
+				// into this balloon without inflating
+				// it.
+				Name:  policyapi.ExcessCPUsAttribute,
+				Value: fmt.Sprintf("%dm", bln.AvailMilliCpus()-blnReqmCpu),
+			},
+		}
+		zone.Attributes = append(zone.Attributes, attributes...)
+		zones = append(zones, zone)
+
+		// Add more zones only if showing containers as part
+		// of node resource topologies is enabled.
+		showContainersOfThisBalloon := showContainers
+		if bln.Def.ShowContainersInNrt != nil {
+			showContainersOfThisBalloon = *bln.Def.ShowContainersInNrt
+		}
+		if !showContainersOfThisBalloon {
+			continue
+		}
+
+		// A container assigned into a balloon is exposed as a
+		// "allocation for container" subzone whose parent is
+		// the balloon zone. The subzone has following
+		// resources:
+		//
+		// "Capacity": container's resource usage limit. CPU
+		// usage of the container is limited by
+		// resources.limits.cpu and the number of allowed CPUs
+		// (balloon cpuset + shared). "Capacity" reflects the
+		// the tighter of these two limits.
+		//
+		// "Allocatable": container resource request. This
+		// reflects how container affects the balloon
+		// size. When the balloon has no "excess cpus", the
+		// sum of "Allocatable" CPUs of its containers equals
+		// to the size of balloon's cpuset.
+		//
+		// "Available": always 0. This prevents any kube
+		// scheduler extension from allocating resources from
+		// this subzone.
+		//
+		// Attributes of the subzone include cpuset and memory
+		// nodes allowed for the container. The cpuset consist
+		// of balloon's own and shared CPUs. Memory nodes
+		// depend on balloon-type parameters and pod
+		// annotations that specify if memory should be pinned
+		// at all, and which memory types should be used. Set
+		// of allowed memory nodes may be expanded from the
+		// lowest latency nodes due to memory requests that do
+		// not fit on the limited number of node set.
+		for _, ctrIDs := range bln.PodIDs {
+			for _, ctrID := range ctrIDs {
+				c, ok := p.cch.LookupContainer(ctrID)
+				if !ok {
+					continue
+				}
+				czone := &policyapi.TopologyZone{
+					Name: c.PrettyName(),
+					Type: policyapi.ContainerAllocationZoneType,
+				}
+				ctrLimitmCpu := p.containerLimitedMilliCpus(ctrID)
+				ctrReqmCpu := p.containerRequestedMilliCpus(ctrID)
+				ctrCapacitymCpu := ctrLimitmCpu
+				ctrCpusetCpus := c.GetCpusetCpus()
+				ctrAllowedmCpu := sysmCpu
+				if ctrCpusetCpus != "" {
+					ctrAllowedmCpu = 1000 * cpuset.MustParse(ctrCpusetCpus).Size()
+				}
+				if ctrLimitmCpu == 0 || ctrLimitmCpu > ctrAllowedmCpu {
+					ctrCapacitymCpu = ctrAllowedmCpu
+				}
+				czone.Resources = []*policyapi.ZoneResource{
+					{
+						Name: policyapi.CPUResource,
+						Capacity: *resource.NewMilliQuantity(
+							int64(ctrCapacitymCpu),
+							resource.DecimalSI),
+						Allocatable: *resource.NewMilliQuantity(
+							int64(ctrReqmCpu),
+							resource.DecimalSI),
+					},
+				}
+				czone.Parent = zone.Name
+				czone.Attributes = []*policyapi.ZoneAttribute{
+					{
+						Name:  policyapi.CPUsAttribute,
+						Value: ctrCpusetCpus,
+					},
+					{
+						Name:  policyapi.MemsetAttribute,
+						Value: c.GetCpusetMems(),
+					},
+				}
+				zones = append(zones, czone)
+			}
+		}
+	}
+	return zones
 }
 
 // balloonByContainer returns a balloon that contains a container.
@@ -1121,6 +1298,7 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error {
 			}
 		}
 	}
+	p.ifreeCpus = p.freeCpus.Clone()
 
 	// Finish balloon instance initialization.
 	log.Info("%s policy balloons:", PolicyName)

config/crd/bases/config.nri_balloonspolicies.yaml

@@ -285,6 +285,13 @@ spec:
                       - core
                       - thread
                       type: string
+                    showContainersInNrt:
+                      description: |-
+                        ShowContainersInNrt controls showing containers as part of
+                        NodeResourceTopology. Overrides the policy level
+                        ShowContainersInNrt setting for containers in this balloon
+                        type. Requires agent:NodeResourceTopology.
+                      type: boolean
                   required:
                   - name
                   type: object
@@ -513,6 +520,18 @@ spec:
                   type: string
                 description: Reserved (CPU) resources for kube-system namespace.
                 type: object
+              showContainersInNrt:
+                description: |-
+                  ShowContainersInNrt controls whether containers in balloons
+                  are exposed as part of NodeResourceTopology. If true,
+                  noderesourcetopologies.topology.node.k8s.io custom
+                  resources provide visibility to CPU and memory affinity of
+                  containers assigned into balloons on any node. The default
+                  is false. Use balloon-type option with the same name to
+                  override the policy-level default. Note that this has no
+                  effect unless agent:NodeResourceTopology enables basic
+                  topology exposure.
+                type: boolean
             required:
             - reservedResources
             type: object

deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml

@@ -285,6 +285,13 @@ spec:
                       - core
                       - thread
                       type: string
+                    showContainersInNrt:
+                      description: |-
+                        ShowContainersInNrt controls showing containers as part of
+                        NodeResourceTopology. Overrides the policy level
+                        ShowContainersInNrt setting for containers in this balloon
+                        type. Requires agent:NodeResourceTopology.
+                      type: boolean
                   required:
                   - name
                   type: object
@@ -513,6 +520,18 @@ spec:
                   type: string
                 description: Reserved (CPU) resources for kube-system namespace.
                 type: object
+              showContainersInNrt:
+                description: |-
+                  ShowContainersInNrt controls whether containers in balloons
+                  are exposed as part of NodeResourceTopology. If true,
+                  noderesourcetopologies.topology.node.k8s.io custom
+                  resources provide visibility to CPU and memory affinity of
+                  containers assigned into balloons on any node. The default
+                  is false. Use balloon-type option with the same name to
+                  override the policy-level default. Note that this has no
+                  effect unless agent:NodeResourceTopology enables basic
+                  topology exposure.
+                type: boolean
             required:
             - reservedResources
             type: object

docs/resource-policy/policy/balloons.md

@@ -8,7 +8,10 @@ can be dynamically inflated and deflated, that is CPUs added and
 removed, based on the CPU resource requests of containers running in
 the balloon. Balloons can be static or dynamically created and
 destroyed. CPUs in balloons can be configured, for example, by setting
-min and max frequencies on CPU cores and uncore.
+min and max frequencies on CPU cores and uncore. Balloons in
+Kubernetes cluster, including CPU and memory affinities of their
+containers, can be exposed and observed through noderesourcetopologies
+custom resources.
 
 ## How It Works
 
@@ -124,6 +127,14 @@ Balloons policy parameters:
   value set here is the default for all balloon types, but it can be
   overridden with the balloon type specific setting with the same
   name.
+- `showContainersInNrt` controls whether containers in balloons are
+  exposed as part of NodeResourceTopology. If `true`,
+  noderesourcetopologies.topology.node.k8s.io custom resources provide
+  visibility to CPU and memory affinity of containers assigned into
+  balloons. The default is `false`. Use balloon-type option with the
+  same name to override the policy-level default. Note that this has
+  no effect unless `agent:NodeResourceTopology` enables node resource
+  topology exposure in general.
 - `balloonTypes` is a list of balloon type definitions. The order of
   the types is significant in two cases.
 
@@ -271,6 +282,13 @@ Balloons policy parameters:
      and assigned containers are readable through `/metrics` from the
      httpEndpoint.
   - `reportPeriod`: `/metrics` aggregation interval for polled metrics.
+- `agent`: controls communicating with the Kubernetes node agent and
+  the API server.
+  - `nodeResourceTopology`: if `true`, expose balloons as node
+    resource topology zones in noderesourcetopologies custom
+    resources. Moreover, showing containers assigned to balloons and
+    their CPU/memory affinities can be enabled with
+    `showContainersInNrt`. The default is `false`.
 
 ### Example
 
@@ -285,6 +303,11 @@ metadata:
   name: default
   namespace: kube-system
 spec:
+  # Expose balloons as node resource topology zones in
+  # noderesourcestopologies custom resources.
+  agent:
+    nodeResourceTopology: true
+
   reservedResources:
     cpu: 1000m
   pinCPU: true
@@ -297,6 +320,7 @@ spec:
       cpuClass: dynamic
       namespaces:
         - "*"
+      showContainersInNrt: true
   control:
     cpu:
       classes:

pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go

@@ -82,6 +82,16 @@ type Config struct {
 	// Preserve specifies containers whose resource pinning must not be
 	// modified by the policy.
 	Preserve *ContainerMatchConfig `json:"preserve,omitempty"`
+	// ShowContainersInNrt controls whether containers in balloons
+	// are exposed as part of NodeResourceTopology. If true,
+	// noderesourcetopologies.topology.node.k8s.io custom
+	// resources provide visibility to CPU and memory affinity of
+	// containers assigned into balloons on any node. The default
+	// is false. Use balloon-type option with the same name to
+	// override the policy-level default. Note that this has no
+	// effect unless agent:NodeResourceTopology enables basic
+	// topology exposure.
+	ShowContainersInNrt *bool `json:"showContainersInNrt,omitempty"`
 }
 
 type CPUTopologyLevel string
@@ -239,6 +249,11 @@ type BalloonDef struct {
 	// +optional
 	// +kubebuilder:validation:Enum=efficient;performance
 	PreferCoreType string `json:"preferCoreType,omitempty"`
+	// ShowContainersInNrt controls showing containers as part of
+	// NodeResourceTopology. Overrides the policy level
+	// ShowContainersInNrt setting for containers in this balloon
+	// type. Requires agent:NodeResourceTopology.
+	ShowContainersInNrt *bool `json:"showContainersInNrt,omitempty"`
 }
 
 // String stringifies a BalloonDef