diff --git a/controllers/devbox/cmd/main.go b/controllers/devbox/cmd/main.go index 62416e9441d..6a9b84efef1 100644 --- a/controllers/devbox/cmd/main.go +++ b/controllers/devbox/cmd/main.go @@ -27,6 +27,7 @@ import ( "k8s.io/client-go/rest" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -42,7 +43,9 @@ import ( devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" "github.com/labring/sealos/controllers/devbox/internal/controller" + "github.com/labring/sealos/controllers/devbox/internal/controller/utils/matcher" "github.com/labring/sealos/controllers/devbox/internal/controller/utils/registry" + utilresource "github.com/labring/sealos/controllers/devbox/internal/controller/utils/resource" // +kubebuilder:scaffold:imports ) @@ -65,19 +68,24 @@ func main() { var secureMetrics bool var enableHTTP2 bool var tlsOpts []func(*tls.Config) + // debug flag + var debugMode bool + // registry flag var registryAddr string var registryUser string var registryPassword string - var authAddr string + // resource flag var requestCPURate float64 var requestMemoryRate float64 var requestEphemeralStorage string var limitEphemeralStorage string - var debugMode bool - flag.StringVar(®istryAddr, "registry-addr", "sealos.hub:5000", "The address of the registry") - flag.StringVar(®istryUser, "registry-user", "admin", "The user of the registry") - flag.StringVar(®istryPassword, "registry-password", "passw0rd", "The password of the registry") - flag.StringVar(&authAddr, "auth-addr", "sealos.hub:5000", "The address of the auth") + var maximumLimitEphemeralStorage string + // pod matcher flag + var enablePodResourceMatcher bool + var enablePodEnvMatcher bool + var enablePodPortMatcher bool + var enablePodEphemeralStorageMatcher bool + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -88,11 +96,24 @@ func main() { "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") + // debug flag flag.BoolVar(&debugMode, "debug", false, "If set, debug mode will be enabled") + // registry flag + flag.StringVar(®istryAddr, "registry-addr", "sealos.hub:5000", "The address of the registry") + flag.StringVar(®istryUser, "registry-user", "admin", "The user of the registry") + flag.StringVar(®istryPassword, "registry-password", "passw0rd", "The password of the registry") + // resource flag flag.Float64Var(&requestCPURate, "request-cpu-rate", 10, "The request rate of cpu limit in devbox.") flag.Float64Var(&requestMemoryRate, "request-memory-rate", 10, "The request rate of memory limit in devbox.") - flag.StringVar(&requestEphemeralStorage, "request-ephemeral-storage", "500Mi", "The request value of ephemeral storage in devbox.") - flag.StringVar(&limitEphemeralStorage, "limit-ephemeral-storage", "10Gi", "The limit value of ephemeral storage in devbox.") + flag.StringVar(&requestEphemeralStorage, "request-ephemeral-storage", "500Mi", "The default request value of ephemeral storage in devbox.") + flag.StringVar(&limitEphemeralStorage, "limit-ephemeral-storage", "10Gi", "The default limit value of ephemeral storage in devbox.") + flag.StringVar(&maximumLimitEphemeralStorage, "maximum-limit-ephemeral-storage", "50Gi", "The maximum limit value of ephemeral storage in devbox.") + // pod matcher flag, pod resource matcher, env matcher, port matcher will be enabled by default, ephemeral storage matcher will be disabled by default + flag.BoolVar(&enablePodResourceMatcher, "enable-pod-resource-matcher", true, "If set, pod resource matcher will be enabled") + flag.BoolVar(&enablePodEnvMatcher, "enable-pod-env-matcher", true, "If set, pod env matcher will be enabled") + flag.BoolVar(&enablePodPortMatcher, "enable-pod-port-matcher", true, "If set, pod port matcher will be enabled") + flag.BoolVar(&enablePodEphemeralStorageMatcher, "enable-pod-ephemeral-storage-matcher", false, "If set, pod ephemeral storage matcher will be enabled") + opts := zap.Options{ Development: true, } @@ -182,16 +203,36 @@ func main() { os.Exit(1) } + podMatchers := []matcher.PodMatcher{} + if enablePodResourceMatcher { + podMatchers = append(podMatchers, matcher.ResourceMatcher{}) + } + if enablePodEnvMatcher { + podMatchers = append(podMatchers, matcher.EnvVarMatcher{}) + } + if enablePodPortMatcher { + podMatchers = append(podMatchers, matcher.PortMatcher{}) + } + if enablePodEphemeralStorageMatcher { + podMatchers = append(podMatchers, matcher.EphemeralStorageMatcher{}) + } + if err = (&controller.DevboxReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - CommitImageRegistry: registryAddr, - Recorder: mgr.GetEventRecorderFor("devbox-controller"), - RequestCPURate: requestCPURate, - RequestMemoryRate: requestMemoryRate, - RequestEphemeralStorage: requestEphemeralStorage, - LimitEphemeralStorage: limitEphemeralStorage, - DebugMode: debugMode, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + CommitImageRegistry: registryAddr, + Recorder: mgr.GetEventRecorderFor("devbox-controller"), + RequestRate: utilresource.RequestRate{ + CPU: requestCPURate, + Memory: requestMemoryRate, + }, + EphemeralStorage: utilresource.EphemeralStorage{ + DefaultRequest: resource.MustParse(requestEphemeralStorage), + DefaultLimit: resource.MustParse(limitEphemeralStorage), + MaximumLimit: resource.MustParse(maximumLimitEphemeralStorage), + }, + PodMatchers: podMatchers, + DebugMode: debugMode, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Devbox") os.Exit(1) diff --git a/controllers/devbox/config/samples/devbox_v1alpha1_devbox.yaml b/controllers/devbox/config/samples/devbox_v1alpha1_devbox.yaml index b0b194ef79c..71e5bf070b0 100644 --- a/controllers/devbox/config/samples/devbox_v1alpha1_devbox.yaml +++ b/controllers/devbox/config/samples/devbox_v1alpha1_devbox.yaml @@ -18,19 +18,23 @@ metadata: labels: app.kubernetes.io/name: devbox app.kubernetes.io/managed-by: kustomize - name: devbox-sample + name: devbox-gpu-sample spec: state: Running + runtimeClassName: nvidia resource: cpu: 2 memory: 4000Mi + nvidia.com/gpu: 1 runtimeRef: - name: go-1-22-5 + name: go-1-22-5-2024-11-12-0651 namespace: devbox-system + nodeSelector: + nvidia.com/gpu.product: Tesla-P40 network: type: NodePort extraPorts: - containerPort: 443 name: 'https' - containerPort: 80 - name: 'http' \ No newline at end of file + name: 'http' diff --git a/controllers/devbox/config/samples/devbox_v1alpha1_runtime.yaml b/controllers/devbox/config/samples/devbox_v1alpha1_runtime.yaml index 774a7a5947e..06badcd4f93 100644 --- a/controllers/devbox/config/samples/devbox_v1alpha1_runtime.yaml +++ b/controllers/devbox/config/samples/devbox_v1alpha1_runtime.yaml @@ -31,6 +31,7 @@ spec: - /home/sealos/project/entrypoint.sh category: - ubuntu + - gpu - go --- apiVersion: devbox.sealos.io/v1alpha1 diff --git a/controllers/devbox/config/samples/devbox_v1alpha1_runtimeclass.yaml b/controllers/devbox/config/samples/devbox_v1alpha1_runtimeclass.yaml index 3886893f263..0032d957bee 100644 --- a/controllers/devbox/config/samples/devbox_v1alpha1_runtimeclass.yaml +++ b/controllers/devbox/config/samples/devbox_v1alpha1_runtimeclass.yaml @@ -70,4 +70,4 @@ metadata: spec: kind: Language title: node.js - description: node.js \ No newline at end of file + description: node.js diff --git a/controllers/devbox/internal/controller/devbox_controller.go b/controllers/devbox/internal/controller/devbox_controller.go index 67842732112..51833e541c6 100644 --- a/controllers/devbox/internal/controller/devbox_controller.go +++ b/controllers/devbox/internal/controller/devbox_controller.go @@ -23,6 +23,8 @@ import ( devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" "github.com/labring/sealos/controllers/devbox/internal/controller/helper" + "github.com/labring/sealos/controllers/devbox/internal/controller/utils/matcher" + "github.com/labring/sealos/controllers/devbox/internal/controller/utils/resource" "github.com/labring/sealos/controllers/devbox/label" corev1 "k8s.io/api/core/v1" @@ -44,11 +46,12 @@ import ( // DevboxReconciler reconciles a Devbox object type DevboxReconciler struct { - CommitImageRegistry string - RequestCPURate float64 - RequestMemoryRate float64 - RequestEphemeralStorage string - LimitEphemeralStorage string + CommitImageRegistry string + + RequestRate resource.RequestRate + EphemeralStorage resource.EphemeralStorage + + PodMatchers []matcher.PodMatcher DebugMode bool @@ -298,7 +301,7 @@ func (r *DevboxReconciler) syncPod(ctx context.Context, devbox *devboxv1alpha1.D logger.Info("pod has been deleted") return r.handlePodDeleted(ctx, devbox, pod) } - switch helper.PodMatchExpectations(expectPod, pod) { + switch matcher.PodMatchExpectations(expectPod, pod, r.PodMatchers...) { case true: // pod match expectations logger.Info("pod match expectations") @@ -557,7 +560,7 @@ func (r *DevboxReconciler) generateDevboxPod(devbox *devboxv1alpha1.Devbox, runt WorkingDir: helper.GenerateWorkingDir(devbox, runtime), Command: helper.GenerateCommand(devbox, runtime), Args: helper.GenerateDevboxArgs(devbox, runtime), - Resources: helper.GenerateResourceRequirements(devbox, r.RequestCPURate, r.RequestMemoryRate, r.RequestEphemeralStorage, r.LimitEphemeralStorage), + Resources: helper.GenerateResourceRequirements(devbox, r.RequestRate, r.EphemeralStorage), }, } diff --git a/controllers/devbox/internal/controller/helper/devbox.go b/controllers/devbox/internal/controller/helper/devbox.go index 1dc263403c5..74c2a601e1a 100644 --- a/controllers/devbox/internal/controller/helper/devbox.go +++ b/controllers/devbox/internal/controller/helper/devbox.go @@ -16,7 +16,6 @@ package helper import ( "fmt" - "log/slog" "sort" "strings" @@ -31,6 +30,7 @@ import ( "k8s.io/utils/ptr" devboxv1alpha1 "github.com/labring/sealos/controllers/devbox/api/v1alpha1" + utilsresource "github.com/labring/sealos/controllers/devbox/internal/controller/utils/resource" "github.com/labring/sealos/controllers/devbox/label" ) @@ -200,10 +200,6 @@ func podContainerID(pod *corev1.Pod) string { } return "" } - -// PredicateCommitStatus returns the commit status of the pod -// if the pod container id is empty, it means the pod is pending or has't started, we can assume the image has not been committed -// otherwise, it means the pod has been started, we can assume the image has been committed func PredicateCommitStatus(pod *corev1.Pod) devboxv1alpha1.CommitStatus { if podContainerID(pod) == "" { return devboxv1alpha1.CommitStatusPending @@ -211,85 +207,6 @@ func PredicateCommitStatus(pod *corev1.Pod) devboxv1alpha1.CommitStatus { return devboxv1alpha1.CommitStatusSuccess } -func PodMatchExpectations(expectPod *corev1.Pod, pod *corev1.Pod) bool { - if len(pod.Spec.Containers) == 0 { - slog.Info("Pod has no containers") - return false - } - container := pod.Spec.Containers[0] - expectContainer := expectPod.Spec.Containers[0] - - // Check CPU and memory limits - if container.Resources.Requests.Cpu().Cmp(*expectContainer.Resources.Requests.Cpu()) != 0 { - slog.Info("CPU requests are not equal") - return false - } - if container.Resources.Limits.Cpu().Cmp(*expectContainer.Resources.Limits.Cpu()) != 0 { - slog.Info("CPU limits are not equal") - return false - } - if container.Resources.Requests.Memory().Cmp(*expectContainer.Resources.Requests.Memory()) != 0 { - slog.Info("Memory requests are not equal") - return false - } - if container.Resources.Limits.Memory().Cmp(*expectContainer.Resources.Limits.Memory()) != 0 { - slog.Info("Memory limits are not equal") - return false - } - - // Check Ephemeral Storage changes - if container.Resources.Requests.StorageEphemeral().Cmp(*expectContainer.Resources.Requests.StorageEphemeral()) != 0 { - slog.Info("Ephemeral-Storage requests are not equal") - return false - } - if container.Resources.Limits.StorageEphemeral().Cmp(*expectContainer.Resources.Limits.StorageEphemeral()) != 0 { - slog.Info("Ephemeral-Storage limits are not equal") - return false - } - - // Check environment variables - if len(container.Env) != len(expectContainer.Env) { - return false - } - for _, env := range container.Env { - found := false - for _, expectEnv := range expectContainer.Env { - if env.Name == "SEALOS_COMMIT_IMAGE_NAME" { - found = true - break - } - if env.Name == expectEnv.Name && env.Value == expectEnv.Value { - found = true - break - } - } - if !found { - slog.Info("Environment variables are not equal", "env not found", env.Name, "env value", env.Value) - return false - } - } - - // Check ports - if len(container.Ports) != len(expectContainer.Ports) { - return false - } - for _, expectPort := range expectContainer.Ports { - found := false - for _, podPort := range container.Ports { - if expectPort.ContainerPort == podPort.ContainerPort && expectPort.Protocol == podPort.Protocol { - found = true - break - } - } - if !found { - slog.Info("Ports are not equal") - return false - } - } - - return true -} - func GenerateDevboxEnvVars(devbox *devboxv1alpha1.Devbox, nextCommitHistory *devboxv1alpha1.CommitHistory) []corev1.EnvVar { // if devbox.Spec.Squash is true, and devbox.Status.CommitHistory has success commit history, we need to set SEALOS_COMMIT_IMAGE_SQUASH to true doSquash := false @@ -398,34 +315,45 @@ func GenerateSSHVolume(devbox *devboxv1alpha1.Devbox) corev1.Volume { } } -func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, requestCPURate, requestMemoryRate float64, requestEphemeralStorage, limitEphemeralStorage string) corev1.ResourceRequirements { - res := corev1.ResourceRequirements{} - res.Limits = devbox.Spec.Resource - if limitEphemeralStorage != "" { - res.Limits[corev1.ResourceEphemeralStorage] = resource.MustParse(limitEphemeralStorage) +// GenerateResourceRequirements generates the resource requirements for the Devbox pod +func GenerateResourceRequirements(devbox *devboxv1alpha1.Devbox, requestRate utilsresource.RequestRate, ephemeralStorage utilsresource.EphemeralStorage) corev1.ResourceRequirements { + return corev1.ResourceRequirements{ + Limits: calculateResourceLimit(devbox.Spec.Resource, ephemeralStorage), + Requests: calculateResourceRequest(devbox.Spec.Resource, requestRate, ephemeralStorage), } - res.Requests = calculateResourceRequest(res.Limits, requestCPURate, requestMemoryRate) - if requestEphemeralStorage != "" { - res.Requests[corev1.ResourceEphemeralStorage] = resource.MustParse(requestEphemeralStorage) +} + +func calculateResourceLimit(original corev1.ResourceList, ephemeralStorage utilsresource.EphemeralStorage) corev1.ResourceList { + limit := original.DeepCopy() + // If ephemeral storage limit is not set, set it to default limit + if l, ok := limit[corev1.ResourceEphemeralStorage]; !ok { + limit[corev1.ResourceEphemeralStorage] = ephemeralStorage.DefaultLimit + } else { + // Check if the resource limit for ephemeral storage is set and compare it, if it is exceeded the maximum limit, set it to maximum limit + if l.AsApproximateFloat64() > ephemeralStorage.MaximumLimit.AsApproximateFloat64() { + limit[corev1.ResourceEphemeralStorage] = ephemeralStorage.MaximumLimit + } } - return res + return limit } -func calculateResourceRequest(limit corev1.ResourceList, requestCPURate, requestMemoryRate float64) corev1.ResourceList { +func calculateResourceRequest(original corev1.ResourceList, requestRate utilsresource.RequestRate, ephemeralStorage utilsresource.EphemeralStorage) corev1.ResourceList { // deep copy limit to request, only cpu and memory are calculated - request := limit.DeepCopy() + request := original.DeepCopy() // Calculate CPU request - if cpu, ok := limit[corev1.ResourceCPU]; ok { + if cpu, ok := original[corev1.ResourceCPU]; ok { cpuValue := cpu.AsApproximateFloat64() - cpuRequest := cpuValue / requestCPURate + cpuRequest := cpuValue / requestRate.CPU request[corev1.ResourceCPU] = *resource.NewMilliQuantity(int64(cpuRequest*1000), resource.DecimalSI) } // Calculate memory request - if memory, ok := limit[corev1.ResourceMemory]; ok { + if memory, ok := original[corev1.ResourceMemory]; ok { memoryValue := memory.AsApproximateFloat64() - memoryRequest := memoryValue / requestMemoryRate + memoryRequest := memoryValue / requestRate.Memory request[corev1.ResourceMemory] = *resource.NewQuantity(int64(memoryRequest), resource.BinarySI) } + // Set ephemeral storage request to default request + request[corev1.ResourceEphemeralStorage] = ephemeralStorage.DefaultRequest return request } diff --git a/controllers/devbox/internal/controller/utils/matcher/matcher.go b/controllers/devbox/internal/controller/utils/matcher/matcher.go new file mode 100644 index 00000000000..61f4567ba21 --- /dev/null +++ b/controllers/devbox/internal/controller/utils/matcher/matcher.go @@ -0,0 +1,140 @@ +package matcher + +import ( + "log/slog" + + corev1 "k8s.io/api/core/v1" +) + +type PodMatcher interface { + Match(expectPod *corev1.Pod, pod *corev1.Pod) bool +} + +type ResourceMatcher struct{} + +func (r ResourceMatcher) Match(expectPod *corev1.Pod, pod *corev1.Pod) bool { + if len(pod.Spec.Containers) == 0 { + slog.Info("Pod has no containers") + return false + } + container := pod.Spec.Containers[0] + expectContainer := expectPod.Spec.Containers[0] + + if container.Resources.Requests.Cpu().Cmp(*expectContainer.Resources.Requests.Cpu()) != 0 { + slog.Info("CPU requests are not equal") + return false + } + if container.Resources.Limits.Cpu().Cmp(*expectContainer.Resources.Limits.Cpu()) != 0 { + slog.Info("CPU limits are not equal") + return false + } + if container.Resources.Requests.Memory().Cmp(*expectContainer.Resources.Requests.Memory()) != 0 { + slog.Info("Memory requests are not equal") + return false + } + if container.Resources.Limits.Memory().Cmp(*expectContainer.Resources.Limits.Memory()) != 0 { + slog.Info("Memory limits are not equal") + return false + } + return true +} + +type EphemeralStorageMatcher struct{} + +func (e EphemeralStorageMatcher) Match(expectPod *corev1.Pod, pod *corev1.Pod) bool { + if len(pod.Spec.Containers) == 0 { + slog.Info("Pod has no containers") + return false + } + container := pod.Spec.Containers[0] + expectContainer := expectPod.Spec.Containers[0] + + if container.Resources.Limits.StorageEphemeral().Cmp(*expectContainer.Resources.Limits.StorageEphemeral()) != 0 { + slog.Info("Ephemeral-Storage limits are not equal") + return false + } + if container.Resources.Requests.StorageEphemeral().Cmp(*expectContainer.Resources.Requests.StorageEphemeral()) != 0 { + slog.Info("Ephemeral-Storage requests are not equal") + return false + } + return true +} + +type EnvVarMatcher struct{} + +func (e EnvVarMatcher) Match(expectPod *corev1.Pod, pod *corev1.Pod) bool { + if len(pod.Spec.Containers) == 0 { + slog.Info("Pod has no containers") + return false + } + container := pod.Spec.Containers[0] + expectContainer := expectPod.Spec.Containers[0] + + if len(container.Env) != len(expectContainer.Env) { + slog.Info("Environment variable count mismatch") + return false + } + + for _, env := range container.Env { + found := false + for _, expectEnv := range expectContainer.Env { + if env.Name == "SEALOS_COMMIT_IMAGE_NAME" { + found = true + break + } + if env.Name == expectEnv.Name && env.Value == expectEnv.Value { + found = true + break + } + } + if !found { + slog.Info("Environment variables are not equal", "env not found", env.Name, "env value", env.Value) + return false + } + } + return true +} + +type PortMatcher struct{} + +func (p PortMatcher) Match(expectPod *corev1.Pod, pod *corev1.Pod) bool { + if len(pod.Spec.Containers) == 0 { + slog.Info("Pod has no containers") + return false + } + container := pod.Spec.Containers[0] + expectContainer := expectPod.Spec.Containers[0] + + if len(container.Ports) != len(expectContainer.Ports) { + slog.Info("Port count mismatch") + return false + } + + for _, expectPort := range expectContainer.Ports { + found := false + for _, podPort := range container.Ports { + if expectPort.ContainerPort == podPort.ContainerPort && expectPort.Protocol == podPort.Protocol { + found = true + break + } + } + if !found { + slog.Info("Ports are not equal") + return false + } + } + return true +} + +// PredicateCommitStatus returns the commit status of the pod +// if the pod container id is empty, it means the pod is pending or has't started, we can assume the image has not been committed +// otherwise, it means the pod has been started, we can assume the image has been committed + +func PodMatchExpectations(expectPod *corev1.Pod, pod *corev1.Pod, matchers ...PodMatcher) bool { + for _, matcher := range matchers { + if !matcher.Match(expectPod, pod) { + return false + } + } + return true +} diff --git a/controllers/devbox/internal/controller/helper/devbox_test.go b/controllers/devbox/internal/controller/utils/matcher/matcher_test.go similarity index 95% rename from controllers/devbox/internal/controller/helper/devbox_test.go rename to controllers/devbox/internal/controller/utils/matcher/matcher_test.go index f3563b743fb..1317615196a 100644 --- a/controllers/devbox/internal/controller/helper/devbox_test.go +++ b/controllers/devbox/internal/controller/utils/matcher/matcher_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helper +package matcher import ( "testing" @@ -156,9 +156,16 @@ func TestPodMatchExpectations(t *testing.T) { }, } + matchers := []PodMatcher{ + ResourceMatcher{}, + EnvVarMatcher{}, + PortMatcher{}, + EphemeralStorageMatcher{}, + } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := PodMatchExpectations(expectPod, tt.pod) + result := PodMatchExpectations(expectPod, tt.pod, matchers...) if result != tt.expected { t.Errorf("CheckPodConsistency() = %v, expected %v", result, tt.expected) } diff --git a/controllers/devbox/internal/controller/utils/resource/resource.go b/controllers/devbox/internal/controller/utils/resource/resource.go new file mode 100644 index 00000000000..a14adb96307 --- /dev/null +++ b/controllers/devbox/internal/controller/utils/resource/resource.go @@ -0,0 +1,16 @@ +package resource + +import ( + "k8s.io/apimachinery/pkg/api/resource" +) + +type RequestRate struct { + CPU float64 + Memory float64 +} + +type EphemeralStorage struct { + DefaultRequest resource.Quantity + DefaultLimit resource.Quantity + MaximumLimit resource.Quantity +}