diff --git a/go.mod b/go.mod index 0c6f6eb30..c4a39ccd0 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 + github.com/victorspringer/http-cache v0.0.0-20240523143319-7d9f48f8ab91 go.opentelemetry.io/otel v1.24.0 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 @@ -189,6 +190,7 @@ require ( replace ( github.com/openvex/go-vex => github.com/slashben/go-vex v0.0.0-20231012123606-f58e5ee0e14e + github.com/victorspringer/http-cache => github.com/matthyx/http-cache v0.0.0-20240719133808-8a605008c1fd google.golang.org/grpc => google.golang.org/grpc v1.56.3 k8s.io/api => k8s.io/api v0.0.0-20231101171312-cd0ecb048ea5 k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20231102051132-bc0a03b4342c diff --git a/go.sum b/go.sum index 96a63c0c1..ee4b6aba5 100644 --- a/go.sum +++ b/go.sum @@ -1072,6 +1072,8 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matthyx/http-cache v0.0.0-20240719133808-8a605008c1fd h1:KRFRiC+E9H4ux/YYw4ZJtSe/3Gpj1A2Erneuoz4nQho= +github.com/matthyx/http-cache v0.0.0-20240719133808-8a605008c1fd/go.mod h1:4QGcMnbmiLfsUheY+tz0CwBT4fL7gGCOKgDay2WwYX8= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -1127,6 +1129,7 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/olvrng/ujson v1.1.0 h1:8xVUzVlqwdMVWh5d1UHBtLQ1D50nxoPuPEq9Wozs8oA= github.com/olvrng/ujson v1.1.0/go.mod h1:Mz4G3RODTUfbkKyvi0lgmPx/7vd3Saksk+1jgk8s9xo= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.18.0 h1:W9Y7IWXxPUpAit9ieMOLI7PJZGaW22DTKgiVAuhDTLc= github.com/onsi/ginkgo/v2 v2.18.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= diff --git a/pkg/cleanup/cleanup.go b/pkg/cleanup/cleanup.go index f05fcd3af..2460f306d 100644 --- a/pkg/cleanup/cleanup.go +++ b/pkg/cleanup/cleanup.go @@ -19,7 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type TypeCleanupHandlerFunc func(kind, path string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool +type TypeCleanupHandlerFunc func(metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool var resourceKindToHandler = map[string]TypeCleanupHandlerFunc{ // configurationscansummaries is virtual @@ -136,7 +136,7 @@ func (h *ResourcesCleanupHandler) StartCleanupTask() { return nil } - toDelete := handler(resourceKind, path, metadata, h.resources) + toDelete := handler(metadata, h.resources) if toDelete { logger.L().Debug("deleting", helpers.String("kind", resourceKind), helpers.String("namespace", metadata.Namespace), helpers.String("name", metadata.Name)) h.deleteFunc(h.appFs, path) @@ -224,21 +224,21 @@ func unquote(value []byte) string { } // delete deprecated resources -func deleteDeprecated(_, _ string, _ *metav1.ObjectMeta, _ ResourceMaps) bool { +func deleteDeprecated(_ *metav1.ObjectMeta, _ ResourceMaps) bool { return true } -func deleteByInstanceId(_, _ string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { +func deleteByInstanceId(metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { instanceId, ok := metadata.Annotations[helpersv1.InstanceIDMetadataKey] return !ok || !resourceMaps.RunningInstanceIds.Contains(instanceId) } -func deleteByImageId(_, _ string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { +func deleteByImageId(metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { imageId, ok := metadata.Annotations[helpersv1.ImageIDMetadataKey] return !ok || !resourceMaps.RunningContainerImageIds.Contains(imageId) } -func deleteByWlid(_, _ string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { +func deleteByWlid(metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { wlid, ok := metadata.Annotations[helpersv1.WlidMetadataKey] kind := strings.ToLower(wlidPkg.GetKindFromWlid(wlid)) if !Workloads.Contains(kind) { @@ -250,7 +250,7 @@ func deleteByWlid(_, _ string, metadata *metav1.ObjectMeta, resourceMaps Resourc return !ok || !resourceMaps.RunningWlidsToContainerNames.Has(wlidWithoutClusterName(wlid)) } -func deleteByImageIdOrInstanceId(_, _ string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { +func deleteByImageIdOrInstanceId(metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { imageId, imageIdFound := metadata.Annotations[helpersv1.ImageIDMetadataKey] instanceId, instanceIdFound := metadata.Annotations[helpersv1.InstanceIDMetadataKey] return (!instanceIdFound && !imageIdFound) || @@ -258,7 +258,7 @@ func deleteByImageIdOrInstanceId(_, _ string, metadata *metav1.ObjectMeta, resou (instanceIdFound && !resourceMaps.RunningInstanceIds.Contains(instanceId)) } -func deleteByWlidAndContainer(_, _ string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { +func deleteByWlidAndContainer(metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { wlContainerName, wlContainerNameFound := metadata.Annotations[helpersv1.ContainerNameMetadataKey] wlid, wlidFound := metadata.Annotations[helpersv1.WlidMetadataKey] if !wlidFound || !wlContainerNameFound { @@ -268,11 +268,11 @@ func deleteByWlidAndContainer(_, _ string, metadata *metav1.ObjectMeta, resource return !wlidExists || !containerNames.Contains(wlContainerName) } -func deleteByTemplateHashOrWlid(_, _ string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { +func deleteByTemplateHashOrWlid(metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { wlReplica, wlReplicaFound := metadata.Labels[helpersv1.TemplateHashKey] // replica if wlReplicaFound && wlReplica != "" { return !resourceMaps.RunningTemplateHash.Contains(wlReplica) } // fallback to wlid - return deleteByWlid("", "", metadata, resourceMaps) + return deleteByWlid(metadata, resourceMaps) } diff --git a/pkg/cleanup/discovery.go b/pkg/cleanup/discovery.go index b05e2f25f..3af681a98 100644 --- a/pkg/cleanup/discovery.go +++ b/pkg/cleanup/discovery.go @@ -6,6 +6,9 @@ import ( "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/pager" "k8s.io/client-go/discovery" @@ -98,11 +101,10 @@ func (h *KubernetesAPI) fetchWlidsFromRunningWorkloads(resourceMaps *ResourceMap return fmt.Errorf("failed to get group version resource for %s: %w", resource, err) } - workloads, err := h.Client.Resource(gvr).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return fmt.Errorf("failed to list %s: %w", gvr, err) - } - for _, workload := range workloads.Items { + if err := pager.New(func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + return h.Client.Resource(gvr).List(ctx, opts) + }).EachListItem(context.Background(), metav1.ListOptions{}, func(obj runtime.Object) error { + workload := obj.(*unstructured.Unstructured) // we don't care about the cluster name, so we remove it to avoid corner cases wlid := wlidPkg.GetK8sWLID("", workload.GetNamespace(), workload.GetKind(), workload.GetName()) wlid = wlidWithoutClusterName(wlid) @@ -111,7 +113,7 @@ func (h *KubernetesAPI) fetchWlidsFromRunningWorkloads(resourceMaps *ResourceMap c, ok := workloadinterface.InspectMap(workload.Object, append(workloadinterface.PodSpec(workload.GetKind()), "containers")...) if !ok { - continue + return nil } containers := c.([]interface{}) for _, container := range containers { @@ -126,7 +128,7 @@ func (h *KubernetesAPI) fetchWlidsFromRunningWorkloads(resourceMaps *ResourceMap initC, ok := workloadinterface.InspectMap(workload.Object, append(workloadinterface.PodSpec(workload.GetKind()), "initContainers")...) if !ok { - continue + return nil } initContainers := initC.([]interface{}) for _, container := range initContainers { @@ -141,7 +143,7 @@ func (h *KubernetesAPI) fetchWlidsFromRunningWorkloads(resourceMaps *ResourceMap ephemeralC, ok := workloadinterface.InspectMap(workload.Object, append(workloadinterface.PodSpec(workload.GetKind()), "ephemeralContainers")...) if !ok { - continue + return nil } ephemralContainers := ephemeralC.([]interface{}) for _, container := range ephemralContainers { @@ -153,21 +155,21 @@ func (h *KubernetesAPI) fetchWlidsFromRunningWorkloads(resourceMaps *ResourceMap nameStr := name.(string) resourceMaps.RunningWlidsToContainerNames.Get(wlid).Add(nameStr) } - + return nil + }); err != nil { + return fmt.Errorf("failed to list %s: %w", gvr, err) } } return nil } func (h *KubernetesAPI) fetchInstanceIdsAndImageIdsAndReplicasFromRunningPods(resourceMaps *ResourceMaps) error { - pods, err := h.Client.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}).List(context.TODO(), metav1.ListOptions{ + if err := pager.New(func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + return h.Client.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}).List(ctx, opts) + }).EachListItem(context.Background(), metav1.ListOptions{ FieldSelector: "status.phase=Running", - }) - if err != nil { - return fmt.Errorf("failed to list pods: %w", err) - } - - for _, p := range pods.Items { + }, func(obj runtime.Object) error { + p := obj.(*unstructured.Unstructured) pod := workloadinterface.NewWorkloadObj(p.Object) if replicaHash, ok := pod.GetLabel("pod-template-hash"); ok { @@ -184,7 +186,7 @@ func (h *KubernetesAPI) fetchInstanceIdsAndImageIdsAndReplicasFromRunningPods(re s, ok := workloadinterface.InspectMap(p.Object, "status", "containerStatuses") if !ok { - continue + return nil } containerStatuses := s.([]interface{}) for _, cs := range containerStatuses { @@ -198,7 +200,7 @@ func (h *KubernetesAPI) fetchInstanceIdsAndImageIdsAndReplicasFromRunningPods(re initC, ok := workloadinterface.InspectMap(p.Object, "status", "initContainerStatuses") if !ok { - continue + return nil } initContainers := initC.([]interface{}) for _, cs := range initContainers { @@ -212,7 +214,7 @@ func (h *KubernetesAPI) fetchInstanceIdsAndImageIdsAndReplicasFromRunningPods(re ephemeralC, ok := workloadinterface.InspectMap(p.Object, "status", "ephemeralContainerStatuses") if !ok { - continue + return nil } ephemeralContainers := ephemeralC.([]interface{}) for _, cs := range ephemeralContainers { @@ -223,6 +225,9 @@ func (h *KubernetesAPI) fetchInstanceIdsAndImageIdsAndReplicasFromRunningPods(re imageIdStr := containerImageId.(string) resourceMaps.RunningContainerImageIds.Add(imageIdStr) } + return nil + }); err != nil { + return fmt.Errorf("failed to list pods: %w", err) } return nil } diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index 9ba0a5831..d669380bf 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -22,6 +22,7 @@ import ( "net" "net/http" "net/http/pprof" + "time" "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" @@ -32,6 +33,8 @@ import ( informers "github.com/kubescape/storage/pkg/generated/informers/externalversions" sampleopenapi "github.com/kubescape/storage/pkg/generated/openapi" "github.com/spf13/cobra" + "github.com/victorspringer/http-cache" + "github.com/victorspringer/http-cache/adapter/memory" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/endpoints/openapi" @@ -180,6 +183,26 @@ func (o WardleServerOptions) RunWardleServer(stopCh <-chan struct{}) error { return err } + // add http-cache + memcached, err := memory.NewAdapter( + memory.AdapterWithAlgorithm(memory.LRU), + memory.AdapterWithStorageCapacity(500000000), // 500MB + ) + if err != nil { + logger.L().Fatal("failed to create memcached adapter", helpers.Error(err)) + } + + cacheClient, err := cache.NewClient( + cache.ClientWithAdapter(memcached), + cache.ClientWithTTL(10*time.Minute), + cache.ClientWithRefreshKey("opn"), + ) + if err != nil { + logger.L().Fatal("failed to create cache client", helpers.Error(err)) + } + fullHandlerChain := server.GenericAPIServer.Handler.FullHandlerChain + server.GenericAPIServer.Handler.FullHandlerChain = cacheClient.Middleware(fullHandlerChain) + server.GenericAPIServer.AddPostStartHookOrDie("start-sample-server-informers", func(context genericapiserver.PostStartHookContext) error { config.GenericConfig.SharedInformerFactory.Start(context.StopCh) o.SharedInformerFactory.Start(context.StopCh)