wip add presave processor to deduplicate profile info

Signed-off-by: Matthias Bertschy <[email protected]>

build/Dockerfile

@@ -9,7 +9,7 @@ RUN --mount=target=. \
     --mount=type=cache,target=/go/pkg \
     GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /out/storage .
 
-FROM gcr.io/distroless/static-debian11:nonroot
+FROM gcr.io/distroless/static-debian11:debug-nonroot
 
 COPY --from=builder /out/storage /usr/bin/storage
 

pkg/apis/softwarecomposition/types.go

@@ -18,6 +18,7 @@ package softwarecomposition
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"strings"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -314,11 +315,37 @@ type ExecCalls struct {
 	Envs []string
 }
 
+const sep = "␟"
+
+func (e ExecCalls) String() string {
+	s := strings.Builder{}
+	s.WriteString(e.Path)
+	for _, arg := range e.Args {
+		s.WriteString(sep)
+		s.WriteString(arg)
+	}
+	for _, env := range e.Envs {
+		s.WriteString(sep)
+		s.WriteString(env)
+	}
+	return s.String()
+}
+
 type OpenCalls struct {
 	Path  string
 	Flags []string
 }
 
+func (e OpenCalls) String() string {
+	s := strings.Builder{}
+	s.WriteString(e.Path)
+	for _, arg := range e.Flags {
+		s.WriteString(sep)
+		s.WriteString(arg)
+	}
+	return s.String()
+}
+
 type ApplicationProfileStatus struct {
 }
 

pkg/apiserver/apiserver.go

@@ -149,6 +149,7 @@ func (c completedConfig) New() (*WardleServer, error) {
 
 	storageImpl := file.NewStorageImpl(osFs, file.DefaultStorageRoot)
 
+	applicationProfileStorageImpl := file.NewStorageImplWithCollector(osFs, file.DefaultStorageRoot, &file.ApplicationProfileProcessor{})
 	configScanStorageImpl := file.NewConfigurationScanSummaryStorage(&storageImpl)
 	vulnerabilitySummaryStorage := file.NewVulnerabilitySummaryStorage(&storageImpl)
 	generatedNetworkPolicyStorage := file.NewGeneratedNetworkPolicyStorage(&storageImpl)
@@ -168,7 +169,7 @@ func (c completedConfig) New() (*WardleServer, error) {
 	v1beta1storage["configurationscansummaries"] = sbomregistry.RESTInPeace(configurationscansummary.NewREST(Scheme, configScanStorageImpl, c.GenericConfig.RESTOptionsGetter))
 	v1beta1storage["vulnerabilitysummaries"] = sbomregistry.RESTInPeace(vsumstorage.NewREST(Scheme, vulnerabilitySummaryStorage, c.GenericConfig.RESTOptionsGetter))
 
-	v1beta1storage["applicationprofiles"] = sbomregistry.RESTInPeace(applicationprofile.NewREST(Scheme, storageImpl, c.GenericConfig.RESTOptionsGetter))
+	v1beta1storage["applicationprofiles"] = sbomregistry.RESTInPeace(applicationprofile.NewREST(Scheme, applicationProfileStorageImpl, c.GenericConfig.RESTOptionsGetter))
 	v1beta1storage["applicationprofilesummaries"] = sbomregistry.RESTInPeace(applicationprofilesummary.NewREST(Scheme, storageImpl, c.GenericConfig.RESTOptionsGetter))
 	v1beta1storage["applicationactivities"] = sbomregistry.RESTInPeace(applicationactivity.NewREST(Scheme, storageImpl, c.GenericConfig.RESTOptionsGetter))
 

pkg/registry/file/processor.go

@@ -0,0 +1,64 @@
+package file
+
+import (
+	"fmt"
+	sets "github.com/deckarep/golang-set/v2"
+	"github.com/kubescape/storage/pkg/apis/softwarecomposition"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+type Processor interface {
+	PreSave(object runtime.Object) error
+}
+
+type DefaultProcessor struct {
+}
+
+var _ Processor = (*DefaultProcessor)(nil)
+
+func (d DefaultProcessor) PreSave(_ runtime.Object) error {
+	return nil
+}
+
+type ApplicationProfileProcessor struct {
+}
+
+var _ Processor = (*ApplicationProfileProcessor)(nil)
+
+func (a ApplicationProfileProcessor) PreSave(object runtime.Object) error {
+	profile, ok := object.(*softwarecomposition.ApplicationProfile)
+	if !ok {
+		return fmt.Errorf("given object is not an ApplicationProfile")
+	}
+	for i, container := range profile.Spec.Containers {
+		profile.Spec.Containers[i] = deflate(container)
+	}
+	return nil
+}
+
+func deflate(container softwarecomposition.ApplicationProfileContainer) softwarecomposition.ApplicationProfileContainer {
+	return softwarecomposition.ApplicationProfileContainer{
+		Name:         container.Name,
+		Capabilities: sets.NewThreadUnsafeSet(container.Capabilities...).ToSlice(),
+		Execs:        deflateStringer(container.Execs),
+		Opens:        deflateStringer(container.Opens),
+		Syscalls:     sets.NewThreadUnsafeSet(container.Syscalls...).ToSlice(),
+	}
+}
+
+type Stringer interface {
+	String() string
+}
+
+func deflateStringer[T Stringer](in []T) []T {
+	var out []T
+	set := sets.NewSet[string]()
+	for _, item := range in {
+		if set.Contains(item.String()) {
+			continue
+		}
+		set.Add(item.String())
+		out = append(out, item)
+	}
+	return out
+}

pkg/registry/file/storage.go

@@ -41,10 +41,11 @@ type objState struct {
 // hides all the storage-related operations behind it.
 type StorageImpl struct {
 	appFs           afero.Fs
-	watchDispatcher watchDispatcher
 	locks           *Mutex[string]
+	processor       Processor
 	root            string
 	versioner       storage.Versioner
+	watchDispatcher watchDispatcher
 }
 
 // StorageQuerier wraps the storage.Interface and adds some extra methods which are used by the storage implementation.
@@ -62,10 +63,22 @@ var _ StorageQuerier = &StorageImpl{}
 func NewStorageImpl(appFs afero.Fs, root string) StorageQuerier {
 	return &StorageImpl{
 		appFs:           appFs,
+		locks:           NewMapMutex[string](),
+		processor:       DefaultProcessor{},
+		root:            root,
+		versioner:       storage.APIObjectVersioner{},
 		watchDispatcher: newWatchDispatcher(),
+	}
+}
+
+func NewStorageImplWithCollector(appFs afero.Fs, root string, processor Processor) StorageQuerier {
+	return &StorageImpl{
+		appFs:           appFs,
 		locks:           NewMapMutex[string](),
+		processor:       processor,
 		root:            root,
 		versioner:       storage.APIObjectVersioner{},
+		watchDispatcher: newWatchDispatcher(),
 	}
 }
 
@@ -106,6 +119,10 @@ func isPayloadFile(path string) bool {
 }
 
 func (s *StorageImpl) writeFiles(key string, obj runtime.Object, metaOut runtime.Object) error {
+	// call processor on object to be saved
+	if err := s.processor.PreSave(obj); err != nil {
+		return fmt.Errorf("processor.PreSave: %w", err)
+	}
 	// set resourceversion
 	if version, _ := s.versioner.ObjectResourceVersion(obj); version == 0 {
 		if err := s.versioner.UpdateObject(obj, 1); err != nil {
@@ -470,7 +487,7 @@ func (s *StorageImpl) GuaranteedUpdate(
 		if err != nil {
 			// If our data is already up-to-date, return the error
 			if origStateIsCurrent {
-				if !apierrors.IsNotFound(err) {
+				if !apierrors.IsNotFound(err) && !apierrors.IsInvalid(err) {
 					logger.L().Ctx(ctx).Error("tryUpdate func failed", helpers.Error(err), helpers.String("key", key))
 				}
 				return err
@@ -491,7 +508,7 @@ func (s *StorageImpl) GuaranteedUpdate(
 
 			// it turns out our cached data was not stale, return the error
 			if cachedRev == origState.rev {
-				if !apierrors.IsNotFound(err) {
+				if !apierrors.IsNotFound(err) && !apierrors.IsInvalid(err) {
 					logger.L().Ctx(ctx).Error("tryUpdate func failed", helpers.Error(err), helpers.String("key", key))
 				}
 				return cachedUpdateErr