From d475c8f52990febfa5781afc558b79d0215ecaf6 Mon Sep 17 00:00:00 2001 From: Devansh Jain Date: Wed, 15 Feb 2023 15:24:00 +0530 Subject: [PATCH 1/2] add s3 interface for s3 client mocking --- metadata/pkg/drivers/aws/aws.go | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/metadata/pkg/drivers/aws/aws.go b/metadata/pkg/drivers/aws/aws.go index 4fdaa63e6..4b4a068db 100755 --- a/metadata/pkg/drivers/aws/aws.go +++ b/metadata/pkg/drivers/aws/aws.go @@ -23,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" backendpb "github.com/opensds/multi-cloud/backend/proto" "github.com/opensds/multi-cloud/metadata/pkg/db" "github.com/opensds/multi-cloud/metadata/pkg/model" @@ -50,8 +51,7 @@ type AwsAdapter struct { Session *session.Session } -func GetHeadObject(sess *session.Session, bucketName *string, obj *model.MetaObject) { - svc := s3.New(sess) +func GetHeadObject(svc s3iface.S3API, bucketName *string, obj *model.MetaObject) { meta, err := svc.HeadObject(&s3.HeadObjectInput{Bucket: bucketName, Key: &obj.ObjectName}) if err != nil { log.Errorf("cannot perform head object on object %v in bucket %v. failed with error: %v", obj.ObjectName, *bucketName, err) @@ -60,7 +60,9 @@ func GetHeadObject(sess *session.Session, bucketName *string, obj *model.MetaObj if meta.ServerSideEncryption != nil { obj.ServerSideEncryption = *meta.ServerSideEncryption } - obj.ObjectType = *meta.ContentType + if meta.ContentType != nil { + obj.ObjectType = *meta.ContentType + } if meta.Expires != nil { expiresTime, err := time.Parse(time.RFC3339, *meta.Expires) if err != nil { @@ -82,8 +84,7 @@ func GetHeadObject(sess *session.Session, bucketName *string, obj *model.MetaObj obj.Metadata = metadata } -func ObjectList(sess *session.Session, bucket *model.MetaBucket) error { - svc := s3.New(sess) +func ObjectList(svc s3iface.S3API, bucket *model.MetaBucket) error { output, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket.Name}) if err != nil { log.Errorf("unable to list objects in bucket %v. failed with error: %v", bucket.Name, err) @@ -128,7 +129,7 @@ func ObjectList(sess *session.Session, bucket *model.MetaBucket) error { obj.ObjectAcl = access } - GetHeadObject(sess, &bucket.Name, obj) + GetHeadObject(svc, &bucket.Name, obj) } bucket.NumberOfObjects = numObjects bucket.TotalSize = totSize @@ -136,17 +137,16 @@ func ObjectList(sess *session.Session, bucket *model.MetaBucket) error { return nil } -func GetBucketMeta(buckIdx int, bucket *s3.Bucket, sess *session.Session, bucketArray []*model.MetaBucket, wg *sync.WaitGroup) { +func GetBucketMeta(buckIdx int, bucket *s3.Bucket, svc s3iface.S3API, backendRegion *string, bucketArray []*model.MetaBucket, wg *sync.WaitGroup) { defer wg.Done() - svc := s3.New(sess) loc, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: bucket.Name}) if err != nil { log.Errorf("unable to get bucket location. failed with error: %v", err) return } - if *loc.LocationConstraint != *sess.Config.Region { + if *loc.LocationConstraint != *backendRegion { return } @@ -155,7 +155,7 @@ func GetBucketMeta(buckIdx int, bucket *s3.Bucket, sess *session.Session, bucket buck.CreationDate = bucket.CreationDate buck.Region = *loc.LocationConstraint - err = ObjectList(sess, buck) + err = ObjectList(svc, buck) if err != nil { return } @@ -164,13 +164,13 @@ func GetBucketMeta(buckIdx int, bucket *s3.Bucket, sess *session.Session, bucket tags, err := svc.GetBucketTagging(&s3.GetBucketTaggingInput{Bucket: bucket.Name}) - if err == nil { + if err == nil || strings.Contains(err.Error(), "NoSuchTagSet") { tagset := map[string]string{} for _, tag := range tags.TagSet { tagset[*tag.Key] = *tag.Value } buck.BucketTags = tagset - } else if !strings.Contains(err.Error(), "NoSuchTagSet") { + } else { log.Errorf("unable to get bucket tags. failed with error: %v", err) } @@ -186,8 +186,7 @@ func GetBucketMeta(buckIdx int, bucket *s3.Bucket, sess *session.Session, bucket } } -func BucketList(sess *session.Session) ([]*model.MetaBucket, error) { - svc := s3.New(sess) +func BucketList(svc s3iface.S3API, backendRegion *string) ([]*model.MetaBucket, error) { output, err := svc.ListBuckets(&s3.ListBucketsInput{}) if err != nil { @@ -198,7 +197,7 @@ func BucketList(sess *session.Session) ([]*model.MetaBucket, error) { wg := sync.WaitGroup{} for idx, bucket := range output.Buckets { wg.Add(1) - go GetBucketMeta(idx, bucket, sess, bucketArray, &wg) + go GetBucketMeta(idx, bucket, svc, backendRegion, bucketArray, &wg) } wg.Wait() @@ -213,8 +212,9 @@ func BucketList(sess *session.Session) ([]*model.MetaBucket, error) { } func (ad *AwsAdapter) SyncMetadata(ctx context.Context, in *pb.SyncMetadataRequest) error { - - buckArr, err := BucketList(ad.Session) + svc := s3.New(ad.Session) + backendRegion := ad.Session.Config.Region + buckArr, err := BucketList(svc, backendRegion) if err != nil { log.Errorf("metadata collection for backend id: %v failed with error: %v", ad.Backend.Id, err) return err From 1c3487c9c36c9012667f2090e24b7da8e6885ed0 Mon Sep 17 00:00:00 2001 From: Devansh Jain Date: Mon, 20 Feb 2023 23:53:50 +0530 Subject: [PATCH 2/2] refactor ObjectList function --- metadata/pkg/drivers/aws/aws.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/metadata/pkg/drivers/aws/aws.go b/metadata/pkg/drivers/aws/aws.go index 4b4a068db..90138e158 100755 --- a/metadata/pkg/drivers/aws/aws.go +++ b/metadata/pkg/drivers/aws/aws.go @@ -84,11 +84,11 @@ func GetHeadObject(svc s3iface.S3API, bucketName *string, obj *model.MetaObject) obj.Metadata = metadata } -func ObjectList(svc s3iface.S3API, bucket *model.MetaBucket) error { - output, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket.Name}) +var ObjectList = func(svc s3iface.S3API, bucketName string) ([]*model.MetaObject, int64, error) { + output, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucketName}) if err != nil { - log.Errorf("unable to list objects in bucket %v. failed with error: %v", bucket.Name, err) - return err + log.Errorf("unable to list objects in bucket %v. failed with error: %v", bucketName, err) + return nil, 0, err } numObjects := len(output.Contents) @@ -103,7 +103,7 @@ func ObjectList(svc s3iface.S3API, bucket *model.MetaBucket) error { totSize += obj.Size obj.StorageClass = *object.StorageClass - tags, err := svc.GetObjectTagging(&s3.GetObjectTaggingInput{Bucket: &bucket.Name, Key: &obj.ObjectName}) + tags, err := svc.GetObjectTagging(&s3.GetObjectTaggingInput{Bucket: &bucketName, Key: &obj.ObjectName}) if err == nil { tagset := map[string]string{} @@ -118,7 +118,7 @@ func ObjectList(svc s3iface.S3API, bucket *model.MetaBucket) error { log.Errorf("unable to get object tags. failed with error: %v", err) } - acl, err := svc.GetObjectAcl(&s3.GetObjectAclInput{Bucket: &bucket.Name, Key: &obj.ObjectName}) + acl, err := svc.GetObjectAcl(&s3.GetObjectAclInput{Bucket: &bucketName, Key: &obj.ObjectName}) if err != nil { log.Errorf("unable to get object Acl. failed with error: %v", err) } else { @@ -129,12 +129,9 @@ func ObjectList(svc s3iface.S3API, bucket *model.MetaBucket) error { obj.ObjectAcl = access } - GetHeadObject(svc, &bucket.Name, obj) + GetHeadObject(svc, &bucketName, obj) } - bucket.NumberOfObjects = numObjects - bucket.TotalSize = totSize - bucket.Objects = objectArray - return nil + return objectArray, totSize, nil } func GetBucketMeta(buckIdx int, bucket *s3.Bucket, svc s3iface.S3API, backendRegion *string, bucketArray []*model.MetaBucket, wg *sync.WaitGroup) { @@ -155,11 +152,16 @@ func GetBucketMeta(buckIdx int, bucket *s3.Bucket, svc s3iface.S3API, backendReg buck.CreationDate = bucket.CreationDate buck.Region = *loc.LocationConstraint - err = ObjectList(svc, buck) + objectArray, totalSize, err := ObjectList(svc, *bucket.Name) + if err != nil { return } + buck.Objects = objectArray + buck.NumberOfObjects = len(objectArray) + buck.TotalSize = totalSize + bucketArray[buckIdx] = buck tags, err := svc.GetBucketTagging(&s3.GetBucketTaggingInput{Bucket: bucket.Name})