Merge pull request #133 from opensds/development

Merge development into master branch for publishing Capri RC-01

Gopkg.toml

@@ -63,7 +63,7 @@
 
 [[constraint]]
   name = "github.com/golang/protobuf"
-  version = "1.2.0"
+  version = "1.3.1"
 
 [[constraint]]
   branch = "master"

api/cmd/main.go

@@ -49,12 +49,12 @@ func main() {
 	ws.Filter(auth.FilterFactory())
 
 	backend.RegisterRouter(ws)
-	//s3.RegisterRouter(ws)
 	dataflow.RegisterRouter(ws)
 	// add filter for authentication context
-	wc.Filter(logging.FilterFactory())
-	wc.Filter(context.FilterFactory())
-
+	ws.Filter(logging.FilterFactory())
+	ws.Filter(context.FilterFactory())
+	ws.Filter(auth.FilterFactory())
+
 	s3ws := new(restful.WebService)
 	s3ws.Path("/v1/s3")
 	s3ws.Doc("OpenSDS Multi-Cloud API")

api/pkg/examples/policy.json

@@ -30,5 +30,9 @@
   "object:put": "",
   "object:delete": "",
   "routbucket:put": "",
-  "routobject:put": ""
+  "routobject:put": "",
+  "storageclass:get":"",
+  "bucketlifecycle:put": "",
+  "bucketlifecycle:get": "",
+  "bucketlifecycle:delete": ""
 }

api/pkg/s3/abortmultipartupload.go

@@ -43,6 +43,10 @@ func (s *APIService) AbortMultipartUpload(request *restful.Request, response *re
 		return
 	}
 
+	// delete multipart upload record, if delete failed, it will be cleaned by lifecycle management
+	record := s3.MultipartUploadRecord{ObjectKey: objectKey, Bucket: bucketName, UploadId: uploadId}
+	s.s3Client.DeleteUploadRecord(context.Background(), &record)
+
 	deleteInput := s3.DeleteObjectInput{Key: objectKey, Bucket: bucketName}
 	res, err := s.s3Client.DeleteObject(ctx, &deleteInput)
 	if err != nil {

api/pkg/s3/bucketlifecycledelete.go

@@ -0,0 +1,73 @@
+// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/emicklei/go-restful"
+	"github.com/micro/go-log"
+	"github.com/opensds/multi-cloud/api/pkg/policy"
+	s3 "github.com/opensds/multi-cloud/s3/proto"
+	"golang.org/x/net/context"
+)
+
+func (s *APIService) BucketLifecycleDelete(request *restful.Request, response *restful.Response) {
+	if !policy.Authorize(request, response, "bucket:delete") {
+		return
+	}
+	//var foundID int
+	FoundIDArray := []string{}
+	NonFoundIDArray := []string{}
+	bucketName := request.PathParameter("bucketName")
+	ruleID := request.Request.URL.Query()["ruleID"]
+	if ruleID != nil {
+		ctx := context.Background()
+		log.Logf("Received request for bucket lifecycle delete for bucket: %s and the ruleID: %s", bucketName, ruleID)
+		bucket, _ := s.s3Client.GetBucket(ctx, &s3.Bucket{Name: bucketName})
+		for _, id := range ruleID {
+			isfound := false
+			for _, lcRule := range bucket.LifecycleConfiguration {
+				if lcRule.ID == id {
+					isfound = true
+					FoundIDArray = append(FoundIDArray, id)
+				}
+			}
+			if !isfound {
+				NonFoundIDArray = append(NonFoundIDArray, id)
+			}
+		}
+		for _, id := range NonFoundIDArray {
+			response.WriteErrorString(http.StatusBadRequest, strings.Replace("error: rule ID $1 doesn't exist \n\n", "$1", id, 1))
+		}
+
+		for _, id := range FoundIDArray {
+			deleteInput := s3.DeleteLifecycleInput{Bucket: bucketName, RuleID: id}
+			res, err := s.s3Client.DeleteBucketLifecycle(ctx, &deleteInput)
+			if err != nil {
+				response.WriteError(http.StatusBadRequest, err)
+				return
+			}
+			response.WriteEntity(res)
+		}
+
+	} else {
+		response.WriteErrorString(http.StatusBadRequest, NoRuleIDForLifecycleDelete)
+		return
+	}
+	log.Log("Delete bucket lifecycle successfully.")
+}
+

api/pkg/s3/bucketlifecycleget.go

@@ -0,0 +1,124 @@
+// Copyright 2019 The OpenSDS Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+	"fmt"
+
+	"github.com/emicklei/go-restful"
+	"github.com/micro/go-log"
+	"github.com/opensds/multi-cloud/api/pkg/policy"
+	. "github.com/opensds/multi-cloud/api/pkg/utils/constants"
+	"github.com/opensds/multi-cloud/s3/pkg/model"
+	s3 "github.com/opensds/multi-cloud/s3/proto"
+	"golang.org/x/net/context"
+)
+
+//Convert function from storage tier to storage class for XML format output
+func (s *APIService) tier2class(tier int32) (string, error) {
+	{
+		mutext.Lock()
+		defer mutext.Unlock()
+		if len(ClassAndTier) == 0 {
+			err := s.loadStorageClassDefinition()
+			if err != nil {
+				log.Logf("load storage classes failed: %v.\n", err)
+				return "", err
+			}
+		}
+	}
+	className := ""
+	for k, v := range ClassAndTier {
+		if v == tier {
+			className = k
+		}
+	}
+	if className == "" {
+		log.Logf("invalid tier: %d\n", tier)
+		return "", fmt.Errorf(InvalidTier)
+	}
+	return className, nil
+}
+
+//Function for GET Bucket Lifecycle API
+func (s *APIService) BucketLifecycleGet(request *restful.Request, response *restful.Response) {
+	if !policy.Authorize(request, response, "bucket:get") {
+		return
+	}
+	bucketName := request.PathParameter("bucketName")
+	log.Logf("Received request for bucket details in GET lifecycle: %s", bucketName)
+
+	ctx := context.Background()
+	bucket, _ := s.s3Client.GetBucket(ctx, &s3.Bucket{Name: bucketName})
+
+	// convert back to xml struct
+	getLifecycleConf := model.LifecycleConfiguration{}
+
+	// convert lifecycle rule to xml Rule
+	if bucket.LifecycleConfiguration != nil {
+		for _, lcRule := range bucket.LifecycleConfiguration {
+			xmlRule := model.Rule{}
+
+			xmlRule.Status = lcRule.Status
+			xmlRule.ID = lcRule.ID
+			xmlRule.Filter = converts3FilterToRuleFilter(lcRule.Filter)
+			xmlRule.AbortIncompleteMultipartUpload = converts3UploadToRuleUpload(lcRule.AbortIncompleteMultipartUpload)
+			xmlRule.Transition = make([]model.Transition, 0)
+
+			//Arranging the transition and expiration actions in XML
+			for _, action := range lcRule.Actions {
+				log.Logf("Action is : %v\n", action)
+
+				if action.Name == ActionNameTransition {
+					xmlTransition := model.Transition{}
+					xmlTransition.Days = action.Days
+					xmlTransition.Backend = action.Backend
+					className, err := s.tier2class(action.Tier)
+					if err == nil {
+						xmlTransition.StorageClass = className
+					}
+					xmlRule.Transition = append(xmlRule.Transition, xmlTransition)
+				}
+				if action.Name == ActionNameExpiration {
+					xmlExpiration := model.Expiration{}
+					xmlExpiration.Days = action.Days
+					xmlRule.Expiration = append(xmlRule.Expiration, xmlExpiration)
+				}
+			}
+			// append each xml rule to xml array
+			getLifecycleConf.Rule = append(getLifecycleConf.Rule, xmlRule)
+		}
+	}
+
+	// marshall the array back to xml format
+	response.WriteAsXml(getLifecycleConf)
+	log.Log("Get bucket lifecycle successfully.")
+}
+
+func converts3FilterToRuleFilter(filter *s3.LifecycleFilter) model.Filter {
+	retFilter := model.Filter{}
+	if filter != nil {
+		retFilter.Prefix = filter.Prefix
+	}
+	return retFilter
+}
+
+func converts3UploadToRuleUpload(upload *s3.AbortMultipartUpload) model.AbortIncompleteMultipartUpload {
+	retUpload := model.AbortIncompleteMultipartUpload{}
+	if upload != nil {
+		retUpload.DaysAfterInitiation = upload.DaysAfterInitiation
+	}
+	return retUpload
+}