From 5bb898e09625f06d03c6f5c7c4f9677e537d53fa Mon Sep 17 00:00:00 2001 From: bupd Date: Mon, 9 Dec 2024 15:47:40 +0530 Subject: [PATCH 1/2] add Harbor Satellite Replication adapter * This commit enables us to use harbor with harbor-satellite Signed-off-by: bupd --- src/controller/replication/flow/stage.go | 42 +++- .../job/impl/replication/replication.go | 2 + .../reg/adapter/harborsatellite/adapter.go | 238 ++++++++++++++++++ src/pkg/reg/manager.go | 2 + src/pkg/reg/model/registry.go | 1 + 5 files changed, 278 insertions(+), 7 deletions(-) create mode 100644 src/pkg/reg/adapter/harborsatellite/adapter.go diff --git a/src/controller/replication/flow/stage.go b/src/controller/replication/flow/stage.go index 3778767b42e..84c8977b803 100644 --- a/src/controller/replication/flow/stage.go +++ b/src/controller/replication/flow/stage.go @@ -73,7 +73,8 @@ func fetchResources(adapter adp.Adapter, policy *repctlmodel.Policy) ([]*model.R // assemble the source resources by filling the registry information func assembleSourceResources(resources []*model.Resource, - policy *repctlmodel.Policy) []*model.Resource { + policy *repctlmodel.Policy, +) []*model.Resource { for _, resource := range resources { resource.Registry = policy.SrcRegistry } @@ -83,25 +84,51 @@ func assembleSourceResources(resources []*model.Resource, // assemble the destination resources by filling the metadata, registry and override properties func assembleDestinationResources(resources []*model.Resource, - policy *repctlmodel.Policy, dstRepoComponentPathType string) ([]*model.Resource, error) { + policy *repctlmodel.Policy, dstRepoComponentPathType string, +) ([]*model.Resource, error) { var result []*model.Resource for _, resource := range resources { - name, err := replaceNamespace(resource.Metadata.Repository.Name, policy.DestNamespace, policy.DestNamespaceReplaceCount, dstRepoComponentPathType) - if err != nil { - return nil, err + var registry *model.Registry + var repositoryName string + var err error + + log.Debugf("assembling dest resources...") + + // Check condition to determine whether to assemble list or destination resources + if policy.DestRegistry.Type == "harbor-satellite" { + // Assemble list resources + registry = policy.SrcRegistry + repositoryName = resource.Metadata.Repository.Name + + if resource.ExtendedInfo == nil { + resource.ExtendedInfo = make(map[string]interface{}) + } + resource.ExtendedInfo["destinationURL"] = policy.DestRegistry.URL + resource.ExtendedInfo["groupName"] = policy.DestNamespace + } else { + // Assemble destination resources + registry = policy.DestRegistry + repositoryName, err = replaceNamespace(resource.Metadata.Repository.Name, policy.DestNamespace, policy.DestNamespaceReplaceCount, dstRepoComponentPathType) + if err != nil { + return nil, err + } } + + // Create the resource res := &model.Resource{ Type: resource.Type, - Registry: policy.DestRegistry, + Registry: registry, ExtendedInfo: resource.ExtendedInfo, Deleted: resource.Deleted, IsDeleteTag: resource.IsDeleteTag, Override: policy.Override, Skip: resource.Skip, } + + // Fill the resource metadata res.Metadata = &model.ResourceMetadata{ Repository: &model.Repository{ - Name: name, + Name: repositoryName, Metadata: resource.Metadata.Repository.Metadata, }, Vtags: resource.Metadata.Vtags, @@ -109,6 +136,7 @@ func assembleDestinationResources(resources []*model.Resource, } result = append(result, res) } + log.Debug("assemble the destination resources completed") return result, nil } diff --git a/src/jobservice/job/impl/replication/replication.go b/src/jobservice/job/impl/replication/replication.go index 453fa5e032b..f25dbb33785 100644 --- a/src/jobservice/job/impl/replication/replication.go +++ b/src/jobservice/job/impl/replication/replication.go @@ -41,6 +41,8 @@ import ( _ "github.com/goharbor/harbor/src/pkg/reg/adapter/googlegcr" // import harbor adapter _ "github.com/goharbor/harbor/src/pkg/reg/adapter/harbor" + // import harborSatellite adapter + _ "github.com/goharbor/harbor/src/pkg/reg/adapter/harborsatellite" // import huawei adapter _ "github.com/goharbor/harbor/src/pkg/reg/adapter/huawei" // import jfrog adapter diff --git a/src/pkg/reg/adapter/harborsatellite/adapter.go b/src/pkg/reg/adapter/harborsatellite/adapter.go new file mode 100644 index 00000000000..261a092104c --- /dev/null +++ b/src/pkg/reg/adapter/harborsatellite/adapter.go @@ -0,0 +1,238 @@ +package harborsatellite + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/docker/distribution" + "github.com/goharbor/harbor/src/common/secret" + "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/lib/errors" + regadapter "github.com/goharbor/harbor/src/pkg/reg/adapter" + "github.com/goharbor/harbor/src/pkg/reg/model" +) + +var ( + _ regadapter.Adapter = (*adapter)(nil) + _ regadapter.ArtifactRegistry = (*adapter)(nil) +) +var ErrNotImplemented = errors.New("not implemented") + +type Result struct { + Group string `json:"group"` + Registry string `json:"registry"` + Artifacts []Artifact `json:"artifacts"` +} + +type Artifact struct { + Repository string `json:"repository"` + Tags []string `json:"tag"` + Labels []string `json:"labels"` + Type string `json:"type"` + Digest string `json:"digest"` + Deleted bool `json:"deleted"` +} + +func init() { + err := regadapter.RegisterFactory(model.RegistryHarborSatellite, &factory{}) + if err != nil { + return + } +} + +type factory struct{} + +// Create ... +func (f *factory) Create(r *model.Registry) (regadapter.Adapter, error) { + return newAdapter(r) +} + +// AdapterPattern ... +func (f *factory) AdapterPattern() *model.AdapterPattern { + return nil +} + +type adapter struct { + httpClient *http.Client +} + +func (a adapter) RoundTrip(request *http.Request) (*http.Response, error) { + u, err := url.Parse(config.InternalCoreURL()) + if err != nil { + return nil, fmt.Errorf("unable to parse internal core url: %v", err) + } + + // replace request's host with core's address + request.Host = config.InternalCoreURL() + request.URL.Host = u.Host + + request.URL.Scheme = u.Scheme + // adds auth headers + _ = secret.AddToRequest(request, config.JobserviceSecret()) + + return a.httpClient.Do(request) +} + +func (a adapter) Info() (*model.RegistryInfo, error) { + return &model.RegistryInfo{}, nil +} + +func (a adapter) PrepareForPush(resources []*model.Resource) error { + var ( + artifacts []Artifact + registry *model.Registry + destinationURL string + groupName string + ) + + for _, r := range resources { + if r.Metadata == nil { + continue + } + if r.Metadata.Repository == nil { + continue + } + if r.Registry == nil { + continue + } + if r.ExtendedInfo == nil { + return fmt.Errorf("extended_info map is nil") + } + + if registry == nil { + registry = r.Registry + } + if destinationURL == "" { + destURL, ok := r.ExtendedInfo["destinationURL"].(string) + if ok { + destinationURL = destURL + } else { + return fmt.Errorf("destination_url not a string or missing") + } + } + + if groupName == "" { + grp, ok := r.ExtendedInfo["groupName"].(string) + if ok { + groupName = grp + } else { + return fmt.Errorf("groupName not a string or missing") + } + } + + for _, at := range r.Metadata.Artifacts { + artifacts = append(artifacts, Artifact{ + Repository: r.Metadata.Repository.Name, + Deleted: r.Deleted, + Tags: at.Tags, + Labels: at.Labels, + Type: at.Type, + Digest: at.Digest, + }) + } + } + + if registry == nil { + return fmt.Errorf("no registry information found") + } + + result := &Result{ + Group: groupName, + Registry: registry.URL, + Artifacts: artifacts, + } + + data, err := json.Marshal(result) + if err != nil { + return errors.Wrap(err, "failed to marshal result") + } + + // Create a POST request + req, err := http.NewRequest("POST", destinationURL, bytes.NewBuffer(data)) + if err != nil { + return fmt.Errorf("error creating request: %v", err) + } + + // Set the content type header + req.Header.Set("Content-Type", "application/json") + + // Send the request using http.Client + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("error sending request: %v", err) + } + defer resp.Body.Close() + + return nil +} + +func (a adapter) HealthCheck() (string, error) { + return model.Healthy, nil +} + +func (a adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) { + return nil, nil +} + +func (a adapter) ManifestExist(repository, reference string) (exist bool, desc *distribution.Descriptor, err error) { + return true, nil, nil +} + +func (a adapter) PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error) { + return nil, "", ErrNotImplemented +} + +func (a adapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) { + return "", nil +} + +func (a adapter) DeleteManifest(repository, reference string) error { + return ErrNotImplemented +} + +func (a adapter) BlobExist(repository, digest string) (exist bool, err error) { + return true, nil +} + +func (a adapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) { + return 0, nil, ErrNotImplemented +} + +func (a adapter) PullBlobChunk(repository, digest string, blobSize, start, end int64) (size int64, blob io.ReadCloser, err error) { + return 0, nil, ErrNotImplemented +} + +func (a adapter) PushBlobChunk(repository, digest string, size int64, chunk io.Reader, start, end int64, location string) (nextUploadLocation string, endRange int64, err error) { + return "", 0, ErrNotImplemented +} + +func (a adapter) PushBlob(repository, digest string, size int64, blob io.Reader) error { + return nil +} + +func (a adapter) MountBlob(srcRepository, digest, dstRepository string) (err error) { + return nil +} + +func (a adapter) CanBeMount(digest string) (mount bool, repository string, err error) { + return false, "", ErrNotImplemented +} + +func (a adapter) DeleteTag(repository, tag string) error { + return ErrNotImplemented +} + +func (a adapter) ListTags(repository string) (tags []string, err error) { + return nil, nil +} + +func newAdapter(_ *model.Registry) (regadapter.Adapter, error) { + return &adapter{ + httpClient: &http.Client{}, + }, nil +} diff --git a/src/pkg/reg/manager.go b/src/pkg/reg/manager.go index c1d37671d90..83aaea6aeb3 100644 --- a/src/pkg/reg/manager.go +++ b/src/pkg/reg/manager.go @@ -43,6 +43,8 @@ import ( _ "github.com/goharbor/harbor/src/pkg/reg/adapter/googlegcr" // register the Harbor adapter _ "github.com/goharbor/harbor/src/pkg/reg/adapter/harbor" + // register the HarborSatellite adapter + _ "github.com/goharbor/harbor/src/pkg/reg/adapter/harborsatellite" // register the huawei adapter _ "github.com/goharbor/harbor/src/pkg/reg/adapter/huawei" // register the Jfrog Artifactory adapter diff --git a/src/pkg/reg/model/registry.go b/src/pkg/reg/model/registry.go index 1b4f945d713..ee3e03f8c71 100644 --- a/src/pkg/reg/model/registry.go +++ b/src/pkg/reg/model/registry.go @@ -38,6 +38,7 @@ const ( RegistryTypeHelmHub = "helm-hub" RegistryTypeArtifactHub = "artifact-hub" + RegistryHarborSatellite = "harbor-satellite" FilterStyleTypeText = "input" FilterStyleTypeRadio = "radio" From 57b40515d478ff7f92f203dc675344d25b4f0d76 Mon Sep 17 00:00:00 2001 From: bupd Date: Mon, 9 Dec 2024 15:53:52 +0530 Subject: [PATCH 2/2] fix lint Signed-off-by: bupd --- .../reg/adapter/harborsatellite/adapter.go | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/pkg/reg/adapter/harborsatellite/adapter.go b/src/pkg/reg/adapter/harborsatellite/adapter.go index 261a092104c..c7cbd232ccf 100644 --- a/src/pkg/reg/adapter/harborsatellite/adapter.go +++ b/src/pkg/reg/adapter/harborsatellite/adapter.go @@ -1,3 +1,17 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package harborsatellite import ( @@ -9,6 +23,7 @@ import ( "net/url" "github.com/docker/distribution" + "github.com/goharbor/harbor/src/common/secret" "github.com/goharbor/harbor/src/lib/config" "github.com/goharbor/harbor/src/lib/errors" @@ -175,59 +190,59 @@ func (a adapter) HealthCheck() (string, error) { return model.Healthy, nil } -func (a adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) { +func (a adapter) FetchArtifacts(_ []*model.Filter) ([]*model.Resource, error) { return nil, nil } -func (a adapter) ManifestExist(repository, reference string) (exist bool, desc *distribution.Descriptor, err error) { +func (a adapter) ManifestExist(_, _ string) (exist bool, desc *distribution.Descriptor, err error) { return true, nil, nil } -func (a adapter) PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error) { +func (a adapter) PullManifest(_, _ string, _ ...string) (manifest distribution.Manifest, digest string, err error) { return nil, "", ErrNotImplemented } -func (a adapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) { +func (a adapter) PushManifest(_, _, _ string, _ []byte) (string, error) { return "", nil } -func (a adapter) DeleteManifest(repository, reference string) error { +func (a adapter) DeleteManifest(_, _ string) error { return ErrNotImplemented } -func (a adapter) BlobExist(repository, digest string) (exist bool, err error) { +func (a adapter) BlobExist(_, _ string) (exist bool, err error) { return true, nil } -func (a adapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) { +func (a adapter) PullBlob(_, _ string) (size int64, blob io.ReadCloser, err error) { return 0, nil, ErrNotImplemented } -func (a adapter) PullBlobChunk(repository, digest string, blobSize, start, end int64) (size int64, blob io.ReadCloser, err error) { +func (a adapter) PullBlobChunk(_, _ string, _, _, _ int64) (size int64, blob io.ReadCloser, err error) { return 0, nil, ErrNotImplemented } -func (a adapter) PushBlobChunk(repository, digest string, size int64, chunk io.Reader, start, end int64, location string) (nextUploadLocation string, endRange int64, err error) { +func (a adapter) PushBlobChunk(_, _ string, _ int64, _ io.Reader, _, _ int64, _ string) (nextUploadLocation string, endRange int64, err error) { return "", 0, ErrNotImplemented } -func (a adapter) PushBlob(repository, digest string, size int64, blob io.Reader) error { +func (a adapter) PushBlob(_, _ string, _ int64, _ io.Reader) error { return nil } -func (a adapter) MountBlob(srcRepository, digest, dstRepository string) (err error) { +func (a adapter) MountBlob(_, _, _ string) (err error) { return nil } -func (a adapter) CanBeMount(digest string) (mount bool, repository string, err error) { +func (a adapter) CanBeMount(_ string) (mount bool, repository string, err error) { return false, "", ErrNotImplemented } -func (a adapter) DeleteTag(repository, tag string) error { +func (a adapter) DeleteTag(_, _ string) error { return ErrNotImplemented } -func (a adapter) ListTags(repository string) (tags []string, err error) { +func (a adapter) ListTags(_ string) (tags []string, err error) { return nil, nil }