From 329fce1fef6f233601b1637e47f8341d8f0ecf6b Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Tue, 31 Oct 2023 13:04:55 -0600 Subject: [PATCH 01/33] chore: update gitea chart version --- packages/gitea/gitea-values.yaml | 10 +++++++++- packages/gitea/zarf.yaml | 2 +- zarf-config.toml | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/gitea/gitea-values.yaml b/packages/gitea/gitea-values.yaml index 8ff5ee2008..26619cab67 100644 --- a/packages/gitea/gitea-values.yaml +++ b/packages/gitea/gitea-values.yaml @@ -5,7 +5,7 @@ persistence: accessModes: - "###ZARF_VAR_GIT_SERVER_PVC_ACCESS_MODE###" -replicaCount: "###ZARF_VAR_GIT_SERVER_REPLICA_COUNT###" +replicaCount: ###ZARF_VAR_GIT_SERVER_REPLICA_COUNT### gitea: admin: @@ -29,6 +29,8 @@ gitea: repository: ENABLE_PUSH_CREATE_USER: true FORCE_PRIVATE: true + session: + PROVIDER: db resources: requests: cpu: "###ZARF_VAR_GIT_SERVER_CPU_REQ###" @@ -47,3 +49,9 @@ image: repository: "###ZARF_CONST_GITEA_IMAGE###" tag: "###ZARF_CONST_GITEA_SERVER_VERSION###" rootless: true + +postgresql-ha: + enabled: false + +redis-cluster: + enabled: false diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index 08652d3dac..0edece89ed 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -57,7 +57,7 @@ components: - name: gitea releaseName: zarf-gitea url: https://dl.gitea.io/charts - version: 8.3.0 + version: 9.5.1 namespace: zarf valuesFiles: - gitea-values.yaml diff --git a/zarf-config.toml b/zarf-config.toml index 5f0f1b5357..1551059d33 100644 --- a/zarf-config.toml +++ b/zarf-config.toml @@ -16,4 +16,4 @@ registry_image_tag = '2.8.3' # The image reference to use for the optional git-server Zarf deploys gitea_image = 'gitea/gitea' -gitea_server_version = '1.19.3' +gitea_server_version = '1.20.5' From 87a753a5ff40c8b0a482b4950453eb972047c535 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:30:48 -0600 Subject: [PATCH 02/33] wip: refactoring gitea-values and zarf.yaml --- packages/gitea/gitea-values.yaml | 10 ++-------- packages/gitea/zarf.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/gitea/gitea-values.yaml b/packages/gitea/gitea-values.yaml index 26619cab67..4321f5a0f2 100644 --- a/packages/gitea/gitea-values.yaml +++ b/packages/gitea/gitea-values.yaml @@ -1,11 +1,11 @@ persistence: storageClass: "###ZARF_STORAGE_CLASS###" - existingClaim: "###ZARF_VAR_GIT_SERVER_EXISTING_PVC###" + claimName: "###ZARF_VAR_GIT_SERVER_PVC_NAME###" size: "###ZARF_VAR_GIT_SERVER_PVC_SIZE###" accessModes: - "###ZARF_VAR_GIT_SERVER_PVC_ACCESS_MODE###" -replicaCount: ###ZARF_VAR_GIT_SERVER_REPLICA_COUNT### +replicaCount: 1.0 gitea: admin: @@ -39,12 +39,6 @@ resources: cpu: "###ZARF_VAR_GIT_SERVER_CPU_LIMIT###" memory: "###ZARF_VAR_GIT_SERVER_MEM_LIMIT###" -memcached: - enabled: false - -postgresql: - enabled: false - image: repository: "###ZARF_CONST_GITEA_IMAGE###" tag: "###ZARF_CONST_GITEA_SERVER_VERSION###" diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index 0edece89ed..4abad982a3 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -3,9 +3,9 @@ metadata: name: init-package-git-server variables: - - name: GIT_SERVER_EXISTING_PVC - description: "Optional: Use an existing PVC for the git server instead of creating a new one. If this is set, the GIT_SERVER_PVC_SIZE variable will be ignored." - default: "" + - name: GIT_SERVER_PVC_NAME + description: "Use this for an existing PVC for the git server, otherwise a new PVC will be made with the default name. If this is set, the GIT_SERVER_PVC_SIZE variable will be ignored." + default: "gitea-data" - name: GIT_SERVER_PVC_SIZE description: The size of the persistent volume claim for the git server @@ -33,7 +33,7 @@ variables: - name: GIT_SERVER_REPLICA_COUNT description: The number of git server replicas to deploy - default: "1" + default: 1.0 constants: - name: GITEA_IMAGE From 41de1bb316404b69199d2c39fabd2d6bc6606c53 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Thu, 2 Nov 2023 08:14:22 -0600 Subject: [PATCH 03/33] wip: set git-server replicaCount back to zarf var. --- packages/gitea/gitea-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gitea/gitea-values.yaml b/packages/gitea/gitea-values.yaml index 4321f5a0f2..54219f3961 100644 --- a/packages/gitea/gitea-values.yaml +++ b/packages/gitea/gitea-values.yaml @@ -5,7 +5,7 @@ persistence: accessModes: - "###ZARF_VAR_GIT_SERVER_PVC_ACCESS_MODE###" -replicaCount: 1.0 +replicaCount: ###ZARF_VAR_GIT_SERVER_REPLICA_COUNT### gitea: admin: From ba398ce65996a4a098f38b749865aeda9a366cf7 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:01:00 -0700 Subject: [PATCH 04/33] wip: update expected gitea version for create_sbom_test --- src/test/e2e/06_create_sbom_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/e2e/06_create_sbom_test.go b/src/test/e2e/06_create_sbom_test.go index 1af4adf739..a7a4c640d7 100644 --- a/src/test/e2e/06_create_sbom_test.go +++ b/src/test/e2e/06_create_sbom_test.go @@ -54,9 +54,9 @@ func TestCreateSBOM(t *testing.T) { _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "sbom-viewer-docker.io_defenseunicorns_zarf-game_multi-tile-dark.html")) require.NoError(t, err) // Test that the init package generates the SBOMs we expect (images + component files) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.19.3-rootless.html")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.20.5-rootless.html")) require.NoError(t, err) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.19.3-rootless.json")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.20.5-rootless.json")) require.NoError(t, err) _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-zarf-component-k3s.html")) require.NoError(t, err) From 14e477806f06bf6d4856d4d71338ed1be680e6ce Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Tue, 7 Nov 2023 06:02:39 -0700 Subject: [PATCH 05/33] wip: added scope to Artifact token to read:user --- src/internal/packager/git/gitea.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 609095f6ad..37cb55844f 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -178,6 +178,7 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { createTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens", tunnelURL, g.Server.PushUsername) createTokensBody := map[string]interface{}{ "name": config.ZarfArtifactTokenName, + "scopes": []string{"read:user"}, } createTokensData, _ := json.Marshal(createTokensBody) createTokensRequest, _ := netHttp.NewRequest("POST", createTokensEndpoint, bytes.NewBuffer(createTokensData)) From ab28c3c18c3ca6737cad69b2e0d8dcda6ade988f Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Wed, 8 Nov 2023 17:37:02 -0700 Subject: [PATCH 06/33] wip: updated gitea config values to better match recommended configuration. --- packages/gitea/gitea-values.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/gitea/gitea-values.yaml b/packages/gitea/gitea-values.yaml index 54219f3961..0957d0a53b 100644 --- a/packages/gitea/gitea-values.yaml +++ b/packages/gitea/gitea-values.yaml @@ -30,7 +30,11 @@ gitea: ENABLE_PUSH_CREATE_USER: true FORCE_PRIVATE: true session: - PROVIDER: db + PROVIDER: memory + cache: + ADAPTER: memory + queue: + TYPE: level resources: requests: cpu: "###ZARF_VAR_GIT_SERVER_CPU_REQ###" From 087d381b93b4109e6921710c7fde438659787dbf Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:48:09 -0700 Subject: [PATCH 07/33] wip: set existing server pvc name default to pvc name from 8.3.0; add label and annotate commands for helm to take over old 8.3.0 pvc --- packages/gitea/zarf.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index 96945282ee..36b7a34708 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -3,9 +3,10 @@ metadata: name: init-package-git-server variables: + #CHANGE BACK TO EXISTING CLAIM - name: GIT_SERVER_PVC_NAME description: "Use this for an existing PVC for the git server, otherwise a new PVC will be made with the default name. If this is set, the GIT_SERVER_PVC_SIZE variable will be ignored." - default: "gitea-data" + default: "data-zarf-gitea-0" - name: GIT_SERVER_PVC_SIZE description: The size of the persistent volume claim for the git server @@ -67,6 +68,10 @@ components: - gitea-values.yaml actions: onDeploy: + before: + - cmd: zarf tools kubectl label pvc -n zarf data-zarf-gitea-0 app.kubernetes.io/managed-by=Helm || true + - cmd: zarf tools kubectl annotate pvc -n zarf data-zarf-gitea-0 meta.helm.sh/release-name=zarf-gitea || true + - cmd: zarf tools kubectl annotate pvc -n zarf data-zarf-gitea-0 meta.helm.sh/release-namespace=zarf || true after: - cmd: ./zarf internal create-read-only-gitea-user --no-progress maxRetries: 3 From 97fcf40cbe00a25fcb86b47af90237337be3df3f Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:23:01 -0700 Subject: [PATCH 08/33] wip: iterating on updatig gitea creds via api instead of updated helm release --- src/internal/packager/git/gitea.go | 35 ++++++++++++++++++++++++++++ src/internal/packager/helm/zarf.go | 37 +++++++++++++++--------------- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 37cb55844f..f925d5968d 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -120,6 +120,41 @@ func (g *Git) CreateReadOnlyUser() error { return err } +func (g *Git) UpdatePushUserAuth() error { + message.Debugf("git.UpdatePushUserAuth()") + + c, err := cluster.NewCluster() + if err != nil { + return err + } + + // Establish a git tunnel to send the repo + tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, k8s.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort) + if err != nil { + return err + } + _, err = tunnel.Connect() + if err != nil { + return err + } + defer tunnel.Close() + + tunnelURL := tunnel.HTTPEndpoint() + + + // Make sure the user can't create their own repos or orgs + updateUserBody := map[string]interface{}{ + "login_name": g.Server.PushUsername, + "password": g.Server.PushPassword, + } + updateUserData, _ := json.Marshal(updateUserBody) + updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, g.Server.PushUsername) + updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) + out, err := g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, g.Server.PushPassword) + message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) + return err +} + // CreatePackageRegistryToken uses the Gitea API to create a package registry token. func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { message.Debugf("git.CreatePackageRegistryToken()") diff --git a/src/internal/packager/helm/zarf.go b/src/internal/packager/helm/zarf.go index 8503690ab1..5a6d668fb0 100644 --- a/src/internal/packager/helm/zarf.go +++ b/src/internal/packager/helm/zarf.go @@ -50,27 +50,26 @@ func (h *Helm) UpdateZarfRegistryValues() error { // UpdateZarfGiteaValues updates the Zarf git server deployment with the new state values func (h *Helm) UpdateZarfGiteaValues() error { - giteaValues := map[string]interface{}{ - "gitea": map[string]interface{}{ - "admin": map[string]interface{}{ - "username": h.Cfg.State.GitServer.PushUsername, - "password": h.Cfg.State.GitServer.PushPassword, - }, - }, - } - - h.Chart = types.ZarfChart{ - Namespace: "zarf", - } - h.ReleaseName = "zarf-gitea" - - err := h.UpdateReleaseValues(giteaValues) - if err != nil { - return fmt.Errorf("error updating the release values: %w", err) - } + // giteaValues := map[string]interface{}{ + // "gitea": map[string]interface{}{ + // "admin": map[string]interface{}{ + // "username": h.Cfg.State.GitServer.PushUsername, + // "password": h.Cfg.State.GitServer.PushPassword, + // }, + // }, + // } + + // err := h.UpdateReleaseValues(giteaValues) + // if err != nil { + // return fmt.Errorf("error updating the release values: %w", err) + // } g := git.New(h.Cfg.State.GitServer) - err = g.CreateReadOnlyUser() + // err := g.CreateReadOnlyUser() + + // _, err = g.CreatePackageRegistryToken() + + err := g.UpdatePushUserAuth() if err != nil { return fmt.Errorf("unable to create the new Gitea read only user: %w", err) } From 00cd24cbc21f25955d300786441623293842be32 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:04:49 -0700 Subject: [PATCH 09/33] wip: update-creds api calls refactors --- src/cmd/tools/zarf.go | 2 +- src/internal/packager/git/gitea.go | 59 ++++++++++++++++++++++-------- src/internal/packager/helm/zarf.go | 31 +++++----------- 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index dfd5f9e68a..3d244fd270 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -149,7 +149,7 @@ var updateCredsCmd = &cobra.Command{ } } if slices.Contains(args, message.GitKey) && newState.GitServer.InternalServer { - err = h.UpdateZarfGiteaValues() + err = h.UpdateZarfGiteaValues(oldState) if err != nil { // Warn if we couldn't actually update the git server (it might not be installed and we should try to continue) message.Warnf(lang.CmdToolsUpdateCredsUnableUpdateGit, err.Error()) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index f925d5968d..08cd88f787 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -72,16 +72,7 @@ func (g *Git) CreateReadOnlyUser() error { } if hasReadOnlyUser { - // Update the existing user's password - updateUserBody := map[string]interface{}{ - "login_name": g.Server.PullUsername, - "password": g.Server.PullPassword, - } - updateUserData, _ := json.Marshal(updateUserBody) - updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, g.Server.PullUsername) - updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) - out, err = g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, g.Server.PushPassword) - message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) + err = g.UpdateReadOnlyUser(g.Server.PushPassword, tunnelURL) return err } @@ -120,8 +111,46 @@ func (g *Git) CreateReadOnlyUser() error { return err } -func (g *Git) UpdatePushUserAuth() error { - message.Debugf("git.UpdatePushUserAuth()") +func (g *Git) UpdateReadOnlyUser(oldAdminPass string, tunnelURL string) error { + message.Debugf("git.UpdateReadOnlyUser()") + + if tunnelURL == "" { + c, err := cluster.NewCluster() + if err != nil { + return err + } + + // Establish a git tunnel to send the repo + tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, k8s.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort) + if err != nil { + return err + } + _, err = tunnel.Connect() + if err != nil { + return err + } + defer tunnel.Close() + + tunnelURL = tunnel.HTTPEndpoint() + } + + + + // Update the existing user's password + updateUserBody := map[string]interface{}{ + "login_name": g.Server.PullUsername, + "password": g.Server.PullPassword, + } + updateUserData, _ := json.Marshal(updateUserBody) + updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, g.Server.PullUsername) + updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) + out, err := g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, oldAdminPass) + message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) + return err +} + +func (g *Git) UpdatePushUser(oldAdminPass string) error { + message.Debugf("git.UpdatePushUser()") c, err := cluster.NewCluster() if err != nil { @@ -144,13 +173,13 @@ func (g *Git) UpdatePushUserAuth() error { // Make sure the user can't create their own repos or orgs updateUserBody := map[string]interface{}{ - "login_name": g.Server.PushUsername, - "password": g.Server.PushPassword, + "login_name": g.Server.PushUsername, + "password": g.Server.PushPassword, } updateUserData, _ := json.Marshal(updateUserBody) updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, g.Server.PushUsername) updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) - out, err := g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, g.Server.PushPassword) + out, err := g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, oldAdminPass) message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) return err } diff --git a/src/internal/packager/helm/zarf.go b/src/internal/packager/helm/zarf.go index 5a6d668fb0..8d330bfd28 100644 --- a/src/internal/packager/helm/zarf.go +++ b/src/internal/packager/helm/zarf.go @@ -49,31 +49,18 @@ func (h *Helm) UpdateZarfRegistryValues() error { } // UpdateZarfGiteaValues updates the Zarf git server deployment with the new state values -func (h *Helm) UpdateZarfGiteaValues() error { - // giteaValues := map[string]interface{}{ - // "gitea": map[string]interface{}{ - // "admin": map[string]interface{}{ - // "username": h.Cfg.State.GitServer.PushUsername, - // "password": h.Cfg.State.GitServer.PushPassword, - // }, - // }, - // } - - // err := h.UpdateReleaseValues(giteaValues) - // if err != nil { - // return fmt.Errorf("error updating the release values: %w", err) - // } - - g := git.New(h.Cfg.State.GitServer) - // err := g.CreateReadOnlyUser() - - // _, err = g.CreatePackageRegistryToken() - - err := g.UpdatePushUserAuth() +func (h *Helm) UpdateZarfGiteaValues(oldState *types.ZarfState) error { + oG := oldState.GitServer + nG := git.New(h.Cfg.State.GitServer) + err := nG.UpdateReadOnlyUser(oG.PushPassword, "") if err != nil { - return fmt.Errorf("unable to create the new Gitea read only user: %w", err) + return fmt.Errorf("unable to update gitea read only user password: %w", err) } + err = nG.UpdatePushUser(oG.PushPassword) + if err != nil { + return fmt.Errorf("unable to update gitea admin user password: %w", err) + } return nil } From 48558c6a9075d7838991d4e169e8778eec23303d Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 27 Nov 2023 11:19:06 -0700 Subject: [PATCH 10/33] wip: removed has read only user check from CreateReadOnlyUser() --- src/internal/packager/git/gitea.go | 66 ++++++++---------------------- src/internal/packager/helm/zarf.go | 2 +- 2 files changed, 18 insertions(+), 50 deletions(-) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 08cd88f787..c92e0daa60 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -49,33 +49,6 @@ func (g *Git) CreateReadOnlyUser() error { tunnelURL := tunnel.HTTPEndpoint() - // Determine if the read only user already exists - getUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users", tunnelURL) - getUserRequest, _ := netHttp.NewRequest("GET", getUserEndpoint, nil) - out, err := g.DoHTTPThings(getUserRequest, g.Server.PushUsername, g.Server.PushPassword) - message.Debugf("GET %s:\n%s", getUserEndpoint, string(out)) - if err != nil { - return err - } - - hasReadOnlyUser := false - var users []map[string]interface{} - err = json.Unmarshal(out, &users) - if err != nil { - return err - } - - for _, user := range users { - if user["login"] == g.Server.PullUsername { - hasReadOnlyUser = true - } - } - - if hasReadOnlyUser { - err = g.UpdateReadOnlyUser(g.Server.PushPassword, tunnelURL) - return err - } - // Create json representation of the create-user request body createUserBody := map[string]interface{}{ "username": g.Server.PullUsername, @@ -91,7 +64,7 @@ func (g *Git) CreateReadOnlyUser() error { // Send API request to create the user createUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users", tunnelURL) createUserRequest, _ := netHttp.NewRequest("POST", createUserEndpoint, bytes.NewBuffer(createUserData)) - out, err = g.DoHTTPThings(createUserRequest, g.Server.PushUsername, g.Server.PushPassword) + out, err := g.DoHTTPThings(createUserRequest, g.Server.PushUsername, g.Server.PushPassword) message.Debugf("POST %s:\n%s", createUserEndpoint, string(out)) if err != nil { return err @@ -111,30 +84,25 @@ func (g *Git) CreateReadOnlyUser() error { return err } -func (g *Git) UpdateReadOnlyUser(oldAdminPass string, tunnelURL string) error { +func (g *Git) UpdateReadOnlyUser(oldAdminPass string) error { message.Debugf("git.UpdateReadOnlyUser()") - if tunnelURL == "" { - c, err := cluster.NewCluster() - if err != nil { - return err - } - - // Establish a git tunnel to send the repo - tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, k8s.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort) - if err != nil { - return err - } - _, err = tunnel.Connect() - if err != nil { - return err - } - defer tunnel.Close() - - tunnelURL = tunnel.HTTPEndpoint() - } - + c, err := cluster.NewCluster() + if err != nil { + return err + } + // Establish a git tunnel to send the repo + tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, k8s.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort) + if err != nil { + return err + } + _, err = tunnel.Connect() + if err != nil { + return err + } + defer tunnel.Close() + tunnelURL := tunnel.HTTPEndpoint() // Update the existing user's password updateUserBody := map[string]interface{}{ diff --git a/src/internal/packager/helm/zarf.go b/src/internal/packager/helm/zarf.go index 8d330bfd28..7fcc9f5354 100644 --- a/src/internal/packager/helm/zarf.go +++ b/src/internal/packager/helm/zarf.go @@ -52,7 +52,7 @@ func (h *Helm) UpdateZarfRegistryValues() error { func (h *Helm) UpdateZarfGiteaValues(oldState *types.ZarfState) error { oG := oldState.GitServer nG := git.New(h.Cfg.State.GitServer) - err := nG.UpdateReadOnlyUser(oG.PushPassword, "") + err := nG.UpdateReadOnlyUser(oG.PushPassword) if err != nil { return fmt.Errorf("unable to update gitea read only user password: %w", err) } From b70d100977714f27d6ae7232d712a8d7a2d58e50 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 27 Nov 2023 12:31:33 -0700 Subject: [PATCH 11/33] wip: revert zarf var for gitea claim name to git_server_existing_pvc --- packages/gitea/gitea-values.yaml | 2 +- packages/gitea/zarf.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/gitea/gitea-values.yaml b/packages/gitea/gitea-values.yaml index 313c78d89a..149f338849 100644 --- a/packages/gitea/gitea-values.yaml +++ b/packages/gitea/gitea-values.yaml @@ -1,6 +1,6 @@ persistence: storageClass: "###ZARF_STORAGE_CLASS###" - claimName: "###ZARF_VAR_GIT_SERVER_PVC_NAME###" + claimName: "###ZARF_VAR_GIT_SERVER_EXISTING_PVC###" size: "###ZARF_VAR_GIT_SERVER_PVC_SIZE###" accessModes: - "###ZARF_VAR_GIT_SERVER_PVC_ACCESS_MODE###" diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index f7a6f46b1e..91c032731d 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -4,8 +4,8 @@ metadata: variables: #CHANGE BACK TO EXISTING CLAIM - - name: GIT_SERVER_PVC_NAME - description: "Use this for an existing PVC for the git server, otherwise a new PVC will be made with the default name. If this is set, the GIT_SERVER_PVC_SIZE variable will be ignored." + - name: GIT_SERVER_EXISTING_PVC + description: "Optional: Use an existing PVC for the git server instead of creating a new one. If this is set, the GIT_SERVER_PVC_SIZE variable will be ignored." default: "data-zarf-gitea-0" - name: GIT_SERVER_PVC_SIZE @@ -68,7 +68,7 @@ components: - gitea-values.yaml actions: onDeploy: - before: + before: - cmd: zarf tools kubectl label pvc -n zarf data-zarf-gitea-0 app.kubernetes.io/managed-by=Helm || true - cmd: zarf tools kubectl annotate pvc -n zarf data-zarf-gitea-0 meta.helm.sh/release-name=zarf-gitea || true - cmd: zarf tools kubectl annotate pvc -n zarf data-zarf-gitea-0 meta.helm.sh/release-namespace=zarf || true From 86137021f0450f327c5bd8fa74a41d18c2cca51f Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 27 Nov 2023 12:50:44 -0700 Subject: [PATCH 12/33] wip: added comments to new gitea.go methods for updating users --- src/internal/packager/git/gitea.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index c92e0daa60..369a034a43 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -84,6 +84,7 @@ func (g *Git) CreateReadOnlyUser() error { return err } +// UpdateReadOnlyUser uses the Gitea API to update a non-admin Zarf user. func (g *Git) UpdateReadOnlyUser(oldAdminPass string) error { message.Debugf("git.UpdateReadOnlyUser()") @@ -117,6 +118,7 @@ func (g *Git) UpdateReadOnlyUser(oldAdminPass string) error { return err } +// UpdatePushUser uses the Gitea API to update an admin Zarf user. func (g *Git) UpdatePushUser(oldAdminPass string) error { message.Debugf("git.UpdatePushUser()") From a1e832f9a9f487efc8b2e223bff6dfce66b48327 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 27 Nov 2023 13:47:08 -0700 Subject: [PATCH 13/33] wip: removed comment from gitea zarf.yaml --- packages/gitea/zarf.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index 91c032731d..2d469a887e 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -3,7 +3,6 @@ metadata: name: init-package-git-server variables: - #CHANGE BACK TO EXISTING CLAIM - name: GIT_SERVER_EXISTING_PVC description: "Optional: Use an existing PVC for the git server instead of creating a new one. If this is set, the GIT_SERVER_PVC_SIZE variable will be ignored." default: "data-zarf-gitea-0" From 5ed83ae3c35f5a739059457fb8e9ae4f973a3865 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Thu, 30 Nov 2023 07:57:05 -0700 Subject: [PATCH 14/33] wip: fix h.cfg case typo --- src/internal/packager/helm/zarf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/packager/helm/zarf.go b/src/internal/packager/helm/zarf.go index f409fe697e..796d0dc682 100644 --- a/src/internal/packager/helm/zarf.go +++ b/src/internal/packager/helm/zarf.go @@ -51,7 +51,7 @@ func (h *Helm) UpdateZarfRegistryValues() error { // UpdateZarfGiteaValues updates the Zarf git server deployment with the new state values func (h *Helm) UpdateZarfGiteaValues(oldState *types.ZarfState) error { oG := oldState.GitServer - nG := git.New(h.Cfg.State.GitServer) + nG := git.New(h.cfg.State.GitServer) err := nG.UpdateReadOnlyUser(oG.PushPassword) if err != nil { From bea78862ef0a3dd9eee153ad6954e98d2ba1113e Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Wed, 6 Dec 2023 07:41:43 -0700 Subject: [PATCH 15/33] wip: refactored UpdateZarfGiteaValues() and moved to gitea.go; refactored update gitea user api calls into one method --- src/cmd/tools/zarf.go | 3 +- src/internal/packager/git/gitea.go | 56 ++++++++++-------------------- src/internal/packager/helm/zarf.go | 18 ---------- 3 files changed, 20 insertions(+), 57 deletions(-) diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index 24ee3b5b66..6d47cb7313 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -144,7 +144,8 @@ var updateCredsCmd = &cobra.Command{ } } if slices.Contains(args, message.GitKey) && newState.GitServer.InternalServer { - err = h.UpdateZarfGiteaValues(oldState) + g := git.New(newState.GitServer) + err = g.UpdateZarfGiteaValues(oldState) if err != nil { // Warn if we couldn't actually update the git server (it might not be installed and we should try to continue) message.Warnf(lang.CmdToolsUpdateCredsUnableUpdateGit, err.Error()) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 54e07694b7..ad23b0d54c 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" ) // CreateTokenResponse is the response given from creating a token in Gitea @@ -84,49 +85,30 @@ func (g *Git) CreateReadOnlyUser() error { return err } -// UpdateReadOnlyUser uses the Gitea API to update a non-admin Zarf user. -func (g *Git) UpdateReadOnlyUser(oldAdminPass string) error { - message.Debugf("git.UpdateReadOnlyUser()") +// UpdateZarfGiteaValues updates the Zarf git server deployment with the new state values +func (g *Git) UpdateZarfGiteaValues(oldState *types.ZarfState) error { - - c, err := cluster.NewCluster() + //Update git read only user password + err := g.UpdateGitUser(oldState.GitServer.PushPassword, g.Server.PullUsername, g.Server.PullPassword) if err != nil { - return err - } - // Establish a git tunnel to send the repo - tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, k8s.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort) - if err != nil { - return err + return fmt.Errorf("unable to update gitea read only user password: %w", err) } - _, err = tunnel.Connect() + + // Update Git admin password + err = g.UpdateGitUser(oldState.GitServer.PushPassword, g.Server.PushUsername, g.Server.PushPassword) if err != nil { - return err + return fmt.Errorf("unable to update gitea admin user password: %w", err) } - defer tunnel.Close() - tunnelURL := tunnel.HTTPEndpoint() - - // Update the existing user's password - updateUserBody := map[string]interface{}{ - "login_name": g.Server.PullUsername, - "password": g.Server.PullPassword, - } - updateUserData, _ := json.Marshal(updateUserBody) - updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, g.Server.PullUsername) - updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) - out, err := g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, oldAdminPass) - message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) - return err + return nil } -// UpdatePushUser uses the Gitea API to update an admin Zarf user. -func (g *Git) UpdatePushUser(oldAdminPass string) error { - message.Debugf("git.UpdatePushUser()") +func (g *Git) UpdateGitUser(oldAdminPass string, username string, userpass string) error { + message.Debugf("git.UpdateGitUser()") c, err := cluster.NewCluster() if err != nil { return err } - // Establish a git tunnel to send the repo tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, k8s.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort) if err != nil { @@ -137,17 +119,15 @@ func (g *Git) UpdatePushUser(oldAdminPass string) error { return err } defer tunnel.Close() - tunnelURL := tunnel.HTTPEndpoint() - - // Make sure the user can't create their own repos or orgs + // Update the existing user's password updateUserBody := map[string]interface{}{ - "login_name": g.Server.PushUsername, - "password": g.Server.PushPassword, + "login_name": username, + "password": userpass, } updateUserData, _ := json.Marshal(updateUserBody) - updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, g.Server.PushUsername) + updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, username) updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) out, err := g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, oldAdminPass) message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) @@ -211,7 +191,7 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { createTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens", tunnelURL, g.Server.PushUsername) createTokensBody := map[string]interface{}{ - "name": config.ZarfArtifactTokenName, + "name": config.ZarfArtifactTokenName, "scopes": []string{"read:user"}, } createTokensData, _ := json.Marshal(createTokensBody) diff --git a/src/internal/packager/helm/zarf.go b/src/internal/packager/helm/zarf.go index 796d0dc682..b8d42c3f84 100644 --- a/src/internal/packager/helm/zarf.go +++ b/src/internal/packager/helm/zarf.go @@ -7,7 +7,6 @@ package helm import ( "fmt" - "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -48,23 +47,6 @@ func (h *Helm) UpdateZarfRegistryValues() error { return nil } -// UpdateZarfGiteaValues updates the Zarf git server deployment with the new state values -func (h *Helm) UpdateZarfGiteaValues(oldState *types.ZarfState) error { - oG := oldState.GitServer - nG := git.New(h.cfg.State.GitServer) - - err := nG.UpdateReadOnlyUser(oG.PushPassword) - if err != nil { - return fmt.Errorf("unable to update gitea read only user password: %w", err) - } - - err = nG.UpdatePushUser(oG.PushPassword) - if err != nil { - return fmt.Errorf("unable to update gitea admin user password: %w", err) - } - return nil -} - // UpdateZarfAgentValues updates the Zarf agent deployment with the new state values func (h *Helm) UpdateZarfAgentValues() error { spinner := message.NewProgressSpinner("Gathering information to update Zarf Agent TLS") From 99304c960c624ad443f59e65fe04c5cc97722c65 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:08:04 -0700 Subject: [PATCH 16/33] wip: add comment to UpdateGitUser() since it is exported --- src/internal/packager/git/gitea.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index ad23b0d54c..f9827b42a6 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -85,7 +85,7 @@ func (g *Git) CreateReadOnlyUser() error { return err } -// UpdateZarfGiteaValues updates the Zarf git server deployment with the new state values +// UpdateZarfGiteaValues updates the Zarf git server state func (g *Git) UpdateZarfGiteaValues(oldState *types.ZarfState) error { //Update git read only user password @@ -102,6 +102,7 @@ func (g *Git) UpdateZarfGiteaValues(oldState *types.ZarfState) error { return nil } +// UpdateGitUser updates Zarf git server users func (g *Git) UpdateGitUser(oldAdminPass string, username string, userpass string) error { message.Debugf("git.UpdateGitUser()") From 06feb72ee58618236aa6cb0bf43244d8dc4a8e91 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 11 Dec 2023 13:24:37 -0700 Subject: [PATCH 17/33] wip: adding read and write package scopes for git token --- src/internal/packager/git/gitea.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index f9827b42a6..67a3e6789c 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -193,7 +193,7 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { createTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens", tunnelURL, g.Server.PushUsername) createTokensBody := map[string]interface{}{ "name": config.ZarfArtifactTokenName, - "scopes": []string{"read:user"}, + "scopes": []string{"read:user", "read:package", "write:package"}, } createTokensData, _ := json.Marshal(createTokensBody) createTokensRequest, _ := netHttp.NewRequest("POST", createTokensEndpoint, bytes.NewBuffer(createTokensData)) From f7785680ec94206604252e9329eba50031e0b264 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 11 Dec 2023 19:53:31 -0700 Subject: [PATCH 18/33] wip: rename UpdateZarfGiteaValues to UpdateZarfGiteaUsers --- src/cmd/tools/zarf.go | 2 +- src/internal/packager/git/gitea.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index 6d47cb7313..b95e92876e 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -145,7 +145,7 @@ var updateCredsCmd = &cobra.Command{ } if slices.Contains(args, message.GitKey) && newState.GitServer.InternalServer { g := git.New(newState.GitServer) - err = g.UpdateZarfGiteaValues(oldState) + err = g.UpdateZarfGiteaUsers(oldState) if err != nil { // Warn if we couldn't actually update the git server (it might not be installed and we should try to continue) message.Warnf(lang.CmdToolsUpdateCredsUnableUpdateGit, err.Error()) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 67a3e6789c..fc4868e14c 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -85,8 +85,8 @@ func (g *Git) CreateReadOnlyUser() error { return err } -// UpdateZarfGiteaValues updates the Zarf git server state -func (g *Git) UpdateZarfGiteaValues(oldState *types.ZarfState) error { +// UpdateZarfGiteaUsers updates Zarf gitea users +func (g *Git) UpdateZarfGiteaUsers(oldState *types.ZarfState) error { //Update git read only user password err := g.UpdateGitUser(oldState.GitServer.PushPassword, g.Server.PullUsername, g.Server.PullPassword) From b127bb28399cf279c81876adaac5b2dd28f99b10 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Fri, 15 Dec 2023 10:03:05 -0700 Subject: [PATCH 19/33] wip: create zarf internal update-gitea-pvc process --- packages/gitea/gitea-values.yaml | 1 + packages/gitea/zarf.yaml | 12 +++++++++--- src/cmd/internal.go | 22 ++++++++++++++++++++++ src/config/lang/english.go | 5 +++++ src/internal/packager/git/gitea.go | 27 +++++++++++++++++++++++++++ 5 files changed, 64 insertions(+), 3 deletions(-) diff --git a/packages/gitea/gitea-values.yaml b/packages/gitea/gitea-values.yaml index 149f338849..83da05f44a 100644 --- a/packages/gitea/gitea-values.yaml +++ b/packages/gitea/gitea-values.yaml @@ -4,6 +4,7 @@ persistence: size: "###ZARF_VAR_GIT_SERVER_PVC_SIZE###" accessModes: - "###ZARF_VAR_GIT_SERVER_PVC_ACCESS_MODE###" + create: ###ZARF_VAR_GIT_SERVER_CREATE_PVC### replicaCount: ###ZARF_VAR_GIT_SERVER_REPLICA_COUNT### diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index 2d469a887e..bd890d838f 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -7,6 +7,10 @@ variables: description: "Optional: Use an existing PVC for the git server instead of creating a new one. If this is set, the GIT_SERVER_PVC_SIZE variable will be ignored." default: "data-zarf-gitea-0" + - name: GIT_SERVER_CREATE_PVC + desciption: "create pvc" + default: true + - name: GIT_SERVER_PVC_SIZE description: The size of the persistent volume claim for the git server default: 10Gi @@ -68,9 +72,9 @@ components: actions: onDeploy: before: - - cmd: zarf tools kubectl label pvc -n zarf data-zarf-gitea-0 app.kubernetes.io/managed-by=Helm || true - - cmd: zarf tools kubectl annotate pvc -n zarf data-zarf-gitea-0 meta.helm.sh/release-name=zarf-gitea || true - - cmd: zarf tools kubectl annotate pvc -n zarf data-zarf-gitea-0 meta.helm.sh/release-namespace=zarf || true + - cmd: ./zarf internal update-gitea-pvc --no-progress + setVariables: + - name: GIT_SERVER_CREATE_PVC after: - cmd: ./zarf internal create-read-only-gitea-user --no-progress maxRetries: 3 @@ -86,3 +90,5 @@ components: namespace: zarf name: app=gitea condition: Ready + # onFailure: + # - cmd: ./zarf internal update-gitea-pvc --roll-back diff --git a/src/cmd/internal.go b/src/cmd/internal.go index bb2981d13e..b7caf0c9e4 100644 --- a/src/cmd/internal.go +++ b/src/cmd/internal.go @@ -196,6 +196,27 @@ var createPackageRegistryToken = &cobra.Command{ }, } +var updateGiteaPVC = &cobra.Command{ + Use: "update-gitea-pvc", + Short: lang.CmdInternalUpdateGiteaPVCShort, + Long: lang.CmdInternalUpdateGiteaPVCLong, + Run: func(cmd *cobra.Command, args []string) { + c := cluster.NewClusterOrDie() + state, err := c.LoadZarfState() + if err != nil { + message.WarnErr(err, lang.ErrLoadState) + } + + val, err := git.New(state.GitServer).UpdateGiteaPVC() + + if err != nil { + message.WarnErr(err, lang.CmdInternalUpdateGiteaPVCErr) + } + + fmt.Print(val) + }, +} + var isValidHostname = &cobra.Command{ Use: "is-valid-hostname", Short: lang.CmdInternalIsValidHostnameShort, @@ -229,6 +250,7 @@ func init() { internalCmd.AddCommand(genTypesSchemaCmd) internalCmd.AddCommand(createReadOnlyGiteaUser) internalCmd.AddCommand(createPackageRegistryToken) + internalCmd.AddCommand(updateGiteaPVC) internalCmd.AddCommand(isValidHostname) internalCmd.AddCommand(computeCrc32) } diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 47563cf656..c98453507f 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -211,6 +211,11 @@ $ zarf init --artifact-push-password={PASSWORD} --artifact-push-username={USERNA "This is called internally by the supported Gitea package component." CmdInternalArtifactRegistryGiteaTokenErr = "Unable to create an artifact registry token for the Gitea service." + CmdInternalUpdateGiteaPVCShort = "Updates an existing Gitea persistent volume claim" + CmdInternalUpdateGiteaPVCLong = "Updates an existing Gitea persistent volume claim by assessing if claim is a custom user provided claim or default." + + "This is called internally by the supported Gitea package component." + CmdInternalUpdateGiteaPVCErr = "Unable to update the existing Gitea persistent volume claim." + CmdInternalIsValidHostnameShort = "Checks if the current machine's hostname is RFC1123 compliant" CmdInternalIsValidHostnameErr = "The hostname '%s' is not valid. Ensure the hostname meets RFC1123 requirements https://www.rfc-editor.org/rfc/rfc1123.html." diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index fc4868e14c..bafb970de1 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "io" + "os" "time" netHttp "net/http" @@ -18,6 +19,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" + "k8s.io/apimachinery/pkg/runtime/schema" ) // CreateTokenResponse is the response given from creating a token in Gitea @@ -212,6 +214,31 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { return createTokenResponse, nil } +func (g *Git) UpdateGiteaPVC() (string, error) { + create := "true" + c, err := cluster.NewCluster() + if err != nil { + return "false", err + } + + pvcName := os.Getenv("ZARF_VAR_GIT_SERVER_EXISTING_PVC") + groupKind := schema.GroupKind{ + Group: "", + Kind: "PersistentVolumeClaim", + } + labels := map[string]string{"app.kubernetes.io/managed-by": "Helm"} + annotations := map[string]string{"meta.helm.sh/release-name": "zarf-gitea", "meta.helm.sh/release-namespace": "zarf"} + + if pvcName == "data-zarf-gitea-0" { + c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) + return create, nil + } else { + c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) + create = "false" + return create, nil + } +} + // DoHTTPThings adds http request boilerplate and perform the request, checking for a successful response. func (g *Git) DoHTTPThings(request *netHttp.Request, username, secret string) ([]byte, error) { message.Debugf("git.DoHttpThings()") From 4351dadaba42fd9120f134db608fa0769b25e31d Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Fri, 15 Dec 2023 13:12:34 -0700 Subject: [PATCH 20/33] wip: adding roll back to update-gitea-pvc cmd --- packages/gitea/zarf.yaml | 11 ++++++----- src/cmd/internal.go | 10 ++++++++-- src/config/lang/english.go | 3 ++- src/internal/packager/git/gitea.go | 22 ++++++++++++++-------- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index bd890d838f..9626c39f98 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -8,8 +8,8 @@ variables: default: "data-zarf-gitea-0" - name: GIT_SERVER_CREATE_PVC - desciption: "create pvc" - default: true + description: "create pvc" + default: "true" - name: GIT_SERVER_PVC_SIZE description: The size of the persistent volume claim for the git server @@ -72,9 +72,10 @@ components: actions: onDeploy: before: - - cmd: ./zarf internal update-gitea-pvc --no-progress + - cmd: ./zarf internal update-gitea-pvc --no-progress setVariables: - name: GIT_SERVER_CREATE_PVC + mute: false after: - cmd: ./zarf internal create-read-only-gitea-user --no-progress maxRetries: 3 @@ -90,5 +91,5 @@ components: namespace: zarf name: app=gitea condition: Ready - # onFailure: - # - cmd: ./zarf internal update-gitea-pvc --roll-back + onFailure: + - cmd: ./zarf internal update-gitea-pvc --roll-back --no-progress diff --git a/src/cmd/internal.go b/src/cmd/internal.go index b7caf0c9e4..e9ffce27d5 100644 --- a/src/cmd/internal.go +++ b/src/cmd/internal.go @@ -23,6 +23,10 @@ import ( "github.com/spf13/pflag" ) +var ( + rollback bool +) + var internalCmd = &cobra.Command{ Use: "internal", Hidden: true, @@ -207,13 +211,13 @@ var updateGiteaPVC = &cobra.Command{ message.WarnErr(err, lang.ErrLoadState) } - val, err := git.New(state.GitServer).UpdateGiteaPVC() + helmShouldCreate, err := git.New(state.GitServer).UpdateGiteaPVC(rollback) if err != nil { message.WarnErr(err, lang.CmdInternalUpdateGiteaPVCErr) } - fmt.Print(val) + fmt.Print(helmShouldCreate) }, } @@ -253,6 +257,8 @@ func init() { internalCmd.AddCommand(updateGiteaPVC) internalCmd.AddCommand(isValidHostname) internalCmd.AddCommand(computeCrc32) + + updateGiteaPVC.Flags().BoolVarP(&rollback, "roll-back", "r", false, lang.CmdInternalFlagUpdateGiteaPVCRollback) } func addHiddenDummyFlag(cmd *cobra.Command, flagDummy string) { diff --git a/src/config/lang/english.go b/src/config/lang/english.go index c98453507f..81154f4cc4 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -214,7 +214,8 @@ $ zarf init --artifact-push-password={PASSWORD} --artifact-push-username={USERNA CmdInternalUpdateGiteaPVCShort = "Updates an existing Gitea persistent volume claim" CmdInternalUpdateGiteaPVCLong = "Updates an existing Gitea persistent volume claim by assessing if claim is a custom user provided claim or default." + "This is called internally by the supported Gitea package component." - CmdInternalUpdateGiteaPVCErr = "Unable to update the existing Gitea persistent volume claim." + CmdInternalUpdateGiteaPVCErr = "Unable to update the existing Gitea persistent volume claim." + CmdInternalFlagUpdateGiteaPVCRollback = "Roll back previous Gitea persistent volume claim updates." CmdInternalIsValidHostnameShort = "Checks if the current machine's hostname is RFC1123 compliant" CmdInternalIsValidHostnameErr = "The hostname '%s' is not valid. Ensure the hostname meets RFC1123 requirements https://www.rfc-editor.org/rfc/rfc1123.html." diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index bafb970de1..0e04ca24db 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -214,8 +214,7 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { return createTokenResponse, nil } -func (g *Git) UpdateGiteaPVC() (string, error) { - create := "true" +func (g *Git) UpdateGiteaPVC(shouldRollBack bool) (string, error) { c, err := cluster.NewCluster() if err != nil { return "false", err @@ -229,14 +228,21 @@ func (g *Git) UpdateGiteaPVC() (string, error) { labels := map[string]string{"app.kubernetes.io/managed-by": "Helm"} annotations := map[string]string{"meta.helm.sh/release-name": "zarf-gitea", "meta.helm.sh/release-namespace": "zarf"} - if pvcName == "data-zarf-gitea-0" { - c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) - return create, nil + if shouldRollBack { + labels = map[string]string{"app.kubernetes.io/managed-by": "-"} + annotations = map[string]string{"meta.helm.sh/release-name": "-", "meta.helm.sh/release-namespace": "-"} + err = c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) + return "false", err } else { - c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) - create = "false" - return create, nil + if pvcName == "data-zarf-gitea-0" { + err = c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) + return "true", err + } else { + err = c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) + return "true", err + } } + } // DoHTTPThings adds http request boilerplate and perform the request, checking for a successful response. From ebfd7cd35d2a727da040a98961d2c5501396a10e Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Fri, 15 Dec 2023 13:19:25 -0700 Subject: [PATCH 21/33] wip: fix testing bug --- src/internal/packager/git/gitea.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 0e04ca24db..786d54d133 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -239,7 +239,7 @@ func (g *Git) UpdateGiteaPVC(shouldRollBack bool) (string, error) { return "true", err } else { err = c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) - return "true", err + return "false", err } } From 29f2b278ac1da0c8538038ee89a23dd51813e6af Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Fri, 15 Dec 2023 14:05:49 -0700 Subject: [PATCH 22/33] wip: fix UpdateGiteaPVC() custom claim logic; refactor AddLabelsAndAnnotations into a facade and add RemoveLabelsAndAnnotations --- src/internal/packager/git/gitea.go | 6 ++---- src/pkg/k8s/dynamic.go | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 786d54d133..9edea9c8d0 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -214,6 +214,7 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { return createTokenResponse, nil } +// UpdateGiteaPVC updates the existing Gitea persistent volume claim and tells Gitea whether to create or not. func (g *Git) UpdateGiteaPVC(shouldRollBack bool) (string, error) { c, err := cluster.NewCluster() if err != nil { @@ -229,16 +230,13 @@ func (g *Git) UpdateGiteaPVC(shouldRollBack bool) (string, error) { annotations := map[string]string{"meta.helm.sh/release-name": "zarf-gitea", "meta.helm.sh/release-namespace": "zarf"} if shouldRollBack { - labels = map[string]string{"app.kubernetes.io/managed-by": "-"} - annotations = map[string]string{"meta.helm.sh/release-name": "-", "meta.helm.sh/release-namespace": "-"} - err = c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) + err = c.K8s.RemoveLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) return "false", err } else { if pvcName == "data-zarf-gitea-0" { err = c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) return "true", err } else { - err = c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) return "false", err } } diff --git a/src/pkg/k8s/dynamic.go b/src/pkg/k8s/dynamic.go index 51a1a29e90..6688f28441 100644 --- a/src/pkg/k8s/dynamic.go +++ b/src/pkg/k8s/dynamic.go @@ -16,6 +16,16 @@ import ( // AddLabelsAndAnnotations adds the provided labels and annotations to the specified K8s resource func (k *K8s) AddLabelsAndAnnotations(resourceNamespace string, resourceName string, groupKind schema.GroupKind, labels map[string]string, annotations map[string]string) error { + return k.updateLabelsAndAnnotations(resourceNamespace, resourceName, groupKind, labels, annotations, false) +} + +// RemoveLabelsAndAnnotations adds the provided labels and annotations to the specified K8s resource +func (k *K8s) RemoveLabelsAndAnnotations(resourceNamespace string, resourceName string, groupKind schema.GroupKind, labels map[string]string, annotations map[string]string) error { + return k.updateLabelsAndAnnotations(resourceNamespace, resourceName, groupKind, labels, annotations, true) +} + +// updateLabelsAndAnnotations updates the provided labels and annotations to the specified K8s resource +func (k *K8s) updateLabelsAndAnnotations(resourceNamespace string, resourceName string, groupKind schema.GroupKind, labels map[string]string, annotations map[string]string, isRemove bool) error { dynamicClient := dynamic.NewForConfigOrDie(k.RestConfig) discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(k.RestConfig) @@ -43,7 +53,11 @@ func (k *K8s) AddLabelsAndAnnotations(resourceNamespace string, resourceName str deployedLabels = make(map[string]string) } for key, value := range labels { - deployedLabels[key] = value + if isRemove { + delete(deployedLabels, key) + } else { + deployedLabels[key] = value + } } deployedResource.SetLabels(deployedLabels) @@ -55,7 +69,11 @@ func (k *K8s) AddLabelsAndAnnotations(resourceNamespace string, resourceName str deployedAnnotations = make(map[string]string) } for key, value := range annotations { - deployedAnnotations[key] = value + if isRemove { + delete(deployedAnnotations, key) + } else { + deployedAnnotations[key] = value + } } deployedResource.SetAnnotations(deployedAnnotations) From b8fdf4df9fbb43b04112b8f676dd08675ce33405 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 18 Dec 2023 09:12:48 -0700 Subject: [PATCH 23/33] wip: minor refactors --- packages/gitea/zarf.yaml | 8 ++------ src/cmd/internal.go | 11 +++-------- src/internal/packager/git/gitea.go | 6 +++--- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index 9626c39f98..e72e7a6c09 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -7,10 +7,6 @@ variables: description: "Optional: Use an existing PVC for the git server instead of creating a new one. If this is set, the GIT_SERVER_PVC_SIZE variable will be ignored." default: "data-zarf-gitea-0" - - name: GIT_SERVER_CREATE_PVC - description: "create pvc" - default: "true" - - name: GIT_SERVER_PVC_SIZE description: The size of the persistent volume claim for the git server default: 10Gi @@ -75,7 +71,7 @@ components: - cmd: ./zarf internal update-gitea-pvc --no-progress setVariables: - name: GIT_SERVER_CREATE_PVC - mute: false + mute: true after: - cmd: ./zarf internal create-read-only-gitea-user --no-progress maxRetries: 3 @@ -92,4 +88,4 @@ components: name: app=gitea condition: Ready onFailure: - - cmd: ./zarf internal update-gitea-pvc --roll-back --no-progress + - cmd: ./zarf internal update-gitea-pvc --rollback --no-progress diff --git a/src/cmd/internal.go b/src/cmd/internal.go index e9ffce27d5..a10067398c 100644 --- a/src/cmd/internal.go +++ b/src/cmd/internal.go @@ -205,14 +205,9 @@ var updateGiteaPVC = &cobra.Command{ Short: lang.CmdInternalUpdateGiteaPVCShort, Long: lang.CmdInternalUpdateGiteaPVCLong, Run: func(cmd *cobra.Command, args []string) { - c := cluster.NewClusterOrDie() - state, err := c.LoadZarfState() - if err != nil { - message.WarnErr(err, lang.ErrLoadState) - } - - helmShouldCreate, err := git.New(state.GitServer).UpdateGiteaPVC(rollback) + // There is a possibility that the pvc does not yet exist and Gitea helm chart should create it + helmShouldCreate, err := git.UpdateGiteaPVC(rollback) if err != nil { message.WarnErr(err, lang.CmdInternalUpdateGiteaPVCErr) } @@ -258,7 +253,7 @@ func init() { internalCmd.AddCommand(isValidHostname) internalCmd.AddCommand(computeCrc32) - updateGiteaPVC.Flags().BoolVarP(&rollback, "roll-back", "r", false, lang.CmdInternalFlagUpdateGiteaPVCRollback) + updateGiteaPVC.Flags().BoolVarP(&rollback, "rollback", "r", false, lang.CmdInternalFlagUpdateGiteaPVCRollback) } func addHiddenDummyFlag(cmd *cobra.Command, flagDummy string) { diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 76cc22f3db..7d4a39a62f 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -239,7 +239,7 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { } // UpdateGiteaPVC updates the existing Gitea persistent volume claim and tells Gitea whether to create or not. -func (g *Git) UpdateGiteaPVC(shouldRollBack bool) (string, error) { +func UpdateGiteaPVC(shouldRollBack bool) (string, error) { c, err := cluster.NewCluster() if err != nil { return "false", err @@ -254,11 +254,11 @@ func (g *Git) UpdateGiteaPVC(shouldRollBack bool) (string, error) { annotations := map[string]string{"meta.helm.sh/release-name": "zarf-gitea", "meta.helm.sh/release-namespace": "zarf"} if shouldRollBack { - err = c.K8s.RemoveLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) + err = c.K8s.RemoveLabelsAndAnnotations(cluster.ZarfNamespaceName, pvcName, groupKind, labels, annotations) return "false", err } else { if pvcName == "data-zarf-gitea-0" { - err = c.K8s.AddLabelsAndAnnotations("zarf", pvcName, groupKind, labels, annotations) + err = c.K8s.AddLabelsAndAnnotations(cluster.ZarfNamespaceName, pvcName, groupKind, labels, annotations) return "true", err } else { return "false", err From 892f6a1ad941ec0e54fbfc28206303ee7e13ce61 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:30:33 -0700 Subject: [PATCH 24/33] wip: updating gitea chart version to 10.0.0 -- updating gitea values and custom init package tutorial. --- docs/5-zarf-tutorials/8-custom-init-packages.md | 5 ++--- packages/gitea/gitea-values.yaml | 3 +-- packages/gitea/zarf.yaml | 6 ++---- zarf-config.toml | 3 +-- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/docs/5-zarf-tutorials/8-custom-init-packages.md b/docs/5-zarf-tutorials/8-custom-init-packages.md index 6f2e455b68..a99ebc1b59 100644 --- a/docs/5-zarf-tutorials/8-custom-init-packages.md +++ b/docs/5-zarf-tutorials/8-custom-init-packages.md @@ -91,11 +91,10 @@ $ zarf package create . \ --set REGISTRY_IMAGE_TAG=2.8.3 \ --set REGISTRY_IMAGE="opensource/registry" \ --set REGISTRY_IMAGE_DOMAIN="custom.enterprise.corp" \ ---set GITEA_IMAGE="custom.enterprise.corp/opensource/gitea" \ ---set GITEA_SERVER_VERSION="v1.19.3" +--set GITEA_IMAGE="custom.enterprise.corp/opensource/gitea:v1.21.0-rootless" ``` -⚠️ - The Gitea image and version are different than the Agent and Registry in that Zarf will always prefer the `rootless` version of a given server image. This means that the above reference would template out to be `custom.enterprise.corp/opensource/gitea:v1.19.3-rootless`. If you need to change this, edit the `packages/gitea` package. +⚠️ - The Gitea image is different from the Agent and Registry in that Zarf will always prefer the `rootless` version of a given server image. The image no longer must be tagged with `-rootless`, but it still needs to implement the [Gitea configuration of a rootless image](https://github.com/go-gitea/gitea/blob/main/Dockerfile.rootless). If you need to change this, edit the `packages/gitea` package. You can find all of the `--set` configurations by looking at the `zarf-config.toml` in the root of the repository. diff --git a/packages/gitea/gitea-values.yaml b/packages/gitea/gitea-values.yaml index 83da05f44a..4a780198b3 100644 --- a/packages/gitea/gitea-values.yaml +++ b/packages/gitea/gitea-values.yaml @@ -45,8 +45,7 @@ resources: memory: "###ZARF_VAR_GIT_SERVER_MEM_LIMIT###" image: - repository: "###ZARF_CONST_GITEA_IMAGE###" - tag: "###ZARF_CONST_GITEA_SERVER_VERSION###" + fullOverride: "###ZARF_CONST_GITEA_IMAGE###" rootless: true postgresql-ha: diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index e72e7a6c09..2fd860405a 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -42,8 +42,6 @@ variables: constants: - name: GITEA_IMAGE value: "###ZARF_PKG_TMPL_GITEA_IMAGE###" - - name: GITEA_SERVER_VERSION - value: "###ZARF_PKG_TMPL_GITEA_SERVER_VERSION###" components: - name: git-server @@ -51,7 +49,7 @@ components: Deploys Gitea to provide git repositories for Kubernetes configurations. Required for GitOps deployments if no other git server is available. images: - - "###ZARF_PKG_TMPL_GITEA_IMAGE###:###ZARF_PKG_TMPL_GITEA_SERVER_VERSION###-rootless" + - "###ZARF_PKG_TMPL_GITEA_IMAGE###" manifests: - name: git-connect namespace: zarf @@ -61,7 +59,7 @@ components: - name: gitea releaseName: zarf-gitea url: https://dl.gitea.io/charts - version: 9.5.1 + version: 10.0.0 namespace: zarf valuesFiles: - gitea-values.yaml diff --git a/zarf-config.toml b/zarf-config.toml index 1551059d33..998d31ed54 100644 --- a/zarf-config.toml +++ b/zarf-config.toml @@ -15,5 +15,4 @@ registry_image = 'library/registry' registry_image_tag = '2.8.3' # The image reference to use for the optional git-server Zarf deploys -gitea_image = 'gitea/gitea' -gitea_server_version = '1.20.5' +gitea_image = 'gitea/gitea:1.21.0-rootless' From 1fba065cb9ddd86610f8372df3173e0f13067ffd Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:39:14 -0700 Subject: [PATCH 25/33] wip: update e2e sbom test gitea version. --- src/test/e2e/06_create_sbom_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/e2e/06_create_sbom_test.go b/src/test/e2e/06_create_sbom_test.go index a7a4c640d7..2ccb21b499 100644 --- a/src/test/e2e/06_create_sbom_test.go +++ b/src/test/e2e/06_create_sbom_test.go @@ -54,9 +54,9 @@ func TestCreateSBOM(t *testing.T) { _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "sbom-viewer-docker.io_defenseunicorns_zarf-game_multi-tile-dark.html")) require.NoError(t, err) // Test that the init package generates the SBOMs we expect (images + component files) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.20.5-rootless.html")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.21.0-rootless.html")) require.NoError(t, err) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.20.5-rootless.json")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.21.0-rootless.json")) require.NoError(t, err) _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-zarf-component-k3s.html")) require.NoError(t, err) From 21a9c7d39932402c4e9da364bc0a8d2b90b2ad90 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Tue, 19 Dec 2023 11:28:08 -0700 Subject: [PATCH 26/33] wip: moved wait and added 422 check in CreateReadOnlyUser() --- packages/gitea/zarf.yaml | 13 +++++++------ src/internal/packager/git/gitea.go | 7 ++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index 2fd860405a..8fb37939eb 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -71,6 +71,12 @@ components: - name: GIT_SERVER_CREATE_PVC mute: true after: + - wait: + cluster: + kind: pod + namespace: zarf + name: app=gitea + condition: Ready - cmd: ./zarf internal create-read-only-gitea-user --no-progress maxRetries: 3 maxTotalSeconds: 60 @@ -79,11 +85,6 @@ components: maxRetries: 3 maxTotalSeconds: 60 description: Create an artifact registry token - - wait: - cluster: - kind: pod - namespace: zarf - name: app=gitea - condition: Ready + onFailure: - cmd: ./zarf internal update-gitea-pvc --rollback --no-progress diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 7d4a39a62f..b2364921f7 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "os" + "strings" "time" netHttp "net/http" @@ -75,7 +76,11 @@ func (g *Git) CreateReadOnlyUser() error { }) message.Debugf("POST %s:\n%s", createUserEndpoint, string(out)) if err != nil { - return err + if strings.Contains(err.Error(), "422") { + message.Debugf("Read-only git user already exists. Skipping...") + } else { + return err + } } // Make sure the user can't create their own repos or orgs From ecf13b2769bb79bd73504fe989a227af98cf628f Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Tue, 19 Dec 2023 11:39:03 -0700 Subject: [PATCH 27/33] wip: removing else blocks in UpdateGiteaPVC() according GH Actions Validate. --- src/internal/packager/git/gitea.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index b2364921f7..c9ca80f8f6 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -261,15 +261,14 @@ func UpdateGiteaPVC(shouldRollBack bool) (string, error) { if shouldRollBack { err = c.K8s.RemoveLabelsAndAnnotations(cluster.ZarfNamespaceName, pvcName, groupKind, labels, annotations) return "false", err - } else { - if pvcName == "data-zarf-gitea-0" { - err = c.K8s.AddLabelsAndAnnotations(cluster.ZarfNamespaceName, pvcName, groupKind, labels, annotations) - return "true", err - } else { - return "false", err - } } + if pvcName == "data-zarf-gitea-0" { + err = c.K8s.AddLabelsAndAnnotations(cluster.ZarfNamespaceName, pvcName, groupKind, labels, annotations) + return "true", err + } + + return "false", err } // DoHTTPThings adds http request boilerplate and perform the request, checking for a successful response. From 6f24e904f7fcb339e80a05f6ec548f068f3fd6fb Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:48:18 -0700 Subject: [PATCH 28/33] wip: add statusCode to return of DoHTTPThings(); check statusCode in CreateReadOnlyUser() --- src/internal/packager/git/gitea.go | 27 +++++++++++++------------- src/test/e2e/22_git_and_gitops_test.go | 6 +++--- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index c9ca80f8f6..268f7e7ae4 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "os" - "strings" "time" netHttp "net/http" @@ -66,18 +65,20 @@ func (g *Git) CreateReadOnlyUser() error { } var out []byte + var statusCode int // Send API request to create the user createUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users", tunnelURL) createUserRequest, _ := netHttp.NewRequest("POST", createUserEndpoint, bytes.NewBuffer(createUserData)) err = tunnel.Wrap(func() error { - out, err = g.DoHTTPThings(createUserRequest, g.Server.PushUsername, g.Server.PushPassword) + out, statusCode, err = g.DoHTTPThings(createUserRequest, g.Server.PushUsername, g.Server.PushPassword) return err }) message.Debugf("POST %s:\n%s", createUserEndpoint, string(out)) if err != nil { - if strings.Contains(err.Error(), "422") { + if statusCode == 422 { message.Debugf("Read-only git user already exists. Skipping...") + return nil } else { return err } @@ -93,7 +94,7 @@ func (g *Git) CreateReadOnlyUser() error { updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, g.Server.PullUsername) updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) err = tunnel.Wrap(func() error { - out, err = g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, g.Server.PushPassword) + out, _, err = g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, g.Server.PushPassword) return err }) message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) @@ -148,7 +149,7 @@ func (g *Git) UpdateGitUser(oldAdminPass string, username string, userpass strin updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, username) updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) err = tunnel.Wrap(func() error { - out, err = g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, oldAdminPass) + out, _, err = g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, oldAdminPass) return err }) message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) @@ -183,7 +184,7 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { getTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens", tunnelURL, g.Server.PushUsername) getTokensRequest, _ := netHttp.NewRequest("GET", getTokensEndpoint, nil) err = tunnel.Wrap(func() error { - out, err = g.DoHTTPThings(getTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + out, _, err = g.DoHTTPThings(getTokensRequest, g.Server.PushUsername, g.Server.PushPassword) return err }) message.Debugf("GET %s:\n%s", getTokensEndpoint, string(out)) @@ -209,7 +210,7 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { deleteTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens/%s", tunnelURL, g.Server.PushUsername, config.ZarfArtifactTokenName) deleteTokensRequest, _ := netHttp.NewRequest("DELETE", deleteTokensEndpoint, nil) err = tunnel.Wrap(func() error { - out, err = g.DoHTTPThings(deleteTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + out, _, err = g.DoHTTPThings(deleteTokensRequest, g.Server.PushUsername, g.Server.PushPassword) return err }) message.Debugf("DELETE %s:\n%s", deleteTokensEndpoint, string(out)) @@ -226,7 +227,7 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { createTokensData, _ := json.Marshal(createTokensBody) createTokensRequest, _ := netHttp.NewRequest("POST", createTokensEndpoint, bytes.NewBuffer(createTokensData)) err = tunnel.Wrap(func() error { - out, err = g.DoHTTPThings(createTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + out, _, err = g.DoHTTPThings(createTokensRequest, g.Server.PushUsername, g.Server.PushPassword) return err }) message.Debugf("POST %s:\n%s", createTokensEndpoint, string(out)) @@ -272,7 +273,7 @@ func UpdateGiteaPVC(shouldRollBack bool) (string, error) { } // DoHTTPThings adds http request boilerplate and perform the request, checking for a successful response. -func (g *Git) DoHTTPThings(request *netHttp.Request, username, secret string) ([]byte, error) { +func (g *Git) DoHTTPThings(request *netHttp.Request, username, secret string) ([]byte, int, error) { message.Debugf("git.DoHttpThings()") // Prep the request with boilerplate @@ -284,17 +285,17 @@ func (g *Git) DoHTTPThings(request *netHttp.Request, username, secret string) ([ // Perform the request and get the response response, err := client.Do(request) if err != nil { - return []byte{}, err + return []byte{}, 0, err } responseBody, _ := io.ReadAll(response.Body) // If we get a 'bad' status code we will have no error, create a useful one to return if response.StatusCode < 200 || response.StatusCode >= 300 { err = fmt.Errorf("got status code of %d during http request with body of: %s", response.StatusCode, string(responseBody)) - return []byte{}, err + return []byte{}, response.StatusCode, err } - return responseBody, nil + return responseBody, response.StatusCode, nil } func (g *Git) addReadOnlyUserToRepo(tunnelURL, repo string) error { @@ -312,7 +313,7 @@ func (g *Git) addReadOnlyUserToRepo(tunnelURL, repo string) error { // Send API request to add a user as a read-only collaborator to a repo addColabEndpoint := fmt.Sprintf("%s/api/v1/repos/%s/%s/collaborators/%s", tunnelURL, g.Server.PushUsername, repo, g.Server.PullUsername) addColabRequest, _ := netHttp.NewRequest("PUT", addColabEndpoint, bytes.NewBuffer(addColabData)) - out, err := g.DoHTTPThings(addColabRequest, g.Server.PushUsername, g.Server.PushPassword) + out, _, err := g.DoHTTPThings(addColabRequest, g.Server.PushUsername, g.Server.PushPassword) message.Debugf("PUT %s:\n%s", addColabEndpoint, string(out)) return err } diff --git a/src/test/e2e/22_git_and_gitops_test.go b/src/test/e2e/22_git_and_gitops_test.go index 6f0e7efc55..d9c4b572d4 100644 --- a/src/test/e2e/22_git_and_gitops_test.go +++ b/src/test/e2e/22_git_and_gitops_test.go @@ -77,7 +77,7 @@ func testGitServerReadOnly(t *testing.T, gitURL string) { // Get the repo as the readonly user repoName := "zarf-public-test-2469062884" getRepoRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s", gitURL, state.GitServer.PushUsername, repoName), nil) - getRepoResponseBody, err := gitCfg.DoHTTPThings(getRepoRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) + getRepoResponseBody, _, err := gitCfg.DoHTTPThings(getRepoRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) require.NoError(t, err) // Make sure the only permissions are pull (read) @@ -100,7 +100,7 @@ func testGitServerTagAndHash(t *testing.T, gitURL string) { // Get the Zarf repo tag repoTag := "v0.0.1" getRepoTagsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/tags/%s", gitURL, config.ZarfGitPushUser, repoName, repoTag), nil) - getRepoTagsResponseBody, err := gitCfg.DoHTTPThings(getRepoTagsRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) + getRepoTagsResponseBody, _, err := gitCfg.DoHTTPThings(getRepoTagsRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) require.NoError(t, err) // Make sure the pushed tag exists @@ -111,7 +111,7 @@ func testGitServerTagAndHash(t *testing.T, gitURL string) { // Get the Zarf repo commit repoHash := "01a23218923f24194133b5eb11268cf8d73ff1bb" getRepoCommitsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/git/commits/%s", gitURL, config.ZarfGitPushUser, repoName, repoHash), nil) - getRepoCommitsResponseBody, err := gitCfg.DoHTTPThings(getRepoCommitsRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) + getRepoCommitsResponseBody, _, err := gitCfg.DoHTTPThings(getRepoCommitsRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) require.NoError(t, err) require.Contains(t, string(getRepoCommitsResponseBody), repoHash) } From ef009c5738771f112aad71850003359bca0b3a1e Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 19 Dec 2023 14:35:21 -0700 Subject: [PATCH 29/33] fix linting issue --- src/internal/packager/git/gitea.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 268f7e7ae4..222ce3c5e6 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -79,9 +79,9 @@ func (g *Git) CreateReadOnlyUser() error { if statusCode == 422 { message.Debugf("Read-only git user already exists. Skipping...") return nil - } else { - return err } + + return err } // Make sure the user can't create their own repos or orgs From 07a60609b91d37eaf26d73d8924a0181ff067301 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 19 Dec 2023 14:42:32 -0700 Subject: [PATCH 30/33] Update src/pkg/k8s/dynamic.go --- src/pkg/k8s/dynamic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/k8s/dynamic.go b/src/pkg/k8s/dynamic.go index 6688f28441..daf87c7a1a 100644 --- a/src/pkg/k8s/dynamic.go +++ b/src/pkg/k8s/dynamic.go @@ -19,7 +19,7 @@ func (k *K8s) AddLabelsAndAnnotations(resourceNamespace string, resourceName str return k.updateLabelsAndAnnotations(resourceNamespace, resourceName, groupKind, labels, annotations, false) } -// RemoveLabelsAndAnnotations adds the provided labels and annotations to the specified K8s resource +// RemoveLabelsAndAnnotations removes the provided labels and annotations to the specified K8s resource func (k *K8s) RemoveLabelsAndAnnotations(resourceNamespace string, resourceName string, groupKind schema.GroupKind, labels map[string]string, annotations map[string]string) error { return k.updateLabelsAndAnnotations(resourceNamespace, resourceName, groupKind, labels, annotations, true) } From e754b548975044f23a8348a0e329fea321b6e4bb Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 19 Dec 2023 14:48:35 -0700 Subject: [PATCH 31/33] Update zarf-config.toml --- zarf-config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarf-config.toml b/zarf-config.toml index 998d31ed54..2010031c92 100644 --- a/zarf-config.toml +++ b/zarf-config.toml @@ -15,4 +15,4 @@ registry_image = 'library/registry' registry_image_tag = '2.8.3' # The image reference to use for the optional git-server Zarf deploys -gitea_image = 'gitea/gitea:1.21.0-rootless' +gitea_image = 'gitea/gitea:1.21.2-rootless' From 0f8ec5a5d252cac4b172e5c95d4b81612c2ee34d Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 19 Dec 2023 14:48:43 -0700 Subject: [PATCH 32/33] Update src/test/e2e/06_create_sbom_test.go --- src/test/e2e/06_create_sbom_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/e2e/06_create_sbom_test.go b/src/test/e2e/06_create_sbom_test.go index 2ccb21b499..ea2a5240a5 100644 --- a/src/test/e2e/06_create_sbom_test.go +++ b/src/test/e2e/06_create_sbom_test.go @@ -56,7 +56,7 @@ func TestCreateSBOM(t *testing.T) { // Test that the init package generates the SBOMs we expect (images + component files) _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.21.0-rootless.html")) require.NoError(t, err) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.21.0-rootless.json")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.21.2-rootless.json")) require.NoError(t, err) _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-zarf-component-k3s.html")) require.NoError(t, err) From f838e1027b34bddff5ff012b2f54e0e6f7764100 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 19 Dec 2023 14:48:51 -0700 Subject: [PATCH 33/33] Update src/test/e2e/06_create_sbom_test.go --- src/test/e2e/06_create_sbom_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/e2e/06_create_sbom_test.go b/src/test/e2e/06_create_sbom_test.go index ea2a5240a5..88e153860b 100644 --- a/src/test/e2e/06_create_sbom_test.go +++ b/src/test/e2e/06_create_sbom_test.go @@ -54,7 +54,7 @@ func TestCreateSBOM(t *testing.T) { _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "sbom-viewer-docker.io_defenseunicorns_zarf-game_multi-tile-dark.html")) require.NoError(t, err) // Test that the init package generates the SBOMs we expect (images + component files) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.21.0-rootless.html")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.21.2-rootless.html")) require.NoError(t, err) _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.21.2-rootless.json")) require.NoError(t, err)