[release tool] fix hashrelease hanging on image checks (#9681)

* rewrite image check with timeout

* pass components list to manager

* cleanup

* address review feedback

release/cmd/hashrelease.go

@@ -209,8 +209,10 @@ func hashreleaseSubCommands(cfg *Config) []*cli.Command {
 					operator.WithValidate(!c.Bool(skipValidationFlag.Name)),
 					operator.WithTempDirectory(cfg.TmpDir),
 				)
-				if err := o.Publish(); err != nil {
-					return err
+				if !c.Bool(skipOperatorFlag.Name) {
+					if err := o.Publish(); err != nil {
+						return err
+					}
 				}
 
 				opts := []calico.Option{
@@ -231,6 +233,15 @@ func hashreleaseSubCommands(cfg *Config) []*cli.Command {
 				if reg := c.StringSlice(registryFlag.Name); len(reg) > 0 {
 					opts = append(opts, calico.WithImageRegistries(reg))
 				}
+				// Note: We only need to check that the correct images exist if we haven't built them ourselves.
+				// So, skip this check if we're configured to build and publish images from the local codebase.
+				if !c.Bool(publishHashreleaseImageFlag.Name) {
+					components, err := pinnedversion.RetrieveImageComponents(cfg.TmpDir)
+					if err != nil {
+						return fmt.Errorf("failed to retrieve images for the hashrelease: %v", err)
+					}
+					opts = append(opts, calico.WithComponents(components))
+				}
 				r := calico.NewManager(opts...)
 				if err := r.PublishRelease(); err != nil {
 					return err

release/internal/pinnedversion/pinnedversion.go

@@ -196,30 +196,9 @@ func (p *CalicoPinnedVersions) ManagerOptions() ([]calico.Option, error) {
 		return nil, err
 	}
 
-	components := pinnedVersion.Components
-	operator := registry.OperatorComponent{Component: pinnedVersion.TigeraOperator}
-	components[operator.Image] = operator.Component
-	initImage := operator.InitImage()
-	components[initImage.Image] = operator.InitImage()
-	for name, component := range components {
-		// Remove components that do not produce images.
-		if utils.Contains(noImageComponents, name) {
-			delete(components, name)
-			continue
-		}
-		img := registry.ImageMap[name]
-		if img != "" {
-			component.Image = img
-		} else if component.Image == "" {
-			component.Image = name
-		}
-		components[name] = component
-	}
-
 	return []calico.Option{
 		calico.WithVersion(pinnedVersion.Title),
 		calico.WithOperatorVersion(pinnedVersion.TigeraOperator.Version),
-		calico.WithComponents(components),
 	}, nil
 }
 
@@ -297,6 +276,33 @@ func LoadHashrelease(repoRootDir, outputDir, hashreleaseSrcBaseDir string, lates
 	}, nil
 }
 
+func RetrieveImageComponents(outputDir string) (map[string]registry.Component, error) {
+	pinnedVersion, err := retrievePinnedVersion(outputDir)
+	if err != nil {
+		return nil, err
+	}
+	components := pinnedVersion.Components
+	operator := registry.OperatorComponent{Component: pinnedVersion.TigeraOperator}
+	components[operator.Image] = operator.Component
+	initImage := operator.InitImage()
+	components[initImage.Image] = operator.InitImage()
+	for name, component := range components {
+		// Remove components that do not produce images.
+		if utils.Contains(noImageComponents, name) {
+			delete(components, name)
+			continue
+		}
+		img := registry.ImageMap[name]
+		if img != "" {
+			component.Image = img
+		} else if component.Image == "" {
+			component.Image = name
+		}
+		components[name] = component
+	}
+	return components, nil
+}
+
 func RetrieveVersions(outputDir string) (version.Data, error) {
 	pinnedVersion, err := retrievePinnedVersion(outputDir)
 	if err != nil {

release/pkg/manager/calico/manager.go

@@ -552,13 +552,40 @@ type imageExistsResult struct {
 	err    error
 }
 
-func imgExists(name string, component registry.Component, ch chan imageExistsResult) {
-	r := imageExistsResult{
-		name:  name,
-		image: component.String(),
+// checkHashreleaseImagesPublished checks that the images required for the hashrelease exist in the specified registries.
+func (r *CalicoManager) checkHashreleaseImagesPublished() ([]registry.Component, error) {
+	logrus.Info("Checking images required for hashrelease have already been published")
+	numOfComponents := len(r.imageComponents)
+	if numOfComponents == 0 {
+		logrus.Error("No images to check")
+		return nil, fmt.Errorf("no images to check")
+	}
+
+	resultsCh := make(chan imageExistsResult, numOfComponents)
+
+	for name, component := range r.imageComponents {
+		go func(name string, component registry.Component, ch chan imageExistsResult) {
+			exists, err := registry.ImageExists(component.ImageRef())
+			resultsCh <- imageExistsResult{
+				name:   name,
+				image:  component.String(),
+				exists: exists,
+				err:    err,
+			}
+		}(name, component, resultsCh)
+	}
+
+	var resultsErr error
+	missingImages := []registry.Component{}
+	for range r.imageComponents {
+		result := <-resultsCh
+		if result.err != nil {
+			resultsErr = errors.Join(resultsErr, fmt.Errorf("error checking %s exists: %s", result.image, result.err.Error()))
+		} else if !result.exists {
+			missingImages = append(missingImages, r.imageComponents[result.name])
+		}
 	}
-	r.exists, r.err = registry.ImageExists(component.ImageRef())
-	ch <- r
+	return missingImages, resultsErr
 }
 
 // Check that the environment has the necessary prereqs for publishing hashrelease
@@ -568,36 +595,19 @@ func (r *CalicoManager) hashreleasePrereqs() error {
 			return fmt.Errorf("missing hashrelease server configuration")
 		}
 	}
+
 	if r.publishImages {
 		return r.assertImageVersions()
 	} else {
-		results := make(map[string]imageExistsResult, len(r.imageComponents))
-		ch := make(chan imageExistsResult)
-		for name, component := range r.imageComponents {
-			go imgExists(name, component, ch)
-		}
-		for range images {
-			res := <-ch
-			results[res.name] = res
-		}
-		failedImageList := []string{}
-		for name, result := range results {
-			logrus.WithFields(logrus.Fields{
-				"image":  result.image,
-				"exists": result.exists,
-			}).Info("Validating image")
-			if result.err != nil || !result.exists {
-				logrus.WithError(result.err).WithField("image", name).Error("Error checking image")
-				failedImageList = append(failedImageList, r.imageComponents[name].String())
-			} else {
-				logrus.WithField("image", name).Info("Image exists")
-			}
-		}
-		failedCount := len(failedImageList)
-		if failedCount > 0 {
-			return fmt.Errorf("failed to validate %d images: %s", failedCount, strings.Join(failedImageList, ", "))
+		missingImages, err := r.checkHashreleaseImagesPublished()
+		if err != nil {
+			return fmt.Errorf("errors checking images: %s", err)
+		} else if len(missingImages) > 0 {
+			return fmt.Errorf("missing images for hashrelease: %v", missingImages)
 		}
+		logrus.Info("All images required for hashrelease have been published")
 	}
+
 	if r.imageScanning {
 		logrus.Info("Sending images to ISS")
 		imageList := []string{}
@@ -611,11 +621,13 @@ func (r *CalicoManager) hashreleasePrereqs() error {
 			logrus.WithError(err).Error("Failed to scan images")
 		}
 	}
+
 	return nil
 }
 
 // Check that the images exists with the correct version.
 func (r *CalicoManager) assertImageVersions() error {
+	logrus.Info("Checking built images exists with the correct version")
 	for _, img := range images {
 		imageName := strings.TrimPrefix(img, "calico/")
 		switch img {