Create dedicated postgres container in upgrade and migration

mgradm/cmd/migrate/podman/utils.go

@@ -14,6 +14,7 @@ import (
 	migration_shared "github.com/uyuni-project/uyuni-tools/mgradm/cmd/migrate/shared"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/coco"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/hub"
+	"github.com/uyuni-project/uyuni-tools/mgradm/shared/pgsql"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/podman"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/saline"
 	"github.com/uyuni-project/uyuni-tools/shared"
@@ -94,6 +95,16 @@ func migrateToPodman(
 		}
 	}
 
+	if err := podman_utils.SetupNetwork(false); err != nil {
+		return err
+	}
+
+	if err := pgsql.Upgrade(
+		systemd, authFile, flags.Pgsql,
+	); err != nil {
+		return err
+	}
+
 	schemaUpdateRequired := oldPgVersion != newPgVersion
 	if err := podman.RunPgsqlFinalizeScript(preparedImage, schemaUpdateRequired, true); err != nil {
 		return utils.Errorf(err, L("cannot run PostgreSQL finalize script"))

mgradm/cmd/migrate/shared/flags.go

@@ -17,6 +17,7 @@ func AddMigrateFlags(cmd *cobra.Command) {
 	utils.AddSCCFlag(cmd)
 	utils.AddImageFlag(cmd)
 	utils.AddDBUpgradeImageFlag(cmd)
+	utils.AddUpgradePgsqlFlags(cmd)
 	utils.AddUpgradeCocoFlag(cmd)
 	utils.AddUpgradeHubXmlrpcFlags(cmd)
 	utils.AddUpgradeSalineFlag(cmd)

mgradm/cmd/support/ptf/podman/podman.go

@@ -22,6 +22,7 @@ type podmanPTFFlags struct {
 	Coco       adm_utils.CocoFlags
 	Hubxmlrpc  adm_utils.HubXmlrpcFlags
 	Saline     adm_utils.SalineFlags
+	Pgsql      adm_utils.PgsqlFlags
 }
 
 func newCmd(globalFlags *types.GlobalFlags, run utils.CommandFunc[podmanPTFFlags]) *cobra.Command {

mgradm/cmd/support/ptf/podman/utils.go

@@ -44,7 +44,13 @@ func ptfForPodman(
 		return err
 	}
 
-	return podman.Upgrade(systemd, authFile, "", flags.Image, dummyImage, flags.Coco, flags.Hubxmlrpc, flags.Saline)
+	return podman.Upgrade(systemd, authFile, "",
+		flags.Image, dummyImage,
+		flags.Coco,
+		flags.Hubxmlrpc,
+		flags.Saline,
+		flags.Pgsql,
+	)
 }
 
 // variables for unit testing.

mgradm/cmd/upgrade/podman/utils.go

@@ -28,6 +28,13 @@ func upgradePodman(_ *types.GlobalFlags, flags *podmanUpgradeFlags, _ *cobra.Com
 	defer cleaner()
 
 	return podman.Upgrade(
-		systemd, authFile, flags.Image.Registry, flags.Image, flags.DBUpgradeImage, flags.Coco, flags.HubXmlrpc, flags.Saline,
+		systemd, authFile,
+		flags.Image.Registry,
+		flags.Image,
+		flags.DBUpgradeImage,
+		flags.Coco,
+		flags.HubXmlrpc,
+		flags.Saline,
+		flags.Pgsql,
 	)
 }

mgradm/cmd/upgrade/shared/flags.go

@@ -18,6 +18,7 @@ func AddUpgradeFlags(cmd *cobra.Command) {
 	utils.AddUpgradeCocoFlag(cmd)
 	utils.AddUpgradeHubXmlrpcFlags(cmd)
 	utils.AddUpgradeSalineFlag(cmd)
+	utils.AddUpgradePgsqlFlags(cmd)
 }
 
 // AddUpgradeListFlags add upgrade list flags to a command.

mgradm/shared/kubernetes/dbFinalize.go

@@ -53,7 +53,6 @@ func getDBFinalizeJob(
 
 	// Prepare the script
 	scriptData := templates.FinalizePostgresTemplateData{
-		RunAutotune:     true,
 		RunReindex:      migration,
 		RunSchemaUpdate: schemaUpdateRequired,
 		Migration:       migration,

mgradm/shared/kubernetes/services.go

@@ -40,7 +40,6 @@ func CreateServices(namespace string, debug bool) error {
 // If debug is true, the Java debug ports will be exposed.
 func GetServices(namespace string, debug bool) []*core.Service {
 	ports := utils.GetServerPorts(debug)
-	ports = append(ports, utils.DBPorts...)
 
 	servicesPorts := map[string][]types.PortMap{}
 	for _, port := range ports {

mgradm/shared/pgsql/pgsql.go

@@ -6,8 +6,6 @@ package pgsql
 
 import (
 	"fmt"
-	"os"
-	"path/filepath"
 
 	"github.com/rs/zerolog/log"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/templates"
@@ -50,25 +48,7 @@ func SetupPgsql(
 		return err
 	}
 
-	initdbDir, _, err := utils.TempDir()
-	if err != nil {
-		return err
-	}
-
-	// fixme: for now, we need the script outside of this func, in EnableSSL
-	// defer cleaner()
-
-	_ = os.Chmod(initdbDir, 0555)
-
-	data := templates.PgsqlConfigTemplateData{}
-
-	scriptName := "pgsqlConfig.sh"
-	scriptPath := filepath.Join(initdbDir, scriptName)
-	if err := utils.WriteTemplateToFile(data, scriptPath, 0555, true); err != nil {
-		return fmt.Errorf(L("failed to generate %s"), scriptName)
-	}
-
-	if err := generatePgsqlSystemdService(systemd, preparedImage, initdbDir, admin, password); err != nil {
+	if err := generatePgsqlSystemdService(systemd, preparedImage, admin, password); err != nil {
 		return utils.Errorf(err, L("cannot generate systemd service"))
 	}
 
@@ -82,18 +62,18 @@ func SetupPgsql(
 	if err := cnx.WaitForHealthcheck(); err != nil {
 		return err
 	}
-
 	// Now the servisce is up and ready, the admin credentials are no longer needed
-	if err := generatePgsqlSystemdService(systemd, preparedImage, "", "", ""); err != nil {
+	if err := generatePgsqlSystemdService(systemd, preparedImage, "", ""); err != nil {
 		return utils.Errorf(err, L("cannot generate systemd service"))
 	}
+
 	return nil
 }
 
 // EnableSSL enables ssl in postgres container, as long as the certs are mounted.
 func EnableSSL(systemd podman.Systemd) error {
 	cnx := shared.NewConnection("podman", podman.PgsqlContainerName, "")
-	if _, err := cnx.Exec("/docker-entrypoint-initdb.d/pgsqlConfig.sh"); err != nil {
+	if _, err := cnx.Exec("/docker-entrypoint-initdb.d/uyuni-postgres-config.sh"); err != nil {
 		return err
 	}
 
@@ -126,25 +106,59 @@ func Upgrade(
 	systemd podman.Systemd,
 	authFile string,
 	pgsqlFlags cmd_utils.PgsqlFlags,
-	admin string,
-	password string,
 ) error {
-	if err := SetupPgsql(systemd, authFile, pgsqlFlags, admin, password); err != nil {
+	image := pgsqlFlags.Image
+	currentReplicas := systemd.CurrentReplicaCount(podman.PgsqlService)
+	log.Debug().Msgf("Current HUB replicas running are %d.", currentReplicas)
+
+	if pgsqlFlags.Replicas == 0 {
+		log.Debug().Msg("No pgsql requested.")
+	}
+	if !pgsqlFlags.IsChanged {
+		log.Info().Msgf(L("No changes requested for hub. Keep %d replicas."), currentReplicas)
+	}
+
+	pullEnabled := (pgsqlFlags.Replicas > 0 && pgsqlFlags.IsChanged) || (currentReplicas > 0 && !pgsqlFlags.IsChanged)
+
+	pgsqlImage, err := utils.ComputeImage(pgsqlFlags.Image.Registry, pgsqlFlags.Image.Tag, image)
+
+	if err != nil {
+		return utils.Errorf(err, L("failed to compute image URL"))
+	}
+
+	preparedImage, err := podman.PrepareImage(authFile, pgsqlImage, pgsqlFlags.Image.PullPolicy, pullEnabled)
+	if err != nil {
+		return err
+	}
+	err = podman.RunContainer("uyuni-db-migrate", pgsqlImage, utils.PgsqlRequiredVolumeMounts, []string{},
+		[]string{"chown", "postgres", "/etc/pki/tls/private/pg-spacewalk.key"})
+	if err != nil {
 		return err
 	}
 
+	if err := generatePgsqlSystemdService(systemd, preparedImage, "", ""); err != nil {
+		return utils.Errorf(err, L("cannot generate systemd service"))
+	}
+
 	if err := systemd.ReloadDaemon(false); err != nil {
 		return err
 	}
 
-	return systemd.RestartInstantiated(podman.PgsqlService)
+	if err := EnablePgsql(systemd, 0); err != nil {
+		return err
+	}
+	if err := EnablePgsql(systemd, pgsqlFlags.Replicas); err != nil {
+		return err
+	}
+
+	cnx := shared.NewConnection("podman", podman.PgsqlContainerName, "")
+	return cnx.WaitForHealthcheck()
 }
 
 // generatePgsqlSystemdService creates the Hub XMLRPC systemd files.
 func generatePgsqlSystemdService(
 	systemd podman.Systemd,
 	image string,
-	initdb string,
 	admin string,
 	password string,
 ) error {
@@ -162,9 +176,6 @@ func generatePgsqlSystemdService(
 	}
 
 	environment := fmt.Sprintf("Environment=UYUNI_IMAGE=%s\n", image)
-	if initdb != "" {
-		environment += fmt.Sprintf("Environment=PODMAN_EXTRA_ARGS=\"-v %s:/docker-entrypoint-initdb.d:z\"\n", initdb)
-	}
 	if admin != "" {
 		environment += fmt.Sprintf("Environment=POSTGRES_USER=\"%s\"\n", admin)
 	}

mgradm/shared/podman/podman.go

@@ -18,6 +18,7 @@ import (
 	"github.com/spf13/viper"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/coco"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/hub"
+	"github.com/uyuni-project/uyuni-tools/mgradm/shared/pgsql"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/saline"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/templates"
 	adm_utils "github.com/uyuni-project/uyuni-tools/mgradm/shared/utils"
@@ -187,7 +188,14 @@ func RunMigration(
 	user string,
 	prepare bool,
 ) (*utils.InspectResult, error) {
-	scriptDir, cleaner, err := adm_utils.GenerateMigrationScript(sourceFqdn, user, false, prepare)
+	scriptDir, cleaner, err := adm_utils.GenerateMigrationScript(
+		sourceFqdn,
+		user,
+		false,
+		prepare,
+		"uyuni-pgsql-server.mgr.internal",
+		"uyuni-pgsql-server.mgr.internal",
+	)
 	if err != nil {
 		return nil, utils.Errorf(err, L("cannot generate migration script"))
 	}
@@ -209,7 +217,7 @@ func RunMigration(
 	}
 
 	log.Info().Msg(L("Migrating server"))
-	if err := podman.RunContainer("uyuni-migration", preparedImage, utils.ServerVolumeMounts, extraArgs,
+	if err := podman.RunContainer("uyuni-migration", preparedImage, utils.ServerMigrationVolumeMounts, extraArgs,
 		[]string{"/var/lib/uyuni-tools/migrate.sh"}); err != nil {
 		return nil, utils.Errorf(err, L("cannot run uyuni migration container"))
 	}
@@ -285,7 +293,7 @@ func RunPgsqlVersionUpgrade(
 			return utils.Errorf(err, L("cannot generate PostgreSQL database version upgrade script"))
 		}
 
-		err = podman.RunContainer(pgsqlVersionUpgradeContainer, preparedImage, utils.ServerVolumeMounts, extraArgs,
+		err = podman.RunContainer(pgsqlVersionUpgradeContainer, preparedImage, utils.PgsqlRequiredVolumeMounts, extraArgs,
 			[]string{"/var/lib/uyuni-tools/" + pgsqlVersionUpgradeScriptName})
 		if err != nil {
 			return err
@@ -305,10 +313,11 @@ func RunPgsqlFinalizeScript(serverImage string, schemaUpdateRequired bool, migra
 	extraArgs := []string{
 		"-v", scriptDir + ":/var/lib/uyuni-tools/",
 		"--security-opt", "label=disable",
+		"--network", podman.UyuniNetwork,
 	}
 	pgsqlFinalizeContainer := "uyuni-finalize-pgsql"
 	pgsqlFinalizeScriptName, err := adm_utils.GenerateFinalizePostgresScript(
-		scriptDir, true, schemaUpdateRequired, true, migration, false,
+		scriptDir, true, schemaUpdateRequired, migration, false,
 	)
 	if err != nil {
 		return utils.Errorf(err, L("cannot generate PostgreSQL finalization script"))
@@ -355,6 +364,7 @@ func Upgrade(
 	cocoFlags adm_utils.CocoFlags,
 	hubXmlrpcFlags adm_utils.HubXmlrpcFlags,
 	salineFlags adm_utils.SalineFlags,
+	pgsqlFlags adm_utils.PgsqlFlags,
 ) error {
 	if err := CallCloudGuestRegistryAuth(); err != nil {
 		return err
@@ -395,6 +405,22 @@ func Upgrade(
 	defer func() {
 		err = systemd.StartService(podman.ServerService)
 	}()
+
+	if inspectedValues.CurrentPgVersionNotMigrated != "" ||
+		inspectedValues.DBHost == "localhost" ||
+		inspectedValues.ReportDBHost == "localhost" {
+		log.Info().Msgf(L("Configuring external postgresql %s %s"),
+			inspectedValues.CurrentPgVersion, inspectedValues.CurrentPgVersionNotMigrated)
+
+		if err := RunPgsqlContainerMigration(
+			preparedImage, "uyuni-pgsql-server.mgr.internal", "uyuni-pgsql-server.mgr.internal",
+		); err != nil {
+			return utils.Errorf(err, L("cannot run PostgreSQL version upgrade script"))
+		}
+		inspectedValues.CurrentPgVersion = inspectedValues.CurrentPgVersionNotMigrated
+		pgsqlFlags.Replicas = 1 // migrating pgsql to separate container
+	}
+
 	if inspectedValues.ImagePgVersion > inspectedValues.CurrentPgVersion {
 		log.Info().Msgf(
 			L("Previous postgresql is %[1]s, instead new one is %[2]s. Performing a DB version upgrade…"),
@@ -414,6 +440,12 @@ func Upgrade(
 		)
 	}
 
+	if err := pgsql.Upgrade(
+		systemd, authFile, pgsqlFlags,
+	); err != nil {
+		return err
+	}
+
 	schemaUpdateRequired := inspectedValues.CurrentPgVersion != inspectedValues.ImagePgVersion
 	if err := RunPgsqlFinalizeScript(preparedImage, schemaUpdateRequired, false); err != nil {
 		return utils.Errorf(err, L("cannot run PostgreSQL finalize script"))
@@ -506,7 +538,7 @@ func Inspect(preparedImage string) (*utils.ServerInspectData, error) {
 		"--security-opt", "label=disable",
 	}
 
-	err = podman.RunContainer("uyuni-inspect", preparedImage, utils.ServerVolumeMounts, podmanArgs,
+	err = podman.RunContainer("uyuni-inspect", preparedImage, utils.ServerMigrationVolumeMounts, podmanArgs,
 		[]string{utils.InspectContainerDirectory + "/" + utils.InspectScriptFilename})
 	if err != nil {
 		return nil, err
@@ -520,6 +552,34 @@ func Inspect(preparedImage string) (*utils.ServerInspectData, error) {
 	return inspectResult, err
 }
 
+// RunPgsqlContainerMigration migrate to separate postgres container.
+func RunPgsqlContainerMigration(serverImage string, dbHost string, reportDBHost string) error {
+	scriptDir, cleaner, err := utils.TempDir()
+	if err != nil {
+		return err
+	}
+	defer cleaner()
+
+	data := templates.PgsqlMigrateScriptTemplateData{
+		DBHost:       dbHost,
+		ReportDBHost: reportDBHost,
+	}
+
+	scriptPath := filepath.Join(scriptDir, "pgmigrate.sh")
+	if err = utils.WriteTemplateToFile(data, scriptPath, 0555, true); err != nil {
+		return utils.Errorf(err, L("failed to generate postgresql migration script"))
+	}
+
+	podmanArgs := []string{
+		"-v", scriptDir + ":" + scriptDir,
+		"--security-opt", "label=disable",
+	}
+	err = podman.RunContainer("uyuni-db-migrate", serverImage, utils.ServerMigrationVolumeMounts, podmanArgs,
+		[]string{scriptPath})
+
+	return err
+}
+
 // CallCloudGuestRegistryAuth calls cloudguestregistryauth if it is available.
 func CallCloudGuestRegistryAuth() error {
 	cloudguestregistryauth := "cloudguestregistryauth"

mgradm/shared/templates/migrateScriptTemplate.go

@@ -140,7 +140,8 @@ $SSH {{ .SourceFqdn }} sh -c "systemctl list-unit-files | grep hub-xmlrpc-api |
 (test $($SSH {{ .SourceFqdn }} grep jdwp -r /etc/tomcat/conf.d/ /etc/rhn/taskomatic.conf | wc -l) -gt 0 && echo debug=true || echo debug=false) >>/var/lib/uyuni-tools/data
 
 echo "Altering configuration for domain resolution..."
-sed 's/report_db_host = {{ .SourceFqdn }}/report_db_host = localhost/' -i /etc/rhn/rhn.conf;
+sed 's/^db_host = .*/db_host = {{ .DBHost }}/' -i /etc/rhn/rhn.conf;
+sed 's/^report_db_host = .*/report_db_host = {{ .ReportDBHost }}/' -i /etc/rhn/rhn.conf;
 if ! grep -q '^java.hostname *=' /etc/rhn/rhn.conf; then
     sed 's/server\.jabber_server/java\.hostname/' -i /etc/rhn/rhn.conf;
 fi
@@ -199,11 +200,13 @@ echo "DONE"`
 
 // MigrateScriptTemplateData represents migration information used to create migration script.
 type MigrateScriptTemplateData struct {
-	Volumes    []types.VolumeMount
-	SourceFqdn string
-	User       string
-	Kubernetes bool
-	Prepare    bool
+	Volumes      []types.VolumeMount
+	SourceFqdn   string
+	User         string
+	Kubernetes   bool
+	Prepare      bool
+	DBHost       string
+	ReportDBHost string
 }
 
 // Render will create migration script.