Skip to content

Commit

Permalink
piraeus-server: deal with large backups
Browse files Browse the repository at this point in the history
There is a limit of around 1MB per resource in kubernetes. Big clusters might
need to make DB backups that are larger than that. We improve our current
solution in two ways:

* Provide a meaningful output should the run-migration not succeed. Instead
  of crashing by default, print what needs to be done to stderr and wait.
  Waiting can be skipped with an environment variable.
* Try to split the backup into smaller chunks, that can be reassembled on
  restore.

Signed-off-by: Moritz Wanzenböck <[email protected]>
  • Loading branch information
WanzenBug authored and rck committed Nov 4, 2024
1 parent 3bae14e commit c1b2638
Showing 1 changed file with 48 additions and 3 deletions.
51 changes: 48 additions & 3 deletions dockerfiles/piraeus-server/entry.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,29 @@
#!/bin/bash

set -e
set -eu

LB_WAIT_FOR_BACKUP=${LB_WAIT_FOR_BACKUP:-"true"}

create_backup_secret() {
BACKUP_NAME="$1"
BACKUP_FILE="$2"
VERSION="$3"

if ! kubectl create secret generic "${BACKUP_NAME}" --type piraeus.io/linstor-backup --from-file=backup.tar.gz="${BACKUP_FILE}"; then
echo "Backup ${BACKUP_FILE} may be too large to fit into a single secret, trying to split into chunks" >&2
split -d -b 512k "${BACKUP_FILE}" "${BACKUP_FILE}."
for SPLIT_FILE in "${BACKUP_FILE}".* ; do
kubectl create secret generic "${BACKUP_NAME}-${SPLIT_FILE##"${BACKUP_FILE}".}" --type piraeus.io/linstor-backup-part --from-file=backup.tar.gz="${SPLIT_FILE}"
kubectl label secret "${BACKUP_NAME}-${SPLIT_FILE##"${BACKUP_FILE}".}" piraeus.io/backup="${BACKUP_NAME}"
done

kubectl create secret generic "${BACKUP_NAME}" --type piraeus.io/linstor-backup
else
kubectl label secret "${BACKUP_NAME}" piraeus.io/backup="${BACKUP_NAME}"
fi

kubectl annotate secrets "${BACKUP_NAME}" "piraeus.io/linstor-version=${VERSION}" "piraeus.io/backup-version=2"
}

run_migration() {
if /usr/share/linstor-server/bin/linstor-config all-migrations-applied --logs=/var/log/linstor-controller --config-directory=/etc/linstor "$@" ; then
Expand All @@ -19,8 +42,30 @@ run_migration() {
kubectl get "${CRD}" -oyaml > "${CRD}.yaml"
done
tar -czvf backup.tar.gz -- *.yaml
kubectl create secret generic "${BACKUP_NAME}" --type piraeus.io/linstor-backup --from-file=backup.tar.gz
kubectl annotate secrets "${BACKUP_NAME}" "piraeus.io/linstor-version=${VERSION}"

if ! create_backup_secret "${BACKUP_NAME}" backup.tar.gz "${VERSION}"; then
cat <<EOF >&2
===============================================================================
Backup backup.tar.gz too large, even after chunking, to fit into secrets.
Please manually copy it to a safe location, by running:
kubectl cp -c run-migration $(hostname):/run/migration/backup.tar.gz .
EOF
if [ "${LB_WAIT_FOR_BACKUP}" != "true" ]; then
return 1
fi
cat <<EOF >&2
This container will wait until downloading of the backup has been confirmed.
To confirm the backup has been stored locally, create a secret using:
kubectl create secret generic ${BACKUP_NAME} --type piraeus.io/linstor-backup
EOF
while ! kubectl get secrets "${BACKUP_NAME}" 2>/dev/null ; do
sleep 10
done
fi
fi

/usr/share/linstor-server/bin/linstor-config run-migration --config-directory=/etc/linstor "$@"
Expand Down

0 comments on commit c1b2638

Please sign in to comment.