feat: refs #8869 differential backups

roles/db/defaults/main.yaml

@@ -22,7 +22,7 @@ required_directories:
   - { path: /etc/systemd/system/mariadb.service.d, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
 required_mariabackup_files_and_scripts:
   - { src: mysql-flush.sh, dest: /etc/qemu/fsfreeze-hook.d/mysql-flush.sh, mode: u=rwx,g=rx,o=rx }  
-  - { src: mariabackup/bacula-before.sh, dest: /root/mariabackup/bacula-before.sh, mode: u=rwx,g=rx,o=rx }
+  - { src: mariabackup/make-backup.sh, dest: /root/mariabackup/make-backup.sh, mode: u=rwx,g=rx,o=rx }
   - { src: mariabackup/config.sh, dest: /root/mariabackup/config.sh, mode: u=rwx,g=rx,o=x }
   - { src: mariabackup/inc-backup.sh, dest: /root/mariabackup/inc-backup.sh, mode: u=rwx,g=rx,o=rx }
   - { src: mariabackup/restore-backup.sh, dest: /root/mariabackup/restore-backup.sh, mode: u=rwx,g=rx,o=rx }
@@ -45,7 +45,9 @@ downloads:
     dest: /tmp/percona-release_latest.generic_all.deb
     mode: u=rw,g=r,o=r
 clean_config_and_scripts:
-  - { path: /root/scripts/events-promote.sh }
-  - { path: /root/scripts/events-demote.sh }
-  - { path: /root/scripts/promote-master.sh }
-  - { path: /root/scripts/promote-slave.sh }
+  - /root/scripts/events-promote.sh
+  - /root/scripts/events-demote.sh
+  - /root/scripts/promote-master.sh
+  - /root/scripts/promote-slave.sh
+  - /root/scripts/inc-backup.sh
+  - /root/scripts/bacula-before.sh

roles/db/files/mariabackup/bacula-after.sh

@@ -5,12 +5,12 @@ myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 . "$myDir/config.sh"
 . "$myDir/apply.config.sh"
 
-pattern="$baculaDir/mnt/local-backup/*_full.gz"
+pattern="$backupDir/*_full.gz"
 files=($pattern)
 backupFile="${files[0]}"
 
 "$myDir/restore-backup.sh" "$backupFile"
-rm -r "$baculaDir"
+rm -r "$backupDir/"*
 
 if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then
 	for node in "${dbClusterSiblings[@]}"; do

roles/db/files/mariabackup/bacula-before.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-# https://mariadb.com/kb/en/mariabackup/
-set -e
-
-myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-. "$myDir/config.sh"
-
-todayDir="$(date +%Y-%m-%d)"
-backupName="${todayDir}_$(date +"%H-%M")_full"
-backupFile="$backupDir/$backupName.gz"
-
-if [ -d "$backupDir" ]; then
-	rm -rf "$backupDir/"*
-fi
-
-ulimit -n 8192
-mariabackup \
-	--defaults-extra-file="$myDir/my.cnf" \
-	--backup \
-	--extra-lsndir="$backupDir/$backupName" \
-	--history="$todayDir" \
-	--stream=xbstream \
-	--parallel=4 \
-	2>> "$logFile" \
-	| pigz -p 12 \
-	> "$backupFile"
-
-if [ $? != "0" ]; then
-	echo "An error ocurred during backup, please take a look at log file: $logFile"
-	exit 1
-fi

roles/db/files/mariabackup/config.sh

@@ -18,3 +18,5 @@ restoreDir=/mnt/mysqldata/mysql-restore
 # Directory of MySQL data
 dataDir=/mnt/mysqldata/mysql
 
+# Number of procs created by pigz
+pigzProcs=12

roles/db/files/mariabackup/inc-backup.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-# https://mariadb.com/kb/en/incremental-backup-and-restore-with-mariabackup/
-set -e
-
-myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-. "$myDir/config.sh"
-
-todayDir="$(date +%Y-%m-%d)"
-todayPath="$historyDir/$todayDir"
-
-pattern="$todayPath/${todayDir}_??-??_full.xb.gz.enc"
-files=($pattern)
-backupFile="${files[0]}"
-backupBase=$(basename -- "$backupFile")
-backupName="${backupBase%%.*}"
-
-incrementalName="${todayDir}_$(date +"%H-%M")_incremental"
-incrementalFile="$backupDir/${incrementalName}.xb.gz.enc"
-
-ulimit -n 24098
-mariabackup \
-	--defaults-extra-file="$myDir/my.cnf" \
-	--backup \
-	--incremental-basedir="$backupDir/$backupName" \
-	--extra-lsndir="$backupDir/$incrementalName" \
-	--incremental-history-name="$todayDir" \
-	2>> "$logFile" \
-	| gzip \
-	| openssl enc -aes-256-cbc -pbkdf2 -kfile "$myDir/xbcrypt.key" \
-	> "$incrementalFile"
-
-if [ $? != "0" ]; then
-	echo "An error ocurred during backup, please take a look at log file: $logFile"
-	exit 1
-fi
-
-cp "$incrementalFile" "$todayPath"
-cp -r "$backupDir/$incrementalName" "$todayPath"

roles/db/files/mariabackup/make-backup.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+# https://mariadb.com/kb/en/mariabackup/
+# https://mariadb.com/kb/en/mariabackup-options/
+# https://mariadb.com/kb/en/incremental-backup-and-restore-with-mariabackup/
+# https://mariadb.com/kb/en/setting-up-a-replica-with-mariabackup/
+set -e
+
+myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$myDir/config.sh"
+
+level=$1
+historyName=$2
+todayDir="$(date +%Y-%m-%d)"
+
+usage() {
+	echo "Usage: $0 <Full|Differential|Incremental> [historyName]"
+	exit 1
+}
+backup_error() {
+	echo "An error ocurred during backup, please take a look at log file: $logFile"
+	exit 2
+}
+
+if [ -z "$level" ]; then
+	usage
+fi
+
+case "$level" in
+	Full)
+		backupName="${todayDir}_$(date +"%H-%M")_full"
+		backupFile="$backupDir/$backupName.gz"
+
+		if [ -d "$backupDir" ]; then
+			rm -rf "$backupDir/"*
+		fi
+		if [ -z "$historyName" ]; then
+			historyName=$backupName
+		fi
+
+		ulimit -n 8192
+		mariabackup \
+			--defaults-extra-file="$myDir/my.cnf" \
+			--backup \
+			--history="$historyName" \
+			--extra-lsndir="$backupDir/$backupName" \
+			--slave-info \
+			--safe-slave-backup \
+			2>> "$logFile" \
+			| pigz -p "$pigzProcs" \
+			> "$backupFile"
+
+		if [ $? != "0" ]; then
+			backup_error
+		fi
+		;;
+	Differential|Incremental)
+		pattern="$backupDir/${todayDir}_??-??_full.gz"
+		files=($pattern)
+		backupFile="${files[0]}"
+		backupBase=$(basename -- "$backupFile")
+		backupName="${backupBase%%.*}"
+
+		incrementalName="${backupName%%_full}_diff_$(date +"%H-%M")"
+		incrementalFile="$backupDir/${incrementalName}.gz"
+
+		if [ -z "$historyName" ]; then
+			historyName=$incrementalName
+		fi
+
+		ulimit -n 24098
+		mariabackup \
+			--defaults-extra-file="$myDir/my.cnf" \
+			--backup \
+			--history="$historyName" \
+			--extra-lsndir="$backupDir/$incrementalName" \
+			--incremental-basedir="$backupDir/$backupName" \
+			--slave-info \
+			--safe-slave-backup \
+			2>> "$logFile" \
+			| pigz -p "$pigzProcs" \
+			> "$incrementalFile"
+
+		if [ $? != "0" ]; then
+			backup_error
+		fi
+		;;
+	*)
+		usage
+esac

roles/db/files/mariabackup/restore-backup.sh

@@ -28,7 +28,7 @@ mkdir -p "$restoreDir"
 echo "$(formatted_date)"
 echo "Decompresing backup."
 pigz --decompress --processes 4 --stdout "$backupFile" \
-	| mbstream --extract --parallel=4 --directory="$restoreDir"
+	| mbstream --extract --parallel=6 --directory="$restoreDir"
 
 echo "Preparing backup."
 mariabackup \

roles/db/tasks/mariadb.yml

@@ -125,6 +125,6 @@
 
 - name: Clean old configs or scripts
   file:
-    path: "{{ item.path }}"
+    path: "{{ item }}"
     state: absent
   loop: "{{ clean_config_and_scripts }}"

roles/db/templates/mariabackup/apply.config.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# Bacula directory for restore
-baculaDir=/mnt/local-backup/bacula-restore
-
 # Database branch name
 dbBranch={{ db.branch  }}
 

roles/db/templates/mariabackup/my.cnf

@@ -2,6 +2,6 @@
 host = localhost
 user = mariabackup
 password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }}
-use-memory = 1G
-parallel = 4
+use-memory = 4G
+parallel = 8
 stream = xbstream