Merge pull request 'feat: refs #8869 differential backups' (!84) from 8869-diffBackups into main

Reviewed-on: #84
Reviewed-by: Xavi Lleó <xavi@verdnatura.es>

roles/db/defaults/main.yaml

@@ -22,7 +22,7 @@ required_directories:
   - { path: /etc/systemd/system/mariadb.service.d, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
 required_mariabackup_files_and_scripts:
   - { src: mysql-flush.sh, dest: /etc/qemu/fsfreeze-hook.d/mysql-flush.sh, mode: u=rwx,g=rx,o=rx }  
-  - { src: mariabackup/bacula-before.sh, dest: /root/mariabackup/bacula-before.sh, mode: u=rwx,g=rx,o=rx }
+  - { src: mariabackup/make-backup.sh, dest: /root/mariabackup/make-backup.sh, mode: u=rwx,g=rx,o=rx }
   - { src: mariabackup/config.sh, dest: /root/mariabackup/config.sh, mode: u=rwx,g=rx,o=x }
   - { src: mariabackup/inc-backup.sh, dest: /root/mariabackup/inc-backup.sh, mode: u=rwx,g=rx,o=rx }
   - { src: mariabackup/restore-backup.sh, dest: /root/mariabackup/restore-backup.sh, mode: u=rwx,g=rx,o=rx }
@@ -45,7 +45,9 @@ downloads:
     dest: /tmp/percona-release_latest.generic_all.deb
     mode: u=rw,g=r,o=r
 clean_config_and_scripts:
-  - { path: /root/scripts/events-promote.sh }
-  - { path: /root/scripts/events-demote.sh }
-  - { path: /root/scripts/promote-master.sh }
-  - { path: /root/scripts/promote-slave.sh }
+  - /root/scripts/events-promote.sh
+  - /root/scripts/events-demote.sh
+  - /root/scripts/promote-master.sh
+  - /root/scripts/promote-slave.sh
+  - /root/scripts/inc-backup.sh
+  - /root/scripts/bacula-before.sh

roles/db/files/mariabackup/bacula-after.sh

@@ -5,12 +5,12 @@ myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 . "$myDir/config.sh"
 . "$myDir/apply.config.sh"
 
-pattern="$baculaDir/mnt/local-backup/*_full.gz"
+pattern="$backupDir/*_full.gz"
 files=($pattern)
 backupFile="${files[0]}"
 
 "$myDir/restore-backup.sh" "$backupFile"
-rm -r "$baculaDir"
+rm -r "$backupDir/"*
 
 if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then
 	for node in "${dbClusterSiblings[@]}"; do

roles/db/files/mariabackup/bacula-before.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-# https://mariadb.com/kb/en/mariabackup/
-set -e
-
-myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-. "$myDir/config.sh"
-
-todayDir="$(date +%Y-%m-%d)"
-backupName="${todayDir}_$(date +"%H-%M")_full"
-backupFile="$backupDir/$backupName.gz"
-
-if [ -d "$backupDir" ]; then
-	rm -rf "$backupDir/"*
-fi
-
-ulimit -n 8192
-mariabackup \
-	--defaults-extra-file="$myDir/my.cnf" \
-	--backup \
-	--extra-lsndir="$backupDir/$backupName" \
-	--history="$todayDir" \
-	--stream=xbstream \
-	--parallel=4 \
-	2>> "$logFile" \
-	| pigz -p 12 \
-	> "$backupFile"
-
-if [ $? != "0" ]; then
-	echo "An error ocurred during backup, please take a look at log file: $logFile"
-	exit 1
-fi

roles/db/files/mariabackup/config.sh

@@ -18,3 +18,5 @@ restoreDir=/mnt/mysqldata/mysql-restore
 # Directory of MySQL data
 dataDir=/mnt/mysqldata/mysql
 
+# Number of procs created by pigz
+pigzProcs=12

roles/db/files/mariabackup/inc-backup.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-# https://mariadb.com/kb/en/incremental-backup-and-restore-with-mariabackup/
-set -e
-
-myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-. "$myDir/config.sh"
-
-todayDir="$(date +%Y-%m-%d)"
-todayPath="$historyDir/$todayDir"
-
-pattern="$todayPath/${todayDir}_??-??_full.xb.gz.enc"
-files=($pattern)
-backupFile="${files[0]}"
-backupBase=$(basename -- "$backupFile")
-backupName="${backupBase%%.*}"
-
-incrementalName="${todayDir}_$(date +"%H-%M")_incremental"
-incrementalFile="$backupDir/${incrementalName}.xb.gz.enc"
-
-ulimit -n 24098
-mariabackup \
-	--defaults-extra-file="$myDir/my.cnf" \
-	--backup \
-	--incremental-basedir="$backupDir/$backupName" \
-	--extra-lsndir="$backupDir/$incrementalName" \
-	--incremental-history-name="$todayDir" \
-	2>> "$logFile" \
-	| gzip \
-	| openssl enc -aes-256-cbc -pbkdf2 -kfile "$myDir/xbcrypt.key" \
-	> "$incrementalFile"
-
-if [ $? != "0" ]; then
-	echo "An error ocurred during backup, please take a look at log file: $logFile"
-	exit 1
-fi
-
-cp "$incrementalFile" "$todayPath"
-cp -r "$backupDir/$incrementalName" "$todayPath"

roles/db/files/mariabackup/make-backup.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+#
+# Author: Juan Ferrer
+# Copyright (c) 2025 Verdnatura S.L. All rights reserved.
+# Version: 0.0.2
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# Backups the MariaDB database, it supports three modes:
+#  - Full: Full backup of all DB data, deletes all previous backups.
+#  - Differential: Backup changes since the last full backup, deletes all 
+#    previous differential backups.
+#  - Incremental: Backup changes since the last full backup, same as 
+#    differential, but does not erase any previous backups.
+#
+# References:
+#  - https://mariadb.com/kb/en/mariabackup/
+#  - https://mariadb.com/kb/en/mariabackup-options/
+#  - https://mariadb.com/kb/en/incremental-backup-and-restore-with-mariabackup/
+#  - https://mariadb.com/kb/en/setting-up-a-replica-with-mariabackup/
+#
+set -e
+
+myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$myDir/config.sh"
+
+level=$1
+historyName=$2
+curDate="$(date +%Y-%m-%d)"
+
+usage() {
+	echo "Usage: $0 <Full|Differential|Incremental> [historyName]"
+	exit 1
+}
+backup_error() {
+	echo "An error ocurred during backup, please take a look at log file: $logFile"
+	exit 2
+}
+
+if [ -z "$level" ]; then
+	usage
+fi
+
+case "$level" in
+	Full)
+		backupName="${curDate}_$(date +"%H-%M")_full"
+		backupFile="$backupDir/$backupName.gz"
+
+		if [ -d "$backupDir" ]; then
+			rm -rf "$backupDir/"*
+		fi
+		if [ -z "$historyName" ]; then
+			historyName=$backupName
+		fi
+
+		ulimit -n 8192
+		mariabackup \
+			--defaults-extra-file="$myDir/my.cnf" \
+			--backup \
+			--history="$historyName" \
+			--extra-lsndir="$backupDir/$backupName" \
+			--slave-info \
+			--safe-slave-backup \
+			2>> "$logFile" \
+			| pigz -p "$pigzProcs" \
+			> "$backupFile"
+
+		if [ $? != "0" ]; then
+			backup_error
+		fi
+		;;
+	Differential|Incremental)
+		pattern="$backupDir/${curDate}_??-??_full.gz"
+		files=($pattern)
+		backupFile="${files[0]}"
+		backupBase=$(basename -- "$backupFile")
+		backupName="${backupBase%%.*}"
+
+		incrementalName="${backupName%%_full}_diff_$(date +"%H-%M")"
+		incrementalFile="$backupDir/${incrementalName}.gz"
+
+		if [ "$level" = "Differential" ]; then
+			rm -rf "$backupDir/${curDate}_"??-??_diff_??-??
+		fi
+		if [ -z "$historyName" ]; then
+			historyName=$incrementalName
+		fi
+
+		ulimit -n 24098
+		mariabackup \
+			--defaults-extra-file="$myDir/my.cnf" \
+			--backup \
+			--history="$historyName" \
+			--extra-lsndir="$backupDir/$incrementalName" \
+			--incremental-basedir="$backupDir/$backupName" \
+			--slave-info \
+			--safe-slave-backup \
+			2>> "$logFile" \
+			| pigz -p "$pigzProcs" \
+			> "$incrementalFile"
+
+		if [ $? != "0" ]; then
+			backup_error
+		fi
+		;;
+	*)
+		usage
+		;;
+esac

roles/db/files/mariabackup/restore-backup.sh

@@ -28,7 +28,7 @@ mkdir -p "$restoreDir"
 echo "$(formatted_date)"
 echo "Decompresing backup."
 pigz --decompress --processes 4 --stdout "$backupFile" \
-	| mbstream --extract --parallel=4 --directory="$restoreDir"
+	| mbstream --extract --parallel=6 --directory="$restoreDir"
 
 echo "Preparing backup."
 mariabackup \

roles/db/tasks/mariadb.yml

@@ -125,6 +125,6 @@
 
 - name: Clean old configs or scripts
   file:
-    path: "{{ item.path }}"
+    path: "{{ item }}"
     state: absent
   loop: "{{ clean_config_and_scripts }}"

roles/db/templates/mariabackup/apply.config.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# Bacula directory for restore
-baculaDir=/mnt/local-backup/bacula-restore
-
 # Database branch name
 dbBranch={{ db.branch  }}
 

roles/db/templates/mariabackup/my.cnf

@@ -2,6 +2,6 @@
 host = localhost
 user = mariabackup
 password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }}
-use-memory = 1G
-parallel = 4
+use-memory = 4G
+parallel = 8
 stream = xbstream