diff --git a/roles/db/defaults/main.yaml b/roles/db/defaults/main.yaml index 61f6c73..4828592 100644 --- a/roles/db/defaults/main.yaml +++ b/roles/db/defaults/main.yaml @@ -22,7 +22,7 @@ required_directories: - { path: /etc/systemd/system/mariadb.service.d, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } required_mariabackup_files_and_scripts: - { src: mysql-flush.sh, dest: /etc/qemu/fsfreeze-hook.d/mysql-flush.sh, mode: u=rwx,g=rx,o=rx } - - { src: mariabackup/bacula-before.sh, dest: /root/mariabackup/bacula-before.sh, mode: u=rwx,g=rx,o=rx } + - { src: mariabackup/make-backup.sh, dest: /root/mariabackup/make-backup.sh, mode: u=rwx,g=rx,o=rx } - { src: mariabackup/config.sh, dest: /root/mariabackup/config.sh, mode: u=rwx,g=rx,o=x } - { src: mariabackup/inc-backup.sh, dest: /root/mariabackup/inc-backup.sh, mode: u=rwx,g=rx,o=rx } - { src: mariabackup/restore-backup.sh, dest: /root/mariabackup/restore-backup.sh, mode: u=rwx,g=rx,o=rx } @@ -45,7 +45,9 @@ downloads: dest: /tmp/percona-release_latest.generic_all.deb mode: u=rw,g=r,o=r clean_config_and_scripts: - - { path: /root/scripts/events-promote.sh } - - { path: /root/scripts/events-demote.sh } - - { path: /root/scripts/promote-master.sh } - - { path: /root/scripts/promote-slave.sh } \ No newline at end of file + - /root/scripts/events-promote.sh + - /root/scripts/events-demote.sh + - /root/scripts/promote-master.sh + - /root/scripts/promote-slave.sh + - /root/scripts/inc-backup.sh + - /root/scripts/bacula-before.sh \ No newline at end of file diff --git a/roles/db/files/mariabackup/bacula-after.sh b/roles/db/files/mariabackup/bacula-after.sh index 6cbaa68..d38e399 100755 --- a/roles/db/files/mariabackup/bacula-after.sh +++ b/roles/db/files/mariabackup/bacula-after.sh @@ -5,12 +5,12 @@ myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$myDir/config.sh" . "$myDir/apply.config.sh" -pattern="$baculaDir/mnt/local-backup/*_full.gz" +pattern="$backupDir/*_full.gz" files=($pattern) backupFile="${files[0]}" "$myDir/restore-backup.sh" "$backupFile" -rm -r "$baculaDir" +rm -r "$backupDir/"* if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then for node in "${dbClusterSiblings[@]}"; do diff --git a/roles/db/files/mariabackup/bacula-before.sh b/roles/db/files/mariabackup/bacula-before.sh deleted file mode 100644 index f4723a3..0000000 --- a/roles/db/files/mariabackup/bacula-before.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# https://mariadb.com/kb/en/mariabackup/ -set -e - -myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -. "$myDir/config.sh" - -todayDir="$(date +%Y-%m-%d)" -backupName="${todayDir}_$(date +"%H-%M")_full" -backupFile="$backupDir/$backupName.gz" - -if [ -d "$backupDir" ]; then - rm -rf "$backupDir/"* -fi - -ulimit -n 8192 -mariabackup \ - --defaults-extra-file="$myDir/my.cnf" \ - --backup \ - --extra-lsndir="$backupDir/$backupName" \ - --history="$todayDir" \ - --stream=xbstream \ - --parallel=4 \ - 2>> "$logFile" \ - | pigz -p 12 \ - > "$backupFile" - -if [ $? != "0" ]; then - echo "An error ocurred during backup, please take a look at log file: $logFile" - exit 1 -fi diff --git a/roles/db/files/mariabackup/config.sh b/roles/db/files/mariabackup/config.sh index bb61fd8..3aacc00 100644 --- a/roles/db/files/mariabackup/config.sh +++ b/roles/db/files/mariabackup/config.sh @@ -18,3 +18,5 @@ restoreDir=/mnt/mysqldata/mysql-restore # Directory of MySQL data dataDir=/mnt/mysqldata/mysql +# Number of procs created by pigz +pigzProcs=12 diff --git a/roles/db/files/mariabackup/inc-backup.sh b/roles/db/files/mariabackup/inc-backup.sh deleted file mode 100644 index c6d6e91..0000000 --- a/roles/db/files/mariabackup/inc-backup.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# https://mariadb.com/kb/en/incremental-backup-and-restore-with-mariabackup/ -set -e - -myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -. "$myDir/config.sh" - -todayDir="$(date +%Y-%m-%d)" -todayPath="$historyDir/$todayDir" - -pattern="$todayPath/${todayDir}_??-??_full.xb.gz.enc" -files=($pattern) -backupFile="${files[0]}" -backupBase=$(basename -- "$backupFile") -backupName="${backupBase%%.*}" - -incrementalName="${todayDir}_$(date +"%H-%M")_incremental" -incrementalFile="$backupDir/${incrementalName}.xb.gz.enc" - -ulimit -n 24098 -mariabackup \ - --defaults-extra-file="$myDir/my.cnf" \ - --backup \ - --incremental-basedir="$backupDir/$backupName" \ - --extra-lsndir="$backupDir/$incrementalName" \ - --incremental-history-name="$todayDir" \ - 2>> "$logFile" \ - | gzip \ - | openssl enc -aes-256-cbc -pbkdf2 -kfile "$myDir/xbcrypt.key" \ - > "$incrementalFile" - -if [ $? != "0" ]; then - echo "An error ocurred during backup, please take a look at log file: $logFile" - exit 1 -fi - -cp "$incrementalFile" "$todayPath" -cp -r "$backupDir/$incrementalName" "$todayPath" diff --git a/roles/db/files/mariabackup/make-backup.sh b/roles/db/files/mariabackup/make-backup.sh new file mode 100644 index 0000000..91e1963 --- /dev/null +++ b/roles/db/files/mariabackup/make-backup.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Author: Juan Ferrer +# Copyright (c) 2025 Verdnatura S.L. All rights reserved. +# Version: 0.0.2 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Backups the MariaDB database, it supports three modes: +# - Full: Full backup of all DB data, deletes all previous backups. +# - Differential: Backup changes since the last full backup, deletes all +# previous differential backups. +# - Incremental: Backup changes since the last full backup, same as +# differential, but does not erase any previous backups. +# +# References: +# - https://mariadb.com/kb/en/mariabackup/ +# - https://mariadb.com/kb/en/mariabackup-options/ +# - https://mariadb.com/kb/en/incremental-backup-and-restore-with-mariabackup/ +# - https://mariadb.com/kb/en/setting-up-a-replica-with-mariabackup/ +# +set -e + +myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$myDir/config.sh" + +level=$1 +historyName=$2 +curDate="$(date +%Y-%m-%d)" + +usage() { + echo "Usage: $0 [historyName]" + exit 1 +} +backup_error() { + echo "An error ocurred during backup, please take a look at log file: $logFile" + exit 2 +} + +if [ -z "$level" ]; then + usage +fi + +case "$level" in + Full) + backupName="${curDate}_$(date +"%H-%M")_full" + backupFile="$backupDir/$backupName.gz" + + if [ -d "$backupDir" ]; then + rm -rf "$backupDir/"* + fi + if [ -z "$historyName" ]; then + historyName=$backupName + fi + + ulimit -n 8192 + mariabackup \ + --defaults-extra-file="$myDir/my.cnf" \ + --backup \ + --history="$historyName" \ + --extra-lsndir="$backupDir/$backupName" \ + --slave-info \ + --safe-slave-backup \ + 2>> "$logFile" \ + | pigz -p "$pigzProcs" \ + > "$backupFile" + + if [ $? != "0" ]; then + backup_error + fi + ;; + Differential|Incremental) + pattern="$backupDir/${curDate}_??-??_full.gz" + files=($pattern) + backupFile="${files[0]}" + backupBase=$(basename -- "$backupFile") + backupName="${backupBase%%.*}" + + incrementalName="${backupName%%_full}_diff_$(date +"%H-%M")" + incrementalFile="$backupDir/${incrementalName}.gz" + + if [ "$level" = "Differential" ]; then + rm -rf "$backupDir/${curDate}_"??-??_diff_??-?? + fi + if [ -z "$historyName" ]; then + historyName=$incrementalName + fi + + ulimit -n 24098 + mariabackup \ + --defaults-extra-file="$myDir/my.cnf" \ + --backup \ + --history="$historyName" \ + --extra-lsndir="$backupDir/$incrementalName" \ + --incremental-basedir="$backupDir/$backupName" \ + --slave-info \ + --safe-slave-backup \ + 2>> "$logFile" \ + | pigz -p "$pigzProcs" \ + > "$incrementalFile" + + if [ $? != "0" ]; then + backup_error + fi + ;; + *) + usage + ;; +esac diff --git a/roles/db/files/mariabackup/restore-backup.sh b/roles/db/files/mariabackup/restore-backup.sh index 73d06e2..ecf2fd7 100644 --- a/roles/db/files/mariabackup/restore-backup.sh +++ b/roles/db/files/mariabackup/restore-backup.sh @@ -28,7 +28,7 @@ mkdir -p "$restoreDir" echo "$(formatted_date)" echo "Decompresing backup." pigz --decompress --processes 4 --stdout "$backupFile" \ - | mbstream --extract --parallel=4 --directory="$restoreDir" + | mbstream --extract --parallel=6 --directory="$restoreDir" echo "Preparing backup." mariabackup \ diff --git a/roles/db/tasks/mariadb.yml b/roles/db/tasks/mariadb.yml index db09ac0..4b204cd 100644 --- a/roles/db/tasks/mariadb.yml +++ b/roles/db/tasks/mariadb.yml @@ -125,6 +125,6 @@ - name: Clean old configs or scripts file: - path: "{{ item.path }}" + path: "{{ item }}" state: absent loop: "{{ clean_config_and_scripts }}" diff --git a/roles/db/templates/mariabackup/apply.config.sh b/roles/db/templates/mariabackup/apply.config.sh index 190a00a..ca604ae 100755 --- a/roles/db/templates/mariabackup/apply.config.sh +++ b/roles/db/templates/mariabackup/apply.config.sh @@ -1,8 +1,5 @@ #!/bin/bash -# Bacula directory for restore -baculaDir=/mnt/local-backup/bacula-restore - # Database branch name dbBranch={{ db.branch }} diff --git a/roles/db/templates/mariabackup/my.cnf b/roles/db/templates/mariabackup/my.cnf index 7037bef..47207c2 100644 --- a/roles/db/templates/mariabackup/my.cnf +++ b/roles/db/templates/mariabackup/my.cnf @@ -2,6 +2,6 @@ host = localhost user = mariabackup password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }} -use-memory = 1G -parallel = 4 +use-memory = 4G +parallel = 8 stream = xbstream