Merge pull request 'db: refs #8140 Fix apply SQL, dir structure changes, parallel backups' (!68) from 8140-applyFixesRefactor into main

Reviewed-on: #68
Reviewed-by: Xavi Lleó <xavi@verdnatura.es>

README.md

@@ -105,8 +105,7 @@ ansible-builder build --tag awx-ee:vn1
 
 ## Common playbooks
 
-* **facts.yml**: Collect and display facts from a host
-* **ping.yml**: Check that a host is alive and reachable
+* **debug.yml**: Debugging tasks: facts, vars, ping...
 * **awx.yml**: Create and configure AWX user
 * **debian.yml**: Setup base Debian server
 

playbooks/ceph.yml

@@ -5,3 +5,4 @@
       name: debian
   - import_role:
       name: ceph
+    tags: service

playbooks/db.yml

@@ -5,3 +5,4 @@
       name: debian
   - import_role:
       name: db
+    tags: service

playbooks/debug.yml

@@ -14,7 +14,7 @@
       var: vars
 
   - name: Print variable value
-    tags: var
+    tags: [facts, vars, var]
     when: var_name is defined
     debug:
       msg: "{{ var_name }}: {{ lookup('vars', var_name, default='undefined') }}"

playbooks/kube.yml

@@ -5,3 +5,4 @@
       name: debian
   - import_role:
       name: kube
+    tags: service

playbooks/pve.yml

@@ -5,3 +5,4 @@
       name: debian
   - import_role:
       name: pve
+    tags: service

roles/db/defaults/main.yaml

@@ -2,6 +2,7 @@ mariadb_base_packages:
   - mariadb-server
   - mariadb-backup
   - pmm2-client
+  - pigz
 mariadb_requeriments:
   - curl
   - apt-transport-https
@@ -19,24 +20,23 @@ required_directories:
   - { path: /root/mariabackup, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
   - { path: /mnt/mysqldata/mysql, owner: mysql, group: mysql, mode: 'u=rwx,g=rx,o=rx' }
   - { path: /etc/systemd/system/mariadb.service.d, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
-required_files_and_mariabackup_files_and_root_scripts:
-  - { src: "mariadb_override.conf", dest: "/etc/systemd/system/mariadb.service.d/override.conf", mode: "u=rw,g=r,o=r" }
-  - { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" }  
-  - { src: "files/mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" }
-  - { src: "files/mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/mariabackup/restore-backup.sh", dest: "/root/mariabackup/restore-backup.sh", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/scripts/check-memory.sh", dest: "/root/scripts/check-memory.sh", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/scripts/export-privs.sh", dest: "/root/scripts/export-privs.sh", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/scripts/mysqltuner.pl", dest: "/root/scripts/mysqltuner.pl", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/scripts/promote-master.sh", dest: "/root/scripts/promote-master.sh", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/scripts/promote-slave.sh", dest: "/root/scripts/promote-slave.sh", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/scripts/README.md", dest: "/root/scripts/README.md", mode: "u=rw,g=r,o=r" }
-  - { src: "files/scripts/scheduler-log.sh", dest: "/root/scripts/scheduler-log.sh", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/marianrpe/95-mariadb.cfg", dest: "/etc/nagios/nrpe.d/95-mariadb.cfg", mode: "u=rw,g=r,o=r" }
-mariabackup_test_dev_files:
-  - { src: "mariabackuptest/apply.sql", dest: "/root/mariabackup/", mode: "u=rw,g=r,o=r" }
-  - { src: "mariabackuptest/bacula-after.sh", dest: "/root/mariabackup/", mode: "u=rwx,g=rx,o=rx" }
+required_mariabackup_files_and_scripts:
+  - { src: mysql-flush.sh, dest: /etc/qemu/fsfreeze-hook.d/mysql-flush.sh, mode: u=rwx,g=rx,o=rx }  
+  - { src: mariabackup/bacula-before.sh, dest: /root/mariabackup/bacula-before.sh, mode: u=rwx,g=rx,o=rx }
+  - { src: mariabackup/config.sh, dest: /root/mariabackup/config.sh, mode: u=rwx,g=rx,o=x }
+  - { src: mariabackup/inc-backup.sh, dest: /root/mariabackup/inc-backup.sh, mode: u=rwx,g=rx,o=rx }
+  - { src: mariabackup/restore-backup.sh, dest: /root/mariabackup/restore-backup.sh, mode: u=rwx,g=rx,o=rx }
+  - { src: scripts/check-memory.sh, dest: /root/scripts/check-memory.sh, mode: u=rwx,g=rx,o=rx }
+  - { src: scripts/export-privs.sh, dest: /root/scripts/export-privs.sh, mode: u=rwx,g=rx,o=rx }
+  - { src: scripts/mysqltuner.pl, dest: /root/scripts/mysqltuner.pl, mode: u=rwx,g=rx,o=rx }
+  - { src: scripts/promote-master.sh, dest: /root/scripts/promote-master.sh, mode: u=rwx,g=rx,o=rx }
+  - { src: scripts/promote-slave.sh, dest: /root/scripts/promote-slave.sh, mode: u=rwx,g=rx,o=rx }
+  - { src: scripts/README.md, dest: /root/scripts/README.md, mode: u=rw,g=r,o=r }
+  - { src: scripts/scheduler-log.sh, dest: /root/scripts/scheduler-log.sh, mode: u=rwx,g=rx,o=rx }
 downloads:
-  - { url: "https://r.mariadb.com/downloads/mariadb_repo_setup", dest: "/tmp/mariadb_repo_setup", mode: "u=rwx,g=rx,o=rx" }
-  - { url: "https://repo.percona.com/apt/percona-release_latest.generic_all.deb", dest: "/tmp/percona-release_latest.generic_all.deb", mode: "u=rw,g=r,o=r" }
+  - url: https://r.mariadb.com/downloads/mariadb_repo_setup
+    dest: /tmp/mariadb_repo_setup
+    mode: u=rwx,g=rx,o=rx
+  - url: https://repo.percona.com/apt/percona-release_latest.generic_all.deb
+    dest: /tmp/percona-release_latest.generic_all.deb
+    mode: u=rw,g=r,o=r

roles/db/files/mariabackup/bacula-before.sh

@@ -19,12 +19,13 @@ mariabackup \
 	--backup \
 	--extra-lsndir="$backupDir/$backupName" \
 	--history="$todayDir" \
+	--stream=xbstream \
+	--parallel=4 \
 	2>> "$logFile" \
-	| gzip \
+	| pigz -p 12 \
 	> "$backupFile"
 
 if [ $? != "0" ]; then
 	echo "An error ocurred during backup, please take a look at log file: $logFile"
 	exit 1
 fi
-

roles/db/files/mariabackup/restore-backup.sh

@@ -27,8 +27,8 @@ mkdir -p "$restoreDir"
 
 echo "$(formatted_date)"
 echo "Decompresing backup."
-gzip --decompress --stdout "$backupFile" \
-	| mbstream -x --directory="$restoreDir"
+pigz --decompress --processes 4 --stdout "$backupFile" \
+	| mbstream --extract --parallel=4 --directory="$restoreDir"
 
 echo "Preparing backup."
 mariabackup \

roles/db/files/mariabackuptest/bacula-after.sh

@@ -1,51 +0,0 @@
-#!/bin/bash
-set -e
-
-myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-. "$myDir/config.sh"
-. "$myDir/apply.config.sh"
-
-todayDir=$(date +%Y-%m-%d)
-pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz"
-files=($pattern)
-backupFile="${files[0]}"
-
-"$myDir/restore-backup.sh" "$backupFile"
-rm -r "$baculaDir"
-
-if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then
-	for node in "${dbClusterSiblings[@]}"; do
-		ssh root@$node service mysql stop
-		ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi"
-	done
-	
-	galera_new_cluster
-else
-	service mariadb start
-fi
-
-echo "Applying custom script."
-mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()"
-mysql < "$myDir/apply.sql"
-
-echo "Upgrading tables."
-mysql_upgrade
-
-echo "Applying repository changes."
-curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec"
-
-echo "Waiting for Jenkins job to end."
-jobResult=null
-while [ "$jobResult" = "null" ]; do
-	sleep 10
-	jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result")
-done
-echo "Job result: $jobResult"
-
-echo "Promoting to master."
-"/root/scripts/promote-master.sh"
-
-for node in "${dbClusterSiblings[@]}"; do
-	ssh root@$node service mysql start
-done
-

roles/db/handlers/main.yml

@@ -5,3 +5,7 @@
   systemd:
     name: mariadb
     state: restarted
+- name: restart-nrpe
+  service:
+    name: nagios-nrpe-server
+    state: restarted

roles/db/tasks/main.yml

@@ -1,2 +1,5 @@
 - import_tasks: mariadb.yml
-  tags: service
+- when: db.branch == 'master'
+  import_tasks: production.yml
+- when: db.branch in ['dev', 'test']
+  import_tasks: test.yml

roles/db/tasks/mariadb.yml

@@ -1,6 +1,3 @@
-# Review /root/scripts/check-memory.sh --> It's not optimal to do what this program does
-# Also review the cron task /root/scripts/scheduler-log.sh
-
 - name: Ensure Install requirements for MariaDB repository setup script
   apt:
     name: "{{ mariadb_requeriments }}"
@@ -43,7 +40,7 @@
       tmpfs /mnt/mysqltmp         tmpfs rw,size={{ mysqltmpsize }}         0 0
   register: fstab
 
-- name: Set MariaDB Cron to /etc/cron.d
+- name: Configure MariaDB memory check CRON
   template:
     src: check-memory.cron
     dest: /etc/cron.d/vn-check-memory
@@ -79,115 +76,58 @@
     mode: "{{ item.mode }}"
   loop: "{{ required_directories }}"
 
-- when: db.branch == 'master'
-  block:
+- name: Copy required MariaBackup files and scripts
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: root
+    group: root
+    mode: "{{ item.mode }}"
+  loop: "{{ required_mariabackup_files_and_scripts }}"
 
-  - name: Ensure required directories exist in Production
-    file:
-      path: "{{ item.path }}"
-      state: directory
-      owner: "{{ item.owner }}"
-      group: "{{ item.group }}"
-      mode: "{{ item.mode }}"
-    loop: "{{ required_directories_master }}"
-  
-  - name: Set MariaDB custom configuration Production
-    copy:
-      src: "{{ item }}"
-      dest: /etc/mysql/mariadb.conf.d/
-      owner: root
-      group: root
-      mode: u=rw,g=r,o=r
-    with_items:
-      - "mariaconfigfiles/z90-vn.cnf"
-      - "mariaconfigfiles/z95-production.cnf"
-    notify: restart-mariadb
-  
-  - name: Reminder to check mount points
-    debug:
-      msg: |
-        Remember to check the following mount points:
-        - /var/lib/mysql
-        - /mnt/mysqlbin
-        - /mnt/local-backup
-        Make sure they are correctly configured and accessible.
+- name: Set MariaDB common configuration
+  copy:
+    src: conf/z90-vn.cnf
+    dest: /etc/mysql/mariadb.conf.d/
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify: restart-mariadb
 
-- when: db.branch == 'test'
-
-  block:
-  - name: Set MariaBackup custom template configuration TestDB
-    template:
-      src: test-db_dev/apply.config.sh
-      dest: /root/mariabackup/
-      owner: root
-      group: root
-      mode: u=rw,g=,o=
-
-  - name: Reminder to check mount points environment TestDB
-    debug:
-      msg: |
-        Remember to check the following mount points:
-        - /mnt/mysqltmp
-        Make sure they are correctly configured and accessible.
-
-- when: db.branch == 'dev'
-
-  block:
-  - name: Set MariaBackup custom template configuration DevDB
-    template:
-      src: test-db_dev/apply.config.sh
-      dest: /root/mariabackup/
-      owner: root
-      group: root
-      mode: u=rw,g=,o=
-
-  - name: Reminder to check mount points environment DevDB
-    debug:
-      msg: |
-        Remember to check the following mount points:
-        - /mnt/mysqltmp
-        Make sure they are correctly configured and accessible.
-
-- when: db.branch == 'dev' or db.branch == 'test'
-
-  block:
-  - name: Set MariaDB custom configuration TestDB or DevDB
-    copy:
-      src: "{{ item }}"
-      dest: /etc/mysql/mariadb.conf.d/
-      owner: root
-      group: root
-      mode: u=rw,g=r,o=r
-    with_items:
-      - "files/mariaconfigfiles/z90-vn.cnf"
-      - "files/mariaconfigfiles/z92-testing_dev.cnf"
-    notify: restart-mariadb
-
-  - name: Set Mariabackup files for TestDB or DevDB
-    copy:
-      src: "{{ item.src }}"
-      dest: "{{ item.dest }}"
-      owner: root
-      group: root
-      mode: "{{ item.mode }}"
-    loop: "{{ mariabackup_test_dev_files }}"
-
-- name: Set Custom Configuration local template all Environment
+- name: Set MariaDB local configuration template
   template:
-    src: z99-local.cnf
+    src: conf/z99-local.cnf
     dest: /etc/mysql/mariadb.conf.d/
     owner: root
     group: root
     mode: u=rw,g=r,o=r
 
-- name: Set MariaBackup custom template configuration all Environment
+- name: Set MariaBackup connection configuration
   template:
-    src: "my.cnf"
+    src: mariabackup/my.cnf
     dest: /root/mariabackup/
     owner: root
     group: root
     mode: u=rw,g=,o=
 
+- name: Override MariaDB systemd service configuration
+  copy:
+    src: mariadb_override.conf
+    dest: /etc/systemd/system/mariadb.service.d/override.conf
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify: reload-systemd
+
+- name: Set MariaDB NRPE configuration
+  copy:
+    src: nrpe/95-mariadb.cfg
+    dest: /etc/nagios/nrpe.d/
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify: restart-nrpe
+
 - name: Check if /var/lib/mysql/ exists
   stat:
     path: /var/lib/mysql/
@@ -210,16 +150,6 @@
       path: /var/lib/mysql/
       state: absent
 
-- name: Ensure required custom and Mariabackup files are copied to their destinations and root scripts
-  copy:
-    src: "{{ item.src }}"
-    dest: "{{ item.dest }}"
-    owner: root
-    group: root
-    mode: "{{ item.mode }}"
-  loop: "{{ required_files_and_mariabackup_files_and_root_scripts }}"
-  notify: reload-systemd
-
 - name: Mount all filesystems from /etc/fstab
   command: mount -a
-  when: fstab.changed
+  when: fstab.changed

roles/db/tasks/production.yml

@@ -0,0 +1,26 @@
+- name: Ensure production required directories exist
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ item.owner }}"
+    group: "{{ item.group }}"
+    mode: "{{ item.mode }}"
+  loop: "{{ required_directories_master }}"
+
+- name: Set production MariaDB custom configuration
+  copy:
+    src: conf/z95-production.cnf
+    dest: /etc/mysql/mariadb.conf.d/
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify: restart-mariadb
+
+- name: Reminder to check production mount points
+  debug:
+    msg: |
+      Remember to check the following mount points:
+      - /var/lib/mysql
+      - /mnt/mysqlbin
+      - /mnt/local-backup
+      Make sure they are correctly configured and accessible.

roles/db/tasks/test.yml

@@ -0,0 +1,39 @@
+- name: Set test Mariabackup files
+  copy:
+    src: mariabackup/bacula-after.sh
+    dest: /root/mariabackup/
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=rx
+
+- name: Set test MariaBackup apply config configuration
+  template:
+    src: mariabackup/apply.config.sh
+    dest: /root/mariabackup/
+    owner: root
+    group: root
+    mode: u=rw,g=,o=
+
+- name: Set test MariaBackup apply SQL script
+  copy:
+    src: mariabackup/apply.{{db.branch}}.sql
+    dest: /root/mariabackup/apply.sql
+    owner: root
+    group: root
+    mode: u=rw,g=,o=
+
+- name: Set test MariaDB custom configuration
+  copy:
+    src: conf/z92-test.cnf
+    dest: /etc/mysql/mariadb.conf.d/
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify: restart-mariadb
+
+- name: Reminder to check test mount points environment
+  debug:
+    msg: |
+      Remember to check the following mount points:
+      - /mnt/mysqltmp
+      Make sure they are correctly configured and accessible.

roles/db/templates/mariabackup/apply.config.sh

@@ -17,4 +17,3 @@ jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt
 
 # Jenkins job URL
 jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ db.branch }}
-

roles/db/templates/mariabackup/my.cnf

@@ -3,5 +3,5 @@ host = localhost
 user = mariabackup
 password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }}
 use-memory = 1G
-parallel = 2
-stream = mbstream
+parallel = 4
+stream = xbstream

roles/pve/handlers/main.yml

@@ -2,4 +2,3 @@
   service:
     name: nagios-nrpe-server
     state: restarted
-

run-playbook.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+MY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PY_VENV_DIR="$MY_DIR/venv"
 EXTRA_ARGS=()
 
 if [ -f .passbolt.yml ]; then
@@ -9,5 +11,11 @@ if [ -f .vault-pass ]; then
     EXTRA_ARGS+=("--vault-password-file" ".vault-pass")
 fi
 
-#export PYTHONPATH=./venv/lib/python3.12/site-packages/
+if [[ -z "$VIRTUAL_ENV" && -d "$PY_VENV_DIR" ]]; then
+    WARNING="\e[33m"
+    RESET="\e[0m"
+    echo -e "Warning: ${WARNING}Python virtual environment not activated! Some tasks might fail.${RESET}"
+    #export PYTHONPATH=./venv/lib/python3.12/site-packages/
+fi
+
 ansible-playbook ${EXTRA_ARGS[@]} $@