From ef9c1d54e1ea3b605c5c9484caa68c7f3a777fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Fri, 24 Jan 2025 10:33:40 +0100 Subject: [PATCH 01/17] refs #8414 - modify maribackup config files and templates --- roles/services/defaults/main.yaml | 3 +- .../files/mariabackup/bacula-after.sh | 51 +++++++++++++++++ .../services/files/mariabackuptest/apply.sql | 6 ++ roles/services/tasks/mariadb.yml | 56 ++++++++++++++++++- roles/services/templates/apply.cnf | 5 ++ roles/services/templates/apply.config-test.sh | 20 +++++++ .../{files/mariabackup => templates}/my.cnf | 4 +- 7 files changed, 141 insertions(+), 4 deletions(-) create mode 100755 roles/services/files/mariabackup/bacula-after.sh create mode 100644 roles/services/files/mariabackuptest/apply.sql create mode 100644 roles/services/templates/apply.cnf create mode 100755 roles/services/templates/apply.config-test.sh rename roles/services/{files/mariabackup => templates}/my.cnf (51%) diff --git a/roles/services/defaults/main.yaml b/roles/services/defaults/main.yaml index efbbcdd..fbf2d02 100644 --- a/roles/services/defaults/main.yaml +++ b/roles/services/defaults/main.yaml @@ -32,13 +32,14 @@ required_directories: - { path: /mnt/mysqlbin/binlog, owner: mysql, group: mysql, mode: 'u=rwx,g=,o=' } - { path: /root/scripts, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } - { path: /root/mariabackup, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } + - { path: /mnt/mysqldata/mysql, owner: mysql, group: mysql, mode: 'u=rwx,g=rx,o=rx' } required_files_and_mariabackup_files_and_root_scripts: - { src: "mariadb_override.conf", dest: "/etc/systemd/system/mariadb.service.d/override.conf", mode: "u=rw,g=r,o=r" } - { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "files/mariabackup/bacula-after.sh", dest: "/root/mariabackup/bacula-after.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" } - { src: "files/mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/mariabackup/my.cnf", dest: "/root/mariabackup/my.cnf", mode: "u=rw,g=,o=" } - { src: "files/mariabackup/restore-backup.sh", dest: "/root/mariabackup/restore-backup.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/scripts/check-memory.sh", dest: "/root/scripts/check-memory.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/scripts/export-privs.sh", dest: "/root/scripts/export-privs.sh", mode: "u=rwx,g=rx,o=rx" } diff --git a/roles/services/files/mariabackup/bacula-after.sh b/roles/services/files/mariabackup/bacula-after.sh new file mode 100755 index 0000000..53e37bb --- /dev/null +++ b/roles/services/files/mariabackup/bacula-after.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$myDir/config.sh" +. "$myDir/apply.config.sh" + +todayDir=$(date +%Y-%m-%d) +pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz" +files=($pattern) +backupFile="${files[0]}" + +"$myDir/restore-backup.sh" "$backupFile" +rm -r "$baculaDir" + +if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then + for node in "${dbClusterSiblings[@]}"; do + ssh root@$node service mysql stop + ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi" + done + + galera_new_cluster +else + service mariadb start +fi + +echo "Applying custom script." +mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()" +mysql < "$myDir/apply.sql" + +echo "Upgrading tables." +mysql_upgrade + +echo "Applying repository changes." +curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec" + +echo "Waiting for Jenkins job to end." +jobResult=null +while [ "$jobResult" = "null" ]; do + sleep 10 + jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result") +done +echo "Job result: $jobResult" + +echo "Promoting to master." +"/root/scripts/promote-master.sh" + +for node in "${dbClusterSiblings[@]}"; do + ssh root@$node service mysql start +done + diff --git a/roles/services/files/mariabackuptest/apply.sql b/roles/services/files/mariabackuptest/apply.sql new file mode 100644 index 0000000..b1b26d3 --- /dev/null +++ b/roles/services/files/mariabackuptest/apply.sql @@ -0,0 +1,6 @@ +UPDATE vn2008.tblContadores + SET dbproduccion = FALSE; + +DELETE FROM util.binlogQueue; + +UPDATE `account`.`user` SET `active` = TRUE WHERE `name` = 'mindshore'; diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 8866ad9..e51815e 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -124,6 +124,60 @@ - "files/mariaconfigfiles/z90-vn.cnf" - "files/mariaconfigfiles/z92-testing.cnf" notify: restart-mariadb + + # - name: Extracting passwords + # no_log: true + # set_fact: + # jenkins_token: "{{ lookup(passbolt, jenkins_user_name, folder_parent_id=passbolt_folder).password }}" + + # - name: Debug variable + # debug: + # msg: "{{ jenkins_token }}" + + - name: Set MariaBackup custom configuration Test + template: + src: "apply.config-test.sh" + dest: /root/mariabackup/apply.config.sh + owner: root + group: root + mode: u=rw,g=r,o=r + + - name: Extracting dbusername + set_fact: + dbusername: "{{ lookup(passbolt, dbuser, folder_parent_id=passbolt_folder).username }}" + + - name: Extracting dbpassword + no_log: true + set_fact: + dbpass: "{{ lookup(passbolt, dbuser, folder_parent_id=passbolt_folder).password }}" + + - name: Extracting MariaBackup UserName + set_fact: + mariabackupusernametpl: "{{ lookup(passbolt, mariabackupusername, folder_parent_id=passbolt_folder).username }}" + + - name: Extracting MariaBackup Password + no_log: true + set_fact: + mariabackuppasstpl: "{{ lookup(passbolt, mariabackupusername, folder_parent_id=passbolt_folder).password }}" + + - name: Set MariaBackup custom template configuration Test + template: + src: "{{ item }}" + dest: /root/mariabackup/ + owner: root + group: root + mode: u=rw,g=,o= + with_items: + - "apply.cnf" + - "my.cnf" + + - name: Set MariaDB local configuration file + copy: + src: files/mariabackuptest/apply.sql + dest: /root/mariabackup/ + owner: root + group: root + mode: u=rw,g=r,o=r - name: Reminder to check mount points debug: @@ -132,7 +186,7 @@ - /mnt/mysqltmp Make sure they are correctly configured and accessible. -- name: Set MariaDB local configuration file +- name: template: src: templates/z99-local.cnf dest: /etc/mysql/mariadb.conf.d/ diff --git a/roles/services/templates/apply.cnf b/roles/services/templates/apply.cnf new file mode 100644 index 0000000..0e4ce4f --- /dev/null +++ b/roles/services/templates/apply.cnf @@ -0,0 +1,5 @@ +[client] +host = localhost +user = {{ dbusername }} +password = {{ dbpass }} +socket = /var/run/mysqld/mysqld.sock diff --git a/roles/services/templates/apply.config-test.sh b/roles/services/templates/apply.config-test.sh new file mode 100755 index 0000000..0eab0a8 --- /dev/null +++ b/roles/services/templates/apply.config-test.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Bacula directory for restore +baculaDir=/mnt/mysqldata/bacula-restore + +# Database branch name +dbBranch=test + +# Database environment +dbEnvironment=test + +# MariaDB cluster sibling node hostnames +dbClusterSiblings=() + +# Jenkins authentication string +jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt_folder).password }} + +# Jenkins job URL +jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-test + diff --git a/roles/services/files/mariabackup/my.cnf b/roles/services/templates/my.cnf similarity index 51% rename from roles/services/files/mariabackup/my.cnf rename to roles/services/templates/my.cnf index 14edb89..12effdb 100644 --- a/roles/services/files/mariabackup/my.cnf +++ b/roles/services/templates/my.cnf @@ -1,7 +1,7 @@ [mariabackup] host = localhost -user = {{ user_mariabackup }} -password = {{ password_user_mariabackup }} +user = {{ mariabackupusernametpl }} +password = {{ mariabackuppasstpl }} use-memory = 1G parallel = 2 stream = mbstream -- 2.40.1 From e84cfe3853b6a200f26b4a6e3ad60b3a22cc1337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Fri, 24 Jan 2025 12:42:29 +0100 Subject: [PATCH 02/17] refs 8414-change_ca_verdnatura --- roles/services/defaults/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/services/defaults/main.yaml b/roles/services/defaults/main.yaml index fbf2d02..cd6c649 100644 --- a/roles/services/defaults/main.yaml +++ b/roles/services/defaults/main.yaml @@ -23,7 +23,7 @@ mariadb_requeriments: - curl - apt-transport-https certificates: - - { content: '{{ ca_mysql }}', dest: '/etc/mysql/ca.pem', mode: 'u=rw,g=r,o=r' } + - { content: '{{ ca_verdnatura_deprecated }}', dest: '/etc/mysql/ca.pem', mode: 'u=rw,g=r,o=r' } - { content: '{{ cert_mysql }}', dest: '/etc/mysql/cert.pem', mode: 'u=rw,g=r,o=r' } required_directories: - { path: /mnt/local-backup, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } -- 2.40.1 From 89c18a0e6dfaf1539fd9d62c3a427c2e3cde5b97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Fri, 24 Jan 2025 13:09:58 +0100 Subject: [PATCH 03/17] refs 8414-move_key_cert --- roles/services/defaults/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/services/defaults/main.yaml b/roles/services/defaults/main.yaml index cd6c649..2c20952 100644 --- a/roles/services/defaults/main.yaml +++ b/roles/services/defaults/main.yaml @@ -25,6 +25,7 @@ mariadb_requeriments: certificates: - { content: '{{ ca_verdnatura_deprecated }}', dest: '/etc/mysql/ca.pem', mode: 'u=rw,g=r,o=r' } - { content: '{{ cert_mysql }}', dest: '/etc/mysql/cert.pem', mode: 'u=rw,g=r,o=r' } + - { content: '{{ cert_mysql_key }}', dest: '/etc/mysql/key.pem', mode: 'u=rw,g=,o=' } required_directories: - { path: /mnt/local-backup, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } - { path: /mnt/mysqlbin, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } -- 2.40.1 From 4d70766a41f97ab532d33f221b9a22febb73227e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Fri, 24 Jan 2025 14:49:49 +0100 Subject: [PATCH 04/17] refs #8414 - Refactor extract playbook variables and secrets everywhere in TV Screen --- roles/services/tasks/mariadb.yml | 65 +++++-------------- roles/services/templates/apply.cnf | 5 -- roles/services/templates/my.cnf | 4 +- .../apply.config.sh} | 0 4 files changed, 17 insertions(+), 57 deletions(-) delete mode 100644 roles/services/templates/apply.cnf rename roles/services/templates/{apply.config-test.sh => test-db/apply.config.sh} (100%) diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index e51815e..486d1c3 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -62,13 +62,13 @@ command: mount -a when: fstab.changed -- name: Set MariaDB Cron to /etc/cron.d - template: - src: templates/cron_mariadb - dest: /etc/cron.d/vn - owner: root - group: root - mode: u=rw,g=r,o=r +# - name: Set MariaDB Cron to /etc/cron.d +# template: +# src: templates/cron_mariadb +# dest: /etc/cron.d/vn +# owner: root +# group: root +# mode: u=rw,g=r,o=r - name: Insert MySQL certificates copy: @@ -80,14 +80,6 @@ loop: "{{ certificates }}" notify: restart-mariadb -- name: Configure MySQL master cert - copy: - content: "{{ lookup(passbolt, 'private_mysql', folder_parent_id=passbolt_folder).description }}" - dest: /etc/mysql/key.pem - owner: mysql - group: mysql - mode: u=rw,g=,o= - - when: production is true block: - name: Set MariaDB custom configuration Production @@ -125,41 +117,14 @@ - "files/mariaconfigfiles/z92-testing.cnf" notify: restart-mariadb - # - name: Extracting passwords - # no_log: true - # set_fact: - # jenkins_token: "{{ lookup(passbolt, jenkins_user_name, folder_parent_id=passbolt_folder).password }}" - - # - name: Debug variable - # debug: - # msg: "{{ jenkins_token }}" - - - name: Set MariaBackup custom configuration Test - template: - src: "apply.config-test.sh" - dest: /root/mariabackup/apply.config.sh - owner: root - group: root - mode: u=rw,g=r,o=r + # - name: Set MariaBackup custom configuration Test + # template: + # src: "apply.config-test.sh" + # dest: /root/mariabackup/apply.config.sh + # owner: root + # group: root + # mode: u=rw,g=r,o=r - - name: Extracting dbusername - set_fact: - dbusername: "{{ lookup(passbolt, dbuser, folder_parent_id=passbolt_folder).username }}" - - - name: Extracting dbpassword - no_log: true - set_fact: - dbpass: "{{ lookup(passbolt, dbuser, folder_parent_id=passbolt_folder).password }}" - - - name: Extracting MariaBackup UserName - set_fact: - mariabackupusernametpl: "{{ lookup(passbolt, mariabackupusername, folder_parent_id=passbolt_folder).username }}" - - - name: Extracting MariaBackup Password - no_log: true - set_fact: - mariabackuppasstpl: "{{ lookup(passbolt, mariabackupusername, folder_parent_id=passbolt_folder).password }}" - - name: Set MariaBackup custom template configuration Test template: src: "{{ item }}" @@ -168,7 +133,7 @@ group: root mode: u=rw,g=,o= with_items: - - "apply.cnf" + - "test-db/apply.config.sh" - "my.cnf" - name: Set MariaDB local configuration file diff --git a/roles/services/templates/apply.cnf b/roles/services/templates/apply.cnf deleted file mode 100644 index 0e4ce4f..0000000 --- a/roles/services/templates/apply.cnf +++ /dev/null @@ -1,5 +0,0 @@ -[client] -host = localhost -user = {{ dbusername }} -password = {{ dbpass }} -socket = /var/run/mysqld/mysqld.sock diff --git a/roles/services/templates/my.cnf b/roles/services/templates/my.cnf index 12effdb..b1b4ac9 100644 --- a/roles/services/templates/my.cnf +++ b/roles/services/templates/my.cnf @@ -1,7 +1,7 @@ [mariabackup] host = localhost -user = {{ mariabackupusernametpl }} -password = {{ mariabackuppasstpl }} +user = mariabackup +password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }} use-memory = 1G parallel = 2 stream = mbstream diff --git a/roles/services/templates/apply.config-test.sh b/roles/services/templates/test-db/apply.config.sh similarity index 100% rename from roles/services/templates/apply.config-test.sh rename to roles/services/templates/test-db/apply.config.sh -- 2.40.1 From b0911c4b5e6e7420770502d01a81b8544ea436af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Mon, 27 Jan 2025 09:03:52 +0100 Subject: [PATCH 05/17] refs 8414-change ca_verdnatura_deprecated variable --- roles/services/defaults/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/services/defaults/main.yaml b/roles/services/defaults/main.yaml index 2c20952..8784d55 100644 --- a/roles/services/defaults/main.yaml +++ b/roles/services/defaults/main.yaml @@ -23,7 +23,7 @@ mariadb_requeriments: - curl - apt-transport-https certificates: - - { content: '{{ ca_verdnatura_deprecated }}', dest: '/etc/mysql/ca.pem', mode: 'u=rw,g=r,o=r' } + - { content: '{{ ca_company_deprecated }}', dest: '/etc/mysql/ca.pem', mode: 'u=rw,g=r,o=r' } - { content: '{{ cert_mysql }}', dest: '/etc/mysql/cert.pem', mode: 'u=rw,g=r,o=r' } - { content: '{{ cert_mysql_key }}', dest: '/etc/mysql/key.pem', mode: 'u=rw,g=,o=' } required_directories: -- 2.40.1 From 610d17b51da42c1c74cd74c5d483cfb6f8f80789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Mon, 27 Jan 2025 11:46:15 +0100 Subject: [PATCH 06/17] refs 8414-Refactor --- roles/services/tasks/mariadb.yml | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 486d1c3..3513f54 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -117,26 +117,15 @@ - "files/mariaconfigfiles/z92-testing.cnf" notify: restart-mariadb - # - name: Set MariaBackup custom configuration Test - # template: - # src: "apply.config-test.sh" - # dest: /root/mariabackup/apply.config.sh - # owner: root - # group: root - # mode: u=rw,g=r,o=r - - name: Set MariaBackup custom template configuration Test template: - src: "{{ item }}" + src: test-db/apply.config.sh dest: /root/mariabackup/ owner: root group: root mode: u=rw,g=,o= - with_items: - - "test-db/apply.config.sh" - - "my.cnf" - - name: Set MariaDB local configuration file + - name: Set MariaDB local configuration file for test-db copy: src: files/mariabackuptest/apply.sql dest: /root/mariabackup/ @@ -153,8 +142,16 @@ - name: template: - src: templates/z99-local.cnf + src: z99-local.cnf dest: /etc/mysql/mariadb.conf.d/ owner: root group: root - mode: u=rw,g=r,o=r \ No newline at end of file + mode: u=rw,g=r,o=r + +- name: Set MariaBackup custom template configuration Test + template: + src: "my.cnf" + dest: /root/mariabackup/ + owner: root + group: root + mode: u=rw,g=,o= -- 2.40.1 From cb7b75392c4761d67b150ffae9d8f811ce7df166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Mon, 27 Jan 2025 13:47:27 +0100 Subject: [PATCH 07/17] refs #8414 - Minor changes in description and paths --- roles/services/tasks/mariadb.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 3513f54..d4861bb 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -90,8 +90,8 @@ group: root mode: u=rw,g=r,o=r with_items: - - "files/mariaconfigfiles/z90-vn.cnf" - - "files/mariaconfigfiles/z95-production" + - "mariaconfigfiles/z90-vn.cnf" + - "mariaconfigfiles/z95-production.cnf" notify: restart-mariadb - name: Reminder to check mount points @@ -133,7 +133,7 @@ group: root mode: u=rw,g=r,o=r - - name: Reminder to check mount points + - name: Reminder to check mount points environment test debug: msg: | Remember to check the following mount points: -- 2.40.1 From 7ec9b463eee1e9c024dcecd799929c0e5f446706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Mon, 27 Jan 2025 14:22:46 +0100 Subject: [PATCH 08/17] refs #8414 - Minor changes in description Name --- roles/services/tasks/mariadb.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index d4861bb..1e6b3ea 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -140,7 +140,7 @@ - /mnt/mysqltmp Make sure they are correctly configured and accessible. -- name: +- name: Set Custom Configuration local template template: src: z99-local.cnf dest: /etc/mysql/mariadb.conf.d/ -- 2.40.1 From 315280665ac6b183c9a4712075e2099b9b9a5ef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Tue, 28 Jan 2025 11:17:39 +0100 Subject: [PATCH 09/17] =?UTF-8?q?refs=20#8414=20-=20Add=20No=20Log=20=3D?= =?UTF-8?q?=20true=20to=20don=C2=B4t=20see=20private=20certificates=20in?= =?UTF-8?q?=20stdout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/services/tasks/mariadb.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 1e6b3ea..7c2c430 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -71,6 +71,7 @@ # mode: u=rw,g=r,o=r - name: Insert MySQL certificates + no_log: true copy: content: "{{ item.content }}" dest: "{{ item.dest }}" -- 2.40.1 From e95b64e96f27758f197f3cba0ad436f79c66f007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Tue, 28 Jan 2025 12:15:42 +0100 Subject: [PATCH 10/17] refs #8414 - Better define in Name sections to know environment --- roles/services/tasks/mariadb.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 7c2c430..41d6aef 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -106,7 +106,7 @@ - when: testdb is true block: - - name: Set MariaDB custom configuration Test + - name: Set MariaDB custom configuration Test-DB copy: src: "{{ item }}" dest: /etc/mysql/mariadb.conf.d/ @@ -118,7 +118,7 @@ - "files/mariaconfigfiles/z92-testing.cnf" notify: restart-mariadb - - name: Set MariaBackup custom template configuration Test + - name: Set MariaBackup custom template configuration Test-DB template: src: test-db/apply.config.sh dest: /root/mariabackup/ @@ -126,7 +126,7 @@ group: root mode: u=rw,g=,o= - - name: Set MariaDB local configuration file for test-db + - name: Set MariaDB local configuration file for Test-DB copy: src: files/mariabackuptest/apply.sql dest: /root/mariabackup/ @@ -134,14 +134,14 @@ group: root mode: u=rw,g=r,o=r - - name: Reminder to check mount points environment test + - name: Reminder to check mount points environment Test-DB debug: msg: | Remember to check the following mount points: - /mnt/mysqltmp Make sure they are correctly configured and accessible. -- name: Set Custom Configuration local template +- name: Set Custom Configuration local template all Environment template: src: z99-local.cnf dest: /etc/mysql/mariadb.conf.d/ @@ -149,7 +149,7 @@ group: root mode: u=rw,g=r,o=r -- name: Set MariaBackup custom template configuration Test +- name: Set MariaBackup custom template configuration all Environment template: src: "my.cnf" dest: /root/mariabackup/ -- 2.40.1 From 813c404b197b9a338008954a660957da8d87ebda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Tue, 28 Jan 2025 14:23:00 +0100 Subject: [PATCH 11/17] refs #8414 - Add dev environment --- .../{mariabackuptest => mariabackuptest_dev}/apply.sql | 0 .../{mariabackup => mariabackuptest_dev}/bacula-after.sh | 0 .../{z92-testing.cnf => z92-testing_dev.cnf} | 0 roles/services/tasks/mariadb.yml | 9 +++++---- .../templates/{test-db => test-db_dev}/apply.config.sh | 6 +++--- 5 files changed, 8 insertions(+), 7 deletions(-) rename roles/services/files/{mariabackuptest => mariabackuptest_dev}/apply.sql (100%) rename roles/services/files/{mariabackup => mariabackuptest_dev}/bacula-after.sh (100%) rename roles/services/files/mariaconfigfiles/{z92-testing.cnf => z92-testing_dev.cnf} (100%) rename roles/services/templates/{test-db => test-db_dev}/apply.config.sh (82%) diff --git a/roles/services/files/mariabackuptest/apply.sql b/roles/services/files/mariabackuptest_dev/apply.sql similarity index 100% rename from roles/services/files/mariabackuptest/apply.sql rename to roles/services/files/mariabackuptest_dev/apply.sql diff --git a/roles/services/files/mariabackup/bacula-after.sh b/roles/services/files/mariabackuptest_dev/bacula-after.sh similarity index 100% rename from roles/services/files/mariabackup/bacula-after.sh rename to roles/services/files/mariabackuptest_dev/bacula-after.sh diff --git a/roles/services/files/mariaconfigfiles/z92-testing.cnf b/roles/services/files/mariaconfigfiles/z92-testing_dev.cnf similarity index 100% rename from roles/services/files/mariaconfigfiles/z92-testing.cnf rename to roles/services/files/mariaconfigfiles/z92-testing_dev.cnf diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 41d6aef..225140c 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -104,7 +104,8 @@ - /mnt/local-backup Make sure they are correctly configured and accessible. -- when: testdb is true +- when: testdb is true or dev is true + block: - name: Set MariaDB custom configuration Test-DB copy: @@ -115,12 +116,12 @@ mode: u=rw,g=r,o=r with_items: - "files/mariaconfigfiles/z90-vn.cnf" - - "files/mariaconfigfiles/z92-testing.cnf" + - "files/mariaconfigfiles/z92-testing_dev.cnf" notify: restart-mariadb - name: Set MariaBackup custom template configuration Test-DB template: - src: test-db/apply.config.sh + src: test-db_dev/apply.config.sh dest: /root/mariabackup/ owner: root group: root @@ -128,7 +129,7 @@ - name: Set MariaDB local configuration file for Test-DB copy: - src: files/mariabackuptest/apply.sql + src: files/mariabackuptest_dev/apply.sql dest: /root/mariabackup/ owner: root group: root diff --git a/roles/services/templates/test-db/apply.config.sh b/roles/services/templates/test-db_dev/apply.config.sh similarity index 82% rename from roles/services/templates/test-db/apply.config.sh rename to roles/services/templates/test-db_dev/apply.config.sh index 0eab0a8..f8cb310 100755 --- a/roles/services/templates/test-db/apply.config.sh +++ b/roles/services/templates/test-db_dev/apply.config.sh @@ -4,10 +4,10 @@ baculaDir=/mnt/mysqldata/bacula-restore # Database branch name -dbBranch=test +dbBranch={{ environment }} # Database environment -dbEnvironment=test +dbEnvironment={{ environment }} # MariaDB cluster sibling node hostnames dbClusterSiblings=() @@ -16,5 +16,5 @@ dbClusterSiblings=() jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt_folder).password }} # Jenkins job URL -jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-test +jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ environment }} -- 2.40.1 From 994447329c108cc0b0b21e29dc77e7171a863842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Tue, 28 Jan 2025 15:52:29 +0100 Subject: [PATCH 12/17] refs #8414 - Refactor paths --- roles/services/defaults/main.yaml | 4 +++- roles/services/tasks/mariadb.yml | 12 ++++++------ roles/services/templates/test-db_dev/apply.config.sh | 6 +++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/roles/services/defaults/main.yaml b/roles/services/defaults/main.yaml index 8784d55..0eaff75 100644 --- a/roles/services/defaults/main.yaml +++ b/roles/services/defaults/main.yaml @@ -34,10 +34,12 @@ required_directories: - { path: /root/scripts, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } - { path: /root/mariabackup, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } - { path: /mnt/mysqldata/mysql, owner: mysql, group: mysql, mode: 'u=rwx,g=rx,o=rx' } + - { path: /etc/systemd/system/mariadb.service.d, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } + required_files_and_mariabackup_files_and_root_scripts: - { src: "mariadb_override.conf", dest: "/etc/systemd/system/mariadb.service.d/override.conf", mode: "u=rw,g=r,o=r" } - { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/mariabackup/bacula-after.sh", dest: "/root/mariabackup/bacula-after.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "files/mariabackuptest_dev/bacula-after.sh", dest: "/root/mariabackup/bacula-after.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" } - { src: "files/mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" } diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 225140c..b831a5c 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -71,7 +71,7 @@ # mode: u=rw,g=r,o=r - name: Insert MySQL certificates - no_log: true + #no_log: true copy: content: "{{ item.content }}" dest: "{{ item.dest }}" @@ -104,10 +104,10 @@ - /mnt/local-backup Make sure they are correctly configured and accessible. -- when: testdb is true or dev is true +- when: production is false block: - - name: Set MariaDB custom configuration Test-DB + - name: Set MariaDB custom configuration Test-DB-DEV copy: src: "{{ item }}" dest: /etc/mysql/mariadb.conf.d/ @@ -119,7 +119,7 @@ - "files/mariaconfigfiles/z92-testing_dev.cnf" notify: restart-mariadb - - name: Set MariaBackup custom template configuration Test-DB + - name: Set MariaBackup custom template configuration Test-DB-DEV template: src: test-db_dev/apply.config.sh dest: /root/mariabackup/ @@ -127,7 +127,7 @@ group: root mode: u=rw,g=,o= - - name: Set MariaDB local configuration file for Test-DB + - name: Set MariaDB local configuration file for Test-DB-DEV copy: src: files/mariabackuptest_dev/apply.sql dest: /root/mariabackup/ @@ -135,7 +135,7 @@ group: root mode: u=rw,g=r,o=r - - name: Reminder to check mount points environment Test-DB + - name: Reminder to check mount points environment Test-DB-DEV debug: msg: | Remember to check the following mount points: diff --git a/roles/services/templates/test-db_dev/apply.config.sh b/roles/services/templates/test-db_dev/apply.config.sh index f8cb310..8ac2677 100755 --- a/roles/services/templates/test-db_dev/apply.config.sh +++ b/roles/services/templates/test-db_dev/apply.config.sh @@ -4,10 +4,10 @@ baculaDir=/mnt/mysqldata/bacula-restore # Database branch name -dbBranch={{ environment }} +dbBranch={{ entorno }} # Database environment -dbEnvironment={{ environment }} +dbEnvironment={{ entorno }} # MariaDB cluster sibling node hostnames dbClusterSiblings=() @@ -16,5 +16,5 @@ dbClusterSiblings=() jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt_folder).password }} # Jenkins job URL -jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ environment }} +jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ entorno }} -- 2.40.1 From d1ec1bc57be7844b10d6b6c10a70d075c1c621a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Wed, 29 Jan 2025 09:51:30 +0100 Subject: [PATCH 13/17] =?UTF-8?q?refs=20#8414=20-=20Rsync=20to=20dont?= =?UTF-8?q?=C2=B4t=20fail=20at=20the=20end?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/services/tasks/mariadb.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index b831a5c..a10f5ea 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -71,7 +71,7 @@ # mode: u=rw,g=r,o=r - name: Insert MySQL certificates - #no_log: true + no_log: true copy: content: "{{ item.content }}" dest: "{{ item.dest }}" @@ -104,7 +104,7 @@ - /mnt/local-backup Make sure they are correctly configured and accessible. -- when: production is false +- when: production is false or production is not defined block: - name: Set MariaDB custom configuration Test-DB-DEV @@ -157,3 +157,26 @@ owner: root group: root mode: u=rw,g=,o= + +- name: Check if /var/lib/mysql/ exists + stat: + path: /var/lib/mysql/ + register: mysql_dir + + +- when: mysql_dir.stat.exists + block: + + - name: Sync MySQL data directory + synchronize: + src: /var/lib/mysql/ + dest: /mnt/mysqldata/mysql/ + archive: true + compress: true + recursive: true + delegate_to: "{{ inventory_hostname }}" + + - name: Remove old MySQL data after sync + file: + path: /var/lib/mysql/ + state: absent -- 2.40.1 From fe27a4b2e6a3f1c8af21ce0a0c566e674aec41ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Wed, 29 Jan 2025 10:24:14 +0100 Subject: [PATCH 14/17] refs #8414 - adjust bacula-after.sh only in non production --- roles/services/defaults/main.yaml | 3 +-- roles/services/tasks/mariadb.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/services/defaults/main.yaml b/roles/services/defaults/main.yaml index 0eaff75..9e977f2 100644 --- a/roles/services/defaults/main.yaml +++ b/roles/services/defaults/main.yaml @@ -38,8 +38,7 @@ required_directories: required_files_and_mariabackup_files_and_root_scripts: - { src: "mariadb_override.conf", dest: "/etc/systemd/system/mariadb.service.d/override.conf", mode: "u=rw,g=r,o=r" } - - { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/mariabackuptest_dev/bacula-after.sh", dest: "/root/mariabackup/bacula-after.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" } - { src: "files/mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" } diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index a10f5ea..80c5f3f 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -129,7 +129,7 @@ - name: Set MariaDB local configuration file for Test-DB-DEV copy: - src: files/mariabackuptest_dev/apply.sql + src: files/mariabackuptest_dev/* dest: /root/mariabackup/ owner: root group: root -- 2.40.1 From 73a32dd8e6b399b3a0012c35080575955124373f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Wed, 29 Jan 2025 11:14:10 +0100 Subject: [PATCH 15/17] refs #8414 - minnor changes --- roles/services/handlers/main.yml | 2 +- roles/services/tasks/mariadb.yml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/roles/services/handlers/main.yml b/roles/services/handlers/main.yml index 77780d8..aa291cc 100644 --- a/roles/services/handlers/main.yml +++ b/roles/services/handlers/main.yml @@ -2,7 +2,7 @@ systemd: name: chrony state: restarted -- name: reload systemd +- name: reload-systemd command: cmd: systemctl daemon-reload - name: restart-mariadb diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 80c5f3f..35b97c6 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -48,7 +48,7 @@ group: root mode: "{{ item.mode }}" loop: "{{ required_files_and_mariabackup_files_and_root_scripts }}" - notify: reload systemd + notify: reload-systemd - name: Add tmpfs in /etc/fstab blockinfile: @@ -129,11 +129,14 @@ - name: Set MariaDB local configuration file for Test-DB-DEV copy: - src: files/mariabackuptest_dev/* + src: "{{ item }}" dest: /root/mariabackup/ owner: root group: root mode: u=rw,g=r,o=r + with_items: + - "mariabackuptest_dev/apply.sql" + - "mariabackuptest_dev/bacula-after.sh" - name: Reminder to check mount points environment Test-DB-DEV debug: -- 2.40.1 From 335dbfa3a47fe7672709324137e8621866d34739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Wed, 29 Jan 2025 15:51:49 +0100 Subject: [PATCH 16/17] refs #8142 - Minor changes - Check differences in files --- roles/services/defaults/main.yaml | 3 +-- roles/services/files/scripts/sync-conf.sh | 21 ------------------- roles/services/tasks/mariadb.yml | 2 +- .../templates/test-db_dev/apply.config.sh | 6 +++--- 4 files changed, 5 insertions(+), 27 deletions(-) delete mode 100755 roles/services/files/scripts/sync-conf.sh diff --git a/roles/services/defaults/main.yaml b/roles/services/defaults/main.yaml index 9e977f2..77fbac6 100644 --- a/roles/services/defaults/main.yaml +++ b/roles/services/defaults/main.yaml @@ -49,8 +49,7 @@ required_files_and_mariabackup_files_and_root_scripts: - { src: "files/scripts/promote-master.sh", dest: "/root/scripts/promote-master.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/scripts/promote-slave.sh", dest: "/root/scripts/promote-slave.sh", mode: "u=rwx,g=rx,o=rx" } - { src: "files/scripts/README.md", dest: "/root/scripts/README.md", mode: "u=rw,g=r,o=r" } - - { src: "files/scripts/scheduler-log.sh", dest: "/root/scripts/scheduler-log.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/scripts/sync-conf.sh", dest: "/root/scripts/sync-conf.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "files/scripts/scheduler-log.sh", dest: "/root/scripts/scheduler-log.sh", mode: "u=rwx,g=rx,o=rx" } downloads: - { url: "https://r.mariadb.com/downloads/mariadb_repo_setup", dest: "/tmp/mariadb_repo_setup", mode: "u=rwx,g=rx,o=rx" } - { url: "https://repo.percona.com/apt/percona-release_latest.generic_all.deb", dest: "/tmp/percona-release_latest.generic_all.deb", mode: "u=rw,g=r,o=r" } diff --git a/roles/services/files/scripts/sync-conf.sh b/roles/services/files/scripts/sync-conf.sh deleted file mode 100755 index 3597234..0000000 --- a/roles/services/files/scripts/sync-conf.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -partner=root@db2.static.verdnatura.es -confDir=/etc/mysql/mariadb.conf.d -files=( - z90-vn.cnf - z95-production.cnf -) - -#echo "Reloading service." -#service mariadb reload - -if [ $? -eq "0" ]; then - echo "Synchronizing partner configuration." - for file in "${files[@]}"; do - scp "$confDir/$file" $partner:$confDir - done - - #echo "Reloading partner service." - #ssh $partner service mariadb reload -fi diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 35b97c6..5049a1d 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -55,7 +55,7 @@ path: /etc/fstab marker: "# {mark} ANSIBLE-MANAGED TMPFS ENTRY" block: | - tmpfs /mnt/mysqltmp tmpfs rw,size=6144M 0 0 + tmpfs /mnt/mysqltmp tmpfs rw,size={{ mysqltmpsize }} 0 0 register: fstab - name: Mount all filesystems from /etc/fstab diff --git a/roles/services/templates/test-db_dev/apply.config.sh b/roles/services/templates/test-db_dev/apply.config.sh index 8ac2677..6894c3f 100755 --- a/roles/services/templates/test-db_dev/apply.config.sh +++ b/roles/services/templates/test-db_dev/apply.config.sh @@ -4,10 +4,10 @@ baculaDir=/mnt/mysqldata/bacula-restore # Database branch name -dbBranch={{ entorno }} +dbBranch={{ dbBranch }} # Database environment -dbEnvironment={{ entorno }} +dbEnvironment={{ dbEnvironment }} # MariaDB cluster sibling node hostnames dbClusterSiblings=() @@ -16,5 +16,5 @@ dbClusterSiblings=() jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt_folder).password }} # Jenkins job URL -jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ entorno }} +jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ dbBranch }} -- 2.40.1 From 0d5a6d96ca380293a86fc61a94fa6421c1cefced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= Date: Wed, 29 Jan 2025 15:57:20 +0100 Subject: [PATCH 17/17] refs #8142 - Uncomment cron --- roles/services/tasks/mariadb.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/roles/services/tasks/mariadb.yml b/roles/services/tasks/mariadb.yml index 5049a1d..398d6b1 100644 --- a/roles/services/tasks/mariadb.yml +++ b/roles/services/tasks/mariadb.yml @@ -62,13 +62,13 @@ command: mount -a when: fstab.changed -# - name: Set MariaDB Cron to /etc/cron.d -# template: -# src: templates/cron_mariadb -# dest: /etc/cron.d/vn -# owner: root -# group: root -# mode: u=rw,g=r,o=r +- name: Set MariaDB Cron to /etc/cron.d + template: + src: templates/cron_mariadb + dest: /etc/cron.d/vn + owner: root + group: root + mode: u=rw,g=r,o=r - name: Insert MySQL certificates no_log: true -- 2.40.1