From 4bee560a8d6f065adcae01139a33db6031d825c2 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Mon, 10 Feb 2025 13:38:40 +0100 Subject: [PATCH 01/12] db: refs #8140 Fix apply SQL, dir structure changes --- roles/db/files/conf/z90-vn.cnf | 98 +++++++++++++++++++ roles/db/files/conf/z92-test.cnf | 14 +++ roles/db/files/conf/z95-production.cnf | 29 ++++++ .../files/mariabackup/apply.development.sql | 23 +++++ roles/db/files/mariabackup/apply.test.sql | 6 ++ roles/db/files/mariabackup/bacula-after.sh | 51 ++++++++++ roles/db/files/nrpe/95-mariadb.cfg | 4 + roles/db/tasks/production.yml | 26 +++++ roles/db/tasks/test.yml | 39 ++++++++ roles/db/templates/conf/z99-local.cnf | 7 ++ .../db/templates/mariabackup/apply.config.sh | 19 ++++ roles/db/templates/mariabackup/my.cnf | 7 ++ 12 files changed, 323 insertions(+) create mode 100644 roles/db/files/conf/z90-vn.cnf create mode 100644 roles/db/files/conf/z92-test.cnf create mode 100644 roles/db/files/conf/z95-production.cnf create mode 100644 roles/db/files/mariabackup/apply.development.sql create mode 100644 roles/db/files/mariabackup/apply.test.sql create mode 100755 roles/db/files/mariabackup/bacula-after.sh create mode 100644 roles/db/files/nrpe/95-mariadb.cfg create mode 100644 roles/db/tasks/production.yml create mode 100644 roles/db/tasks/test.yml create mode 100644 roles/db/templates/conf/z99-local.cnf create mode 100755 roles/db/templates/mariabackup/apply.config.sh create mode 100644 roles/db/templates/mariabackup/my.cnf diff --git a/roles/db/files/conf/z90-vn.cnf b/roles/db/files/conf/z90-vn.cnf new file mode 100644 index 0000000..1911135 --- /dev/null +++ b/roles/db/files/conf/z90-vn.cnf @@ -0,0 +1,98 @@ +[mysqld] +# Docs: https://mariadb.com/kb/en/server-system-variables + +lc_messages = es_ES +lc_time_names = es_ES +character-set-server = utf8 +collation-server = utf8_unicode_ci +explicit_defaults_for_timestamp = ON +datadir = /mnt/mysqldata/mysql +tmpdir = /mnt/mysqltmp +log_bin_trust_function_creators = 1 +sql_mode = NO_ENGINE_SUBSTITUTION +bind-address = 0.0.0.0 +max_password_errors = 50 + +#++++++++++++++++++++++++++++++++++++++++ Threads + +thread_stack = 512K +join_buffer_size = 2M +sort_buffer_size = 4M +net_buffer_length = 256K +max_allowed_packet = 16M +read_buffer_size = 1M +read_rnd_buffer_size = 512K + +#++++++++++++++++++++++++++++++++++++++++ Performance + +thread_cache_size = 450 +interactive_timeout = 1800 +wait_timeout = 1800 +open_files_limit = 20000 +low_priority_updates = 1 +table_open_cache = 40000 +table_definition_cache = 10000 +table_open_cache_instances = 1 +key_buffer_size = 256K +max_heap_table_size = 128M +tmp_table_size = 128M +concurrent_insert = ALWAYS +group_concat_max_len = 10000 +max_connect_errors = 50 + +#++++++++++++++++++++++++++++++++++++++++ Binary log + +max_binlog_size = 1GB +binlog_cache_size = 16M +binlog_stmt_cache_size = 16M +binlog_row_image = noblob +binlog_format = row + +#++++++++++++++++++++++++++++++++++++++++ InnoDB + +transaction-isolation = READ-COMMITTED +idle_transaction_timeout = 60 +innodb_io_capacity = 100 +innodb_io_capacity_max = 100 +innodb_monitor_enable = all +innodb_read_io_threads = 16 +innodb_write_io_threads = 16 +innodb_checksum_algorithm = crc32 +innodb_adaptive_hash_index = 0 +innodb_flush_method = O_DIRECT +innodb_log_buffer_size = 32M +innodb_purge_threads = 4 +innodb_buffer_pool_dump_at_shutdown = ON +innodb_buffer_pool_load_at_startup = ON + +#++++++++++++++++++++++++++++++++++++++++ Logging + +log_error = /var/log/mysql/error.log +log_output = TABLE +general_log = OFF +slow_query_log = ON +long_query_time = 2 +min_examined_row_limit = 0 +log_slow_admin_statements = ON +log_queries_not_using_indexes = OFF +max_error_count = 100 + +#++++++++++++++++++++++++++++++++++++++++ SSL + +ssl-ca = /etc/mysql/ca.pem +ssl-cert = /etc/mysql/cert.pem +ssl-key = /etc/mysql/key.pem + +#++++++++++++++++++++++++++++++++++++++++ Query cache + +query_cache_limit = 0 +query_cache_type = OFF +query_cache_size = 0 + +#++++++++++++++++++++++++++++++++++++++++ Performance Schema + +performance_schema = ON +performance_schema_digests_size = 20000 +performance-schema-consumer-events-statements-history = ON +performance_schema_consumer_events_statements_history_long = ON +userstat = ON diff --git a/roles/db/files/conf/z92-test.cnf b/roles/db/files/conf/z92-test.cnf new file mode 100644 index 0000000..e37ef52 --- /dev/null +++ b/roles/db/files/conf/z92-test.cnf @@ -0,0 +1,14 @@ +[mysqld] + +port = 3307 +bind-address = 0.0.0.0 +innodb_buffer_pool_size = 18G +event-scheduler = OFF +innodb_log_file_size = 5G +log_warnings = 2 + +#++++++++++++++++++++++++++++++++++++++++ Binary log + +log-bin = bin.log +expire_logs_days = 1 +relay_log = mysqld-relay-bin diff --git a/roles/db/files/conf/z95-production.cnf b/roles/db/files/conf/z95-production.cnf new file mode 100644 index 0000000..0861a85 --- /dev/null +++ b/roles/db/files/conf/z95-production.cnf @@ -0,0 +1,29 @@ +[mysqld] + +port = 3306 +innodb_log_file_size = 8G +log_warnings = 1 + +#++++++++++++++++++++++++++++++++++++++++ Binary log + +log-bin = /mnt/mysqlbin/binlog/bin.log +max_connections = 1000 +expire_logs_days = 7 +innodb_buffer_pool_size = 64G +relay_log = /mnt/mysqlbin/binlog/relay.log +binlog-ignore-db = tmp +binlog-ignore-db = PERCONA_SCHEMA + +#++++++++++++++++++++++++++++++++++++++++ Replication + +event-scheduler = ON +slave_exec_mode = STRICT + +replicate-ignore-db = tmp +replicate-ignore-table = util.eventLog +replicate-ignore-table = cache.cache_calc +replicate-ignore-table = cache.available +replicate-ignore-table = cache.availableNoRaids +replicate-ignore-table = cache.cache_valid +replicate-ignore-table = cache.stock +replicate-ignore-table = cache.visible diff --git a/roles/db/files/mariabackup/apply.development.sql b/roles/db/files/mariabackup/apply.development.sql new file mode 100644 index 0000000..b6aab80 --- /dev/null +++ b/roles/db/files/mariabackup/apply.development.sql @@ -0,0 +1,23 @@ +UPDATE vn2008.tblContadores + SET dbproduccion = FALSE; + +DELETE FROM util.binlogQueue; + +GRANT + SELECT, + INSERT, + UPDATE, + DELETE, + CREATE, + DROP, + INDEX, + ALTER, + CREATE TEMPORARY TABLES, + CREATE VIEW, + EVENT, + TRIGGER, + SHOW VIEW, + CREATE ROUTINE, + ALTER ROUTINE, + EXECUTE + ON *.* TO developerBoss; diff --git a/roles/db/files/mariabackup/apply.test.sql b/roles/db/files/mariabackup/apply.test.sql new file mode 100644 index 0000000..b1b26d3 --- /dev/null +++ b/roles/db/files/mariabackup/apply.test.sql @@ -0,0 +1,6 @@ +UPDATE vn2008.tblContadores + SET dbproduccion = FALSE; + +DELETE FROM util.binlogQueue; + +UPDATE `account`.`user` SET `active` = TRUE WHERE `name` = 'mindshore'; diff --git a/roles/db/files/mariabackup/bacula-after.sh b/roles/db/files/mariabackup/bacula-after.sh new file mode 100755 index 0000000..53e37bb --- /dev/null +++ b/roles/db/files/mariabackup/bacula-after.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$myDir/config.sh" +. "$myDir/apply.config.sh" + +todayDir=$(date +%Y-%m-%d) +pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz" +files=($pattern) +backupFile="${files[0]}" + +"$myDir/restore-backup.sh" "$backupFile" +rm -r "$baculaDir" + +if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then + for node in "${dbClusterSiblings[@]}"; do + ssh root@$node service mysql stop + ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi" + done + + galera_new_cluster +else + service mariadb start +fi + +echo "Applying custom script." +mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()" +mysql < "$myDir/apply.sql" + +echo "Upgrading tables." +mysql_upgrade + +echo "Applying repository changes." +curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec" + +echo "Waiting for Jenkins job to end." +jobResult=null +while [ "$jobResult" = "null" ]; do + sleep 10 + jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result") +done +echo "Job result: $jobResult" + +echo "Promoting to master." +"/root/scripts/promote-master.sh" + +for node in "${dbClusterSiblings[@]}"; do + ssh root@$node service mysql start +done + diff --git a/roles/db/files/nrpe/95-mariadb.cfg b/roles/db/files/nrpe/95-mariadb.cfg new file mode 100644 index 0000000..79e9e36 --- /dev/null +++ b/roles/db/files/nrpe/95-mariadb.cfg @@ -0,0 +1,4 @@ +command[check_disk_mysqldata]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/mysqldata +command[check_disk_mysqlbin]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/mysqlbin +command[check_disk_backup]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/local-backup +command[check_mysql_scheduler]=/etc/nagios/plugins/check_mysql_scheduler diff --git a/roles/db/tasks/production.yml b/roles/db/tasks/production.yml new file mode 100644 index 0000000..fe08adc --- /dev/null +++ b/roles/db/tasks/production.yml @@ -0,0 +1,26 @@ +- name: Ensure production required directories exist + file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: "{{ required_directories_master }}" + +- name: Set production MariaDB custom configuration + copy: + src: conf/z95-production.cnf + dest: /etc/mysql/mariadb.conf.d/ + owner: root + group: root + mode: u=rw,g=r,o=r + notify: restart-mariadb + +- name: Reminder to check production mount points + debug: + msg: | + Remember to check the following mount points: + - /var/lib/mysql + - /mnt/mysqlbin + - /mnt/local-backup + Make sure they are correctly configured and accessible. diff --git a/roles/db/tasks/test.yml b/roles/db/tasks/test.yml new file mode 100644 index 0000000..1d8d37b --- /dev/null +++ b/roles/db/tasks/test.yml @@ -0,0 +1,39 @@ +- name: Set test Mariabackup files + copy: + src: mariabackup/bacula-after.sh + dest: /root/mariabackup/ + owner: root + group: root + mode: u=rwx,g=rx,o=rx + +- name: Set test MariaBackup apply config template configuration + template: + src: mariabackup/apply.config.sh + dest: /root/mariabackup/ + owner: root + group: root + mode: u=rw,g=,o= + +- name: Set test MariaBackup apply SQL template configuration + copy: + src: mariabackup/apply.{{db.environment}}.sql + dest: /root/mariabackup/apply.sql + owner: root + group: root + mode: u=rw,g=,o= + +- name: Set test MariaDB custom configuration + copy: + src: conf/z92-test.cnf + dest: /etc/mysql/mariadb.conf.d/ + owner: root + group: root + mode: u=rw,g=r,o=r + notify: restart-mariadb + +- name: Reminder to check test mount points environment + debug: + msg: | + Remember to check the following mount points: + - /mnt/mysqltmp + Make sure they are correctly configured and accessible. diff --git a/roles/db/templates/conf/z99-local.cnf b/roles/db/templates/conf/z99-local.cnf new file mode 100644 index 0000000..a434496 --- /dev/null +++ b/roles/db/templates/conf/z99-local.cnf @@ -0,0 +1,7 @@ +[mysqld] + +server-id = {{ serverid }} +#bind-address = 127.0.0.1 +#event-scheduler = OFF +#skip-log-bin +#skip-slave-start diff --git a/roles/db/templates/mariabackup/apply.config.sh b/roles/db/templates/mariabackup/apply.config.sh new file mode 100755 index 0000000..f4e6c1c --- /dev/null +++ b/roles/db/templates/mariabackup/apply.config.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Bacula directory for restore +baculaDir=/mnt/mysqldata/bacula-restore + +# Database branch name +dbBranch={{ db.branch }} + +# Database environment +dbEnvironment={{ db.environment }} + +# MariaDB cluster sibling node hostnames +dbClusterSiblings=() + +# Jenkins authentication string +jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt_folder).password }} + +# Jenkins job URL +jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ db.branch }} diff --git a/roles/db/templates/mariabackup/my.cnf b/roles/db/templates/mariabackup/my.cnf new file mode 100644 index 0000000..b1b4ac9 --- /dev/null +++ b/roles/db/templates/mariabackup/my.cnf @@ -0,0 +1,7 @@ +[mariabackup] +host = localhost +user = mariabackup +password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }} +use-memory = 1G +parallel = 2 +stream = mbstream -- 2.40.1 From 15ae174c1858dc721c05f1295e2f0b26878c286e Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Mon, 10 Feb 2025 13:38:45 +0100 Subject: [PATCH 02/12] db: refs #8140 Fix apply SQL, dir structure changes --- playbooks/ceph.yml | 1 + playbooks/db.yml | 1 + playbooks/kube.yml | 1 + playbooks/pve.yml | 1 + roles/db/defaults/main.yaml | 29 ++-- .../{apply.development.sql => apply.dev.sql} | 0 roles/db/files/mariabackupdev/apply.sql | 23 --- roles/db/files/mariabackupdev/bacula-after.sh | 51 ------ roles/db/files/mariabackuptest/apply.sql | 6 - .../db/files/mariabackuptest/bacula-after.sh | 51 ------ roles/db/files/mariaconfigfiles/z90-vn.cnf | 98 ------------ .../mariaconfigfiles/z92-testing_dev.cnf | 14 -- .../files/mariaconfigfiles/z95-production.cnf | 29 ---- roles/db/files/marianrpe/95-mariadb.cfg | 4 - roles/db/handlers/main.yml | 4 + roles/db/tasks/main.yml | 5 +- roles/db/tasks/mariadb.yml | 150 +++++------------- roles/db/tasks/test.yml | 2 +- roles/db/templates/my.cnf | 7 - .../db/templates/test-db_dev/apply.config.sh | 20 --- roles/db/templates/z99-local.cnf | 7 - roles/pve/handlers/main.yml | 1 - 22 files changed, 65 insertions(+), 440 deletions(-) rename roles/db/files/mariabackup/{apply.development.sql => apply.dev.sql} (100%) delete mode 100644 roles/db/files/mariabackupdev/apply.sql delete mode 100755 roles/db/files/mariabackupdev/bacula-after.sh delete mode 100644 roles/db/files/mariabackuptest/apply.sql delete mode 100755 roles/db/files/mariabackuptest/bacula-after.sh delete mode 100644 roles/db/files/mariaconfigfiles/z90-vn.cnf delete mode 100644 roles/db/files/mariaconfigfiles/z92-testing_dev.cnf delete mode 100644 roles/db/files/mariaconfigfiles/z95-production.cnf delete mode 100644 roles/db/files/marianrpe/95-mariadb.cfg delete mode 100644 roles/db/templates/my.cnf delete mode 100755 roles/db/templates/test-db_dev/apply.config.sh delete mode 100644 roles/db/templates/z99-local.cnf diff --git a/playbooks/ceph.yml b/playbooks/ceph.yml index 0e9e56c..714551e 100644 --- a/playbooks/ceph.yml +++ b/playbooks/ceph.yml @@ -5,3 +5,4 @@ name: debian - import_role: name: ceph + tags: service diff --git a/playbooks/db.yml b/playbooks/db.yml index 2f83f93..a95a96b 100644 --- a/playbooks/db.yml +++ b/playbooks/db.yml @@ -5,3 +5,4 @@ name: debian - import_role: name: db + tags: service diff --git a/playbooks/kube.yml b/playbooks/kube.yml index 8628eb1..907afc1 100644 --- a/playbooks/kube.yml +++ b/playbooks/kube.yml @@ -5,3 +5,4 @@ name: debian - import_role: name: kube + tags: service diff --git a/playbooks/pve.yml b/playbooks/pve.yml index a351148..f26239a 100644 --- a/playbooks/pve.yml +++ b/playbooks/pve.yml @@ -5,3 +5,4 @@ name: debian - import_role: name: pve + tags: service diff --git a/roles/db/defaults/main.yaml b/roles/db/defaults/main.yaml index d648087..e7a2a0a 100644 --- a/roles/db/defaults/main.yaml +++ b/roles/db/defaults/main.yaml @@ -19,24 +19,19 @@ required_directories: - { path: /root/mariabackup, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } - { path: /mnt/mysqldata/mysql, owner: mysql, group: mysql, mode: 'u=rwx,g=rx,o=rx' } - { path: /etc/systemd/system/mariadb.service.d, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } -required_files_and_mariabackup_files_and_root_scripts: - - { src: "mariadb_override.conf", dest: "/etc/systemd/system/mariadb.service.d/override.conf", mode: "u=rw,g=r,o=r" } +required_mariabackup_files_and_scripts: - { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" } - - { src: "files/mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/mariabackup/restore-backup.sh", dest: "/root/mariabackup/restore-backup.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/scripts/check-memory.sh", dest: "/root/scripts/check-memory.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/scripts/export-privs.sh", dest: "/root/scripts/export-privs.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/scripts/mysqltuner.pl", dest: "/root/scripts/mysqltuner.pl", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/scripts/promote-master.sh", dest: "/root/scripts/promote-master.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/scripts/promote-slave.sh", dest: "/root/scripts/promote-slave.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/scripts/README.md", dest: "/root/scripts/README.md", mode: "u=rw,g=r,o=r" } - - { src: "files/scripts/scheduler-log.sh", dest: "/root/scripts/scheduler-log.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "files/marianrpe/95-mariadb.cfg", dest: "/etc/nagios/nrpe.d/95-mariadb.cfg", mode: "u=rw,g=r,o=r" } -mariabackup_test_dev_files: - - { src: "mariabackuptest/apply.sql", dest: "/root/mariabackup/", mode: "u=rw,g=r,o=r" } - - { src: "mariabackuptest/bacula-after.sh", dest: "/root/mariabackup/", mode: "u=rwx,g=rx,o=rx" } + - { src: "mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" } + - { src: "mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "mariabackup/restore-backup.sh", dest: "/root/mariabackup/restore-backup.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "scripts/check-memory.sh", dest: "/root/scripts/check-memory.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "scripts/export-privs.sh", dest: "/root/scripts/export-privs.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "scripts/mysqltuner.pl", dest: "/root/scripts/mysqltuner.pl", mode: "u=rwx,g=rx,o=rx" } + - { src: "scripts/promote-master.sh", dest: "/root/scripts/promote-master.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "scripts/promote-slave.sh", dest: "/root/scripts/promote-slave.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: "scripts/README.md", dest: "/root/scripts/README.md", mode: "u=rw,g=r,o=r" } + - { src: "scripts/scheduler-log.sh", dest: "/root/scripts/scheduler-log.sh", mode: "u=rwx,g=rx,o=rx" } downloads: - { url: "https://r.mariadb.com/downloads/mariadb_repo_setup", dest: "/tmp/mariadb_repo_setup", mode: "u=rwx,g=rx,o=rx" } - { url: "https://repo.percona.com/apt/percona-release_latest.generic_all.deb", dest: "/tmp/percona-release_latest.generic_all.deb", mode: "u=rw,g=r,o=r" } diff --git a/roles/db/files/mariabackup/apply.development.sql b/roles/db/files/mariabackup/apply.dev.sql similarity index 100% rename from roles/db/files/mariabackup/apply.development.sql rename to roles/db/files/mariabackup/apply.dev.sql diff --git a/roles/db/files/mariabackupdev/apply.sql b/roles/db/files/mariabackupdev/apply.sql deleted file mode 100644 index b6aab80..0000000 --- a/roles/db/files/mariabackupdev/apply.sql +++ /dev/null @@ -1,23 +0,0 @@ -UPDATE vn2008.tblContadores - SET dbproduccion = FALSE; - -DELETE FROM util.binlogQueue; - -GRANT - SELECT, - INSERT, - UPDATE, - DELETE, - CREATE, - DROP, - INDEX, - ALTER, - CREATE TEMPORARY TABLES, - CREATE VIEW, - EVENT, - TRIGGER, - SHOW VIEW, - CREATE ROUTINE, - ALTER ROUTINE, - EXECUTE - ON *.* TO developerBoss; diff --git a/roles/db/files/mariabackupdev/bacula-after.sh b/roles/db/files/mariabackupdev/bacula-after.sh deleted file mode 100755 index 53e37bb..0000000 --- a/roles/db/files/mariabackupdev/bacula-after.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -set -e - -myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -. "$myDir/config.sh" -. "$myDir/apply.config.sh" - -todayDir=$(date +%Y-%m-%d) -pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz" -files=($pattern) -backupFile="${files[0]}" - -"$myDir/restore-backup.sh" "$backupFile" -rm -r "$baculaDir" - -if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then - for node in "${dbClusterSiblings[@]}"; do - ssh root@$node service mysql stop - ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi" - done - - galera_new_cluster -else - service mariadb start -fi - -echo "Applying custom script." -mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()" -mysql < "$myDir/apply.sql" - -echo "Upgrading tables." -mysql_upgrade - -echo "Applying repository changes." -curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec" - -echo "Waiting for Jenkins job to end." -jobResult=null -while [ "$jobResult" = "null" ]; do - sleep 10 - jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result") -done -echo "Job result: $jobResult" - -echo "Promoting to master." -"/root/scripts/promote-master.sh" - -for node in "${dbClusterSiblings[@]}"; do - ssh root@$node service mysql start -done - diff --git a/roles/db/files/mariabackuptest/apply.sql b/roles/db/files/mariabackuptest/apply.sql deleted file mode 100644 index b1b26d3..0000000 --- a/roles/db/files/mariabackuptest/apply.sql +++ /dev/null @@ -1,6 +0,0 @@ -UPDATE vn2008.tblContadores - SET dbproduccion = FALSE; - -DELETE FROM util.binlogQueue; - -UPDATE `account`.`user` SET `active` = TRUE WHERE `name` = 'mindshore'; diff --git a/roles/db/files/mariabackuptest/bacula-after.sh b/roles/db/files/mariabackuptest/bacula-after.sh deleted file mode 100755 index 53e37bb..0000000 --- a/roles/db/files/mariabackuptest/bacula-after.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -set -e - -myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -. "$myDir/config.sh" -. "$myDir/apply.config.sh" - -todayDir=$(date +%Y-%m-%d) -pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz" -files=($pattern) -backupFile="${files[0]}" - -"$myDir/restore-backup.sh" "$backupFile" -rm -r "$baculaDir" - -if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then - for node in "${dbClusterSiblings[@]}"; do - ssh root@$node service mysql stop - ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi" - done - - galera_new_cluster -else - service mariadb start -fi - -echo "Applying custom script." -mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()" -mysql < "$myDir/apply.sql" - -echo "Upgrading tables." -mysql_upgrade - -echo "Applying repository changes." -curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec" - -echo "Waiting for Jenkins job to end." -jobResult=null -while [ "$jobResult" = "null" ]; do - sleep 10 - jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result") -done -echo "Job result: $jobResult" - -echo "Promoting to master." -"/root/scripts/promote-master.sh" - -for node in "${dbClusterSiblings[@]}"; do - ssh root@$node service mysql start -done - diff --git a/roles/db/files/mariaconfigfiles/z90-vn.cnf b/roles/db/files/mariaconfigfiles/z90-vn.cnf deleted file mode 100644 index 1911135..0000000 --- a/roles/db/files/mariaconfigfiles/z90-vn.cnf +++ /dev/null @@ -1,98 +0,0 @@ -[mysqld] -# Docs: https://mariadb.com/kb/en/server-system-variables - -lc_messages = es_ES -lc_time_names = es_ES -character-set-server = utf8 -collation-server = utf8_unicode_ci -explicit_defaults_for_timestamp = ON -datadir = /mnt/mysqldata/mysql -tmpdir = /mnt/mysqltmp -log_bin_trust_function_creators = 1 -sql_mode = NO_ENGINE_SUBSTITUTION -bind-address = 0.0.0.0 -max_password_errors = 50 - -#++++++++++++++++++++++++++++++++++++++++ Threads - -thread_stack = 512K -join_buffer_size = 2M -sort_buffer_size = 4M -net_buffer_length = 256K -max_allowed_packet = 16M -read_buffer_size = 1M -read_rnd_buffer_size = 512K - -#++++++++++++++++++++++++++++++++++++++++ Performance - -thread_cache_size = 450 -interactive_timeout = 1800 -wait_timeout = 1800 -open_files_limit = 20000 -low_priority_updates = 1 -table_open_cache = 40000 -table_definition_cache = 10000 -table_open_cache_instances = 1 -key_buffer_size = 256K -max_heap_table_size = 128M -tmp_table_size = 128M -concurrent_insert = ALWAYS -group_concat_max_len = 10000 -max_connect_errors = 50 - -#++++++++++++++++++++++++++++++++++++++++ Binary log - -max_binlog_size = 1GB -binlog_cache_size = 16M -binlog_stmt_cache_size = 16M -binlog_row_image = noblob -binlog_format = row - -#++++++++++++++++++++++++++++++++++++++++ InnoDB - -transaction-isolation = READ-COMMITTED -idle_transaction_timeout = 60 -innodb_io_capacity = 100 -innodb_io_capacity_max = 100 -innodb_monitor_enable = all -innodb_read_io_threads = 16 -innodb_write_io_threads = 16 -innodb_checksum_algorithm = crc32 -innodb_adaptive_hash_index = 0 -innodb_flush_method = O_DIRECT -innodb_log_buffer_size = 32M -innodb_purge_threads = 4 -innodb_buffer_pool_dump_at_shutdown = ON -innodb_buffer_pool_load_at_startup = ON - -#++++++++++++++++++++++++++++++++++++++++ Logging - -log_error = /var/log/mysql/error.log -log_output = TABLE -general_log = OFF -slow_query_log = ON -long_query_time = 2 -min_examined_row_limit = 0 -log_slow_admin_statements = ON -log_queries_not_using_indexes = OFF -max_error_count = 100 - -#++++++++++++++++++++++++++++++++++++++++ SSL - -ssl-ca = /etc/mysql/ca.pem -ssl-cert = /etc/mysql/cert.pem -ssl-key = /etc/mysql/key.pem - -#++++++++++++++++++++++++++++++++++++++++ Query cache - -query_cache_limit = 0 -query_cache_type = OFF -query_cache_size = 0 - -#++++++++++++++++++++++++++++++++++++++++ Performance Schema - -performance_schema = ON -performance_schema_digests_size = 20000 -performance-schema-consumer-events-statements-history = ON -performance_schema_consumer_events_statements_history_long = ON -userstat = ON diff --git a/roles/db/files/mariaconfigfiles/z92-testing_dev.cnf b/roles/db/files/mariaconfigfiles/z92-testing_dev.cnf deleted file mode 100644 index e37ef52..0000000 --- a/roles/db/files/mariaconfigfiles/z92-testing_dev.cnf +++ /dev/null @@ -1,14 +0,0 @@ -[mysqld] - -port = 3307 -bind-address = 0.0.0.0 -innodb_buffer_pool_size = 18G -event-scheduler = OFF -innodb_log_file_size = 5G -log_warnings = 2 - -#++++++++++++++++++++++++++++++++++++++++ Binary log - -log-bin = bin.log -expire_logs_days = 1 -relay_log = mysqld-relay-bin diff --git a/roles/db/files/mariaconfigfiles/z95-production.cnf b/roles/db/files/mariaconfigfiles/z95-production.cnf deleted file mode 100644 index 0861a85..0000000 --- a/roles/db/files/mariaconfigfiles/z95-production.cnf +++ /dev/null @@ -1,29 +0,0 @@ -[mysqld] - -port = 3306 -innodb_log_file_size = 8G -log_warnings = 1 - -#++++++++++++++++++++++++++++++++++++++++ Binary log - -log-bin = /mnt/mysqlbin/binlog/bin.log -max_connections = 1000 -expire_logs_days = 7 -innodb_buffer_pool_size = 64G -relay_log = /mnt/mysqlbin/binlog/relay.log -binlog-ignore-db = tmp -binlog-ignore-db = PERCONA_SCHEMA - -#++++++++++++++++++++++++++++++++++++++++ Replication - -event-scheduler = ON -slave_exec_mode = STRICT - -replicate-ignore-db = tmp -replicate-ignore-table = util.eventLog -replicate-ignore-table = cache.cache_calc -replicate-ignore-table = cache.available -replicate-ignore-table = cache.availableNoRaids -replicate-ignore-table = cache.cache_valid -replicate-ignore-table = cache.stock -replicate-ignore-table = cache.visible diff --git a/roles/db/files/marianrpe/95-mariadb.cfg b/roles/db/files/marianrpe/95-mariadb.cfg deleted file mode 100644 index 79e9e36..0000000 --- a/roles/db/files/marianrpe/95-mariadb.cfg +++ /dev/null @@ -1,4 +0,0 @@ -command[check_disk_mysqldata]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/mysqldata -command[check_disk_mysqlbin]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/mysqlbin -command[check_disk_backup]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/local-backup -command[check_mysql_scheduler]=/etc/nagios/plugins/check_mysql_scheduler diff --git a/roles/db/handlers/main.yml b/roles/db/handlers/main.yml index 0ed2a4d..320b475 100644 --- a/roles/db/handlers/main.yml +++ b/roles/db/handlers/main.yml @@ -5,3 +5,7 @@ systemd: name: mariadb state: restarted +- name: restart-nrpe + service: + name: nagios-nrpe-server + state: restarted diff --git a/roles/db/tasks/main.yml b/roles/db/tasks/main.yml index dbdb57d..77d9ec3 100644 --- a/roles/db/tasks/main.yml +++ b/roles/db/tasks/main.yml @@ -1,2 +1,5 @@ - import_tasks: mariadb.yml - tags: service +- when: db.branch == 'master' + import_tasks: production.yml +- when: db.branch in ['dev', 'test'] + import_tasks: test.yml diff --git a/roles/db/tasks/mariadb.yml b/roles/db/tasks/mariadb.yml index 6569dac..2f67be8 100644 --- a/roles/db/tasks/mariadb.yml +++ b/roles/db/tasks/mariadb.yml @@ -1,6 +1,3 @@ -# Review /root/scripts/check-memory.sh --> It's not optimal to do what this program does -# Also review the cron task /root/scripts/scheduler-log.sh - - name: Ensure Install requirements for MariaDB repository setup script apt: name: "{{ mariadb_requeriments }}" @@ -43,7 +40,7 @@ tmpfs /mnt/mysqltmp tmpfs rw,size={{ mysqltmpsize }} 0 0 register: fstab -- name: Set MariaDB Cron to /etc/cron.d +- name: Configure MariaDB memory check CRON template: src: check-memory.cron dest: /etc/cron.d/vn-check-memory @@ -79,115 +76,58 @@ mode: "{{ item.mode }}" loop: "{{ required_directories }}" -- when: db.branch == 'master' - block: +- name: Copy required MariaBackup files and scripts + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: "{{ required_mariabackup_files_and_scripts }}" - - name: Ensure required directories exist in Production - file: - path: "{{ item.path }}" - state: directory - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "{{ item.mode }}" - loop: "{{ required_directories_master }}" - - - name: Set MariaDB custom configuration Production - copy: - src: "{{ item }}" - dest: /etc/mysql/mariadb.conf.d/ - owner: root - group: root - mode: u=rw,g=r,o=r - with_items: - - "mariaconfigfiles/z90-vn.cnf" - - "mariaconfigfiles/z95-production.cnf" - notify: restart-mariadb - - - name: Reminder to check mount points - debug: - msg: | - Remember to check the following mount points: - - /var/lib/mysql - - /mnt/mysqlbin - - /mnt/local-backup - Make sure they are correctly configured and accessible. +- name: Set MariaDB common configuration + copy: + src: conf/z90-vn.cnf + dest: /etc/mysql/mariadb.conf.d/ + owner: root + group: root + mode: u=rw,g=r,o=r + notify: restart-mariadb -- when: db.branch == 'test' - - block: - - name: Set MariaBackup custom template configuration TestDB - template: - src: test-db_dev/apply.config.sh - dest: /root/mariabackup/ - owner: root - group: root - mode: u=rw,g=,o= - - - name: Reminder to check mount points environment TestDB - debug: - msg: | - Remember to check the following mount points: - - /mnt/mysqltmp - Make sure they are correctly configured and accessible. - -- when: db.branch == 'dev' - - block: - - name: Set MariaBackup custom template configuration DevDB - template: - src: test-db_dev/apply.config.sh - dest: /root/mariabackup/ - owner: root - group: root - mode: u=rw,g=,o= - - - name: Reminder to check mount points environment DevDB - debug: - msg: | - Remember to check the following mount points: - - /mnt/mysqltmp - Make sure they are correctly configured and accessible. - -- when: db.branch == 'dev' or db.branch == 'test' - - block: - - name: Set MariaDB custom configuration TestDB or DevDB - copy: - src: "{{ item }}" - dest: /etc/mysql/mariadb.conf.d/ - owner: root - group: root - mode: u=rw,g=r,o=r - with_items: - - "files/mariaconfigfiles/z90-vn.cnf" - - "files/mariaconfigfiles/z92-testing_dev.cnf" - notify: restart-mariadb - - - name: Set Mariabackup files for TestDB or DevDB - copy: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: root - group: root - mode: "{{ item.mode }}" - loop: "{{ mariabackup_test_dev_files }}" - -- name: Set Custom Configuration local template all Environment +- name: Set MariaDB local configuration template template: - src: z99-local.cnf + src: conf/z99-local.cnf dest: /etc/mysql/mariadb.conf.d/ owner: root group: root mode: u=rw,g=r,o=r -- name: Set MariaBackup custom template configuration all Environment +- name: Set MariaBackup connection configuration template: - src: "my.cnf" + src: mariabackup/my.cnf dest: /root/mariabackup/ owner: root group: root mode: u=rw,g=,o= +- name: Override MariaDB systemd service configuration + copy: + src: mariadb_override.conf + dest: /etc/systemd/system/mariadb.service.d/override.conf + owner: root + group: root + mode: u=rw,g=r,o=r + notify: reload-systemd + +- name: Set MariaDB NRPE configuration + copy: + src: nrpe/95-mariadb.cfg + dest: /etc/nagios/nrpe.d/ + owner: root + group: root + mode: u=rw,g=r,o=r + notify: restart-nrpe + - name: Check if /var/lib/mysql/ exists stat: path: /var/lib/mysql/ @@ -210,16 +150,6 @@ path: /var/lib/mysql/ state: absent -- name: Ensure required custom and Mariabackup files are copied to their destinations and root scripts - copy: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: root - group: root - mode: "{{ item.mode }}" - loop: "{{ required_files_and_mariabackup_files_and_root_scripts }}" - notify: reload-systemd - - name: Mount all filesystems from /etc/fstab command: mount -a - when: fstab.changed \ No newline at end of file + when: fstab.changed diff --git a/roles/db/tasks/test.yml b/roles/db/tasks/test.yml index 1d8d37b..d474cd8 100644 --- a/roles/db/tasks/test.yml +++ b/roles/db/tasks/test.yml @@ -16,7 +16,7 @@ - name: Set test MariaBackup apply SQL template configuration copy: - src: mariabackup/apply.{{db.environment}}.sql + src: mariabackup/apply.{{db.branch}}.sql dest: /root/mariabackup/apply.sql owner: root group: root diff --git a/roles/db/templates/my.cnf b/roles/db/templates/my.cnf deleted file mode 100644 index b1b4ac9..0000000 --- a/roles/db/templates/my.cnf +++ /dev/null @@ -1,7 +0,0 @@ -[mariabackup] -host = localhost -user = mariabackup -password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }} -use-memory = 1G -parallel = 2 -stream = mbstream diff --git a/roles/db/templates/test-db_dev/apply.config.sh b/roles/db/templates/test-db_dev/apply.config.sh deleted file mode 100755 index d42f5f7..0000000 --- a/roles/db/templates/test-db_dev/apply.config.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Bacula directory for restore -baculaDir=/mnt/mysqldata/bacula-restore - -# Database branch name -dbBranch={{ db.branch }} - -# Database environment -dbEnvironment={{ db.environment }} - -# MariaDB cluster sibling node hostnames -dbClusterSiblings=() - -# Jenkins authentication string -jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt_folder).password }} - -# Jenkins job URL -jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ db.branch }} - diff --git a/roles/db/templates/z99-local.cnf b/roles/db/templates/z99-local.cnf deleted file mode 100644 index a434496..0000000 --- a/roles/db/templates/z99-local.cnf +++ /dev/null @@ -1,7 +0,0 @@ -[mysqld] - -server-id = {{ serverid }} -#bind-address = 127.0.0.1 -#event-scheduler = OFF -#skip-log-bin -#skip-slave-start diff --git a/roles/pve/handlers/main.yml b/roles/pve/handlers/main.yml index 1cbeed2..0399734 100644 --- a/roles/pve/handlers/main.yml +++ b/roles/pve/handlers/main.yml @@ -2,4 +2,3 @@ service: name: nagios-nrpe-server state: restarted - -- 2.40.1 From 4b9243ce4f2a7a1aad168ef4c01d2df9496abfc4 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Mon, 10 Feb 2025 14:04:11 +0100 Subject: [PATCH 03/12] db: refs #8140 Description fixes --- roles/db/tasks/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/db/tasks/test.yml b/roles/db/tasks/test.yml index d474cd8..13c53ef 100644 --- a/roles/db/tasks/test.yml +++ b/roles/db/tasks/test.yml @@ -6,7 +6,7 @@ group: root mode: u=rwx,g=rx,o=rx -- name: Set test MariaBackup apply config template configuration +- name: Set test MariaBackup apply config configuration template: src: mariabackup/apply.config.sh dest: /root/mariabackup/ @@ -14,7 +14,7 @@ group: root mode: u=rw,g=,o= -- name: Set test MariaBackup apply SQL template configuration +- name: Set test MariaBackup apply SQL script copy: src: mariabackup/apply.{{db.branch}}.sql dest: /root/mariabackup/apply.sql -- 2.40.1 From 48c2fccf3ad4fed6115699e0552f1f323c58d879 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Mon, 10 Feb 2025 14:09:30 +0100 Subject: [PATCH 04/12] db: refs #8140 Defaults quotes removed --- roles/db/defaults/main.yaml | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/roles/db/defaults/main.yaml b/roles/db/defaults/main.yaml index e7a2a0a..0a37042 100644 --- a/roles/db/defaults/main.yaml +++ b/roles/db/defaults/main.yaml @@ -20,18 +20,22 @@ required_directories: - { path: /mnt/mysqldata/mysql, owner: mysql, group: mysql, mode: 'u=rwx,g=rx,o=rx' } - { path: /etc/systemd/system/mariadb.service.d, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' } required_mariabackup_files_and_scripts: - - { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" } - - { src: "mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "mariabackup/restore-backup.sh", dest: "/root/mariabackup/restore-backup.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "scripts/check-memory.sh", dest: "/root/scripts/check-memory.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "scripts/export-privs.sh", dest: "/root/scripts/export-privs.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "scripts/mysqltuner.pl", dest: "/root/scripts/mysqltuner.pl", mode: "u=rwx,g=rx,o=rx" } - - { src: "scripts/promote-master.sh", dest: "/root/scripts/promote-master.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "scripts/promote-slave.sh", dest: "/root/scripts/promote-slave.sh", mode: "u=rwx,g=rx,o=rx" } - - { src: "scripts/README.md", dest: "/root/scripts/README.md", mode: "u=rw,g=r,o=r" } - - { src: "scripts/scheduler-log.sh", dest: "/root/scripts/scheduler-log.sh", mode: "u=rwx,g=rx,o=rx" } + - { src: mysql-flush.sh, dest: /etc/qemu/fsfreeze-hook.d/mysql-flush.sh, mode: u=rwx,g=rx,o=rx } + - { src: mariabackup/bacula-before.sh, dest: /root/mariabackup/bacula-before.sh, mode: u=rwx,g=rx,o=rx } + - { src: mariabackup/config.sh, dest: /root/mariabackup/config.sh, mode: u=rwx,g=rx,o=x } + - { src: mariabackup/inc-backup.sh, dest: /root/mariabackup/inc-backup.sh, mode: u=rwx,g=rx,o=rx } + - { src: mariabackup/restore-backup.sh, dest: /root/mariabackup/restore-backup.sh, mode: u=rwx,g=rx,o=rx } + - { src: scripts/check-memory.sh, dest: /root/scripts/check-memory.sh, mode: u=rwx,g=rx,o=rx } + - { src: scripts/export-privs.sh, dest: /root/scripts/export-privs.sh, mode: u=rwx,g=rx,o=rx } + - { src: scripts/mysqltuner.pl, dest: /root/scripts/mysqltuner.pl, mode: u=rwx,g=rx,o=rx } + - { src: scripts/promote-master.sh, dest: /root/scripts/promote-master.sh, mode: u=rwx,g=rx,o=rx } + - { src: scripts/promote-slave.sh, dest: /root/scripts/promote-slave.sh, mode: u=rwx,g=rx,o=rx } + - { src: scripts/README.md, dest: /root/scripts/README.md, mode: u=rw,g=r,o=r } + - { src: scripts/scheduler-log.sh, dest: /root/scripts/scheduler-log.sh, mode: u=rwx,g=rx,o=rx } downloads: - - { url: "https://r.mariadb.com/downloads/mariadb_repo_setup", dest: "/tmp/mariadb_repo_setup", mode: "u=rwx,g=rx,o=rx" } - - { url: "https://repo.percona.com/apt/percona-release_latest.generic_all.deb", dest: "/tmp/percona-release_latest.generic_all.deb", mode: "u=rw,g=r,o=r" } + - url: https://r.mariadb.com/downloads/mariadb_repo_setup + dest: /tmp/mariadb_repo_setup + mode: u=rwx,g=rx,o=rx + - url: https://repo.percona.com/apt/percona-release_latest.generic_all.deb + dest: /tmp/percona-release_latest.generic_all.deb + mode: u=rw,g=r,o=r -- 2.40.1 From d08f73445c77705a6f195cc7a87e3ec786f5de05 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Fri, 14 Feb 2025 08:28:42 +0100 Subject: [PATCH 05/12] db: refs #8140 Backup & restore parallelized --- roles/db/defaults/main.yaml | 1 + roles/db/files/mariabackup/bacula-before.sh | 5 +++-- roles/db/files/mariabackup/restore-backup.sh | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/roles/db/defaults/main.yaml b/roles/db/defaults/main.yaml index 0a37042..b26beb7 100644 --- a/roles/db/defaults/main.yaml +++ b/roles/db/defaults/main.yaml @@ -2,6 +2,7 @@ mariadb_base_packages: - mariadb-server - mariadb-backup - pmm2-client + - pigz mariadb_requeriments: - curl - apt-transport-https diff --git a/roles/db/files/mariabackup/bacula-before.sh b/roles/db/files/mariabackup/bacula-before.sh index f1628fd..f4723a3 100644 --- a/roles/db/files/mariabackup/bacula-before.sh +++ b/roles/db/files/mariabackup/bacula-before.sh @@ -19,12 +19,13 @@ mariabackup \ --backup \ --extra-lsndir="$backupDir/$backupName" \ --history="$todayDir" \ + --stream=xbstream \ + --parallel=4 \ 2>> "$logFile" \ - | gzip \ + | pigz -p 12 \ > "$backupFile" if [ $? != "0" ]; then echo "An error ocurred during backup, please take a look at log file: $logFile" exit 1 fi - diff --git a/roles/db/files/mariabackup/restore-backup.sh b/roles/db/files/mariabackup/restore-backup.sh index 0e70424..73d06e2 100644 --- a/roles/db/files/mariabackup/restore-backup.sh +++ b/roles/db/files/mariabackup/restore-backup.sh @@ -27,8 +27,8 @@ mkdir -p "$restoreDir" echo "$(formatted_date)" echo "Decompresing backup." -gzip --decompress --stdout "$backupFile" \ - | mbstream -x --directory="$restoreDir" +pigz --decompress --processes 4 --stdout "$backupFile" \ + | mbstream --extract --parallel=4 --directory="$restoreDir" echo "Preparing backup." mariabackup \ -- 2.40.1 From dabd7305f947cea78532c334012aff4a290dca7e Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Fri, 14 Feb 2025 08:54:51 +0100 Subject: [PATCH 06/12] db: refs #8140 Warn when python venv is not enabled --- run-playbook.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/run-playbook.sh b/run-playbook.sh index 23f5d6a..ea7db4e 100755 --- a/run-playbook.sh +++ b/run-playbook.sh @@ -1,5 +1,7 @@ #!/bin/bash +MY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PY_VENV_DIR="$MY_DIR/venv" EXTRA_ARGS=() if [ -f .passbolt.yml ]; then @@ -9,5 +11,11 @@ if [ -f .vault-pass ]; then EXTRA_ARGS+=("--vault-password-file" ".vault-pass") fi -#export PYTHONPATH=./venv/lib/python3.12/site-packages/ +if [[ -z "$VIRTUAL_ENV" && -d "$PY_VENV_DIR" ]]; then + WARNING="\e[33m" + RESET="\e[0m" + echo -e "Warning: ${WARNING}Python virtual environment not enabled! Some tasks might fail.${RESET}" + #export PYTHONPATH=./venv/lib/python3.12/site-packages/ +fi + ansible-playbook ${EXTRA_ARGS[@]} $@ -- 2.40.1 From 207b12ad5ced708fcdc5b8a8478984c29513450f Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Fri, 14 Feb 2025 08:59:17 +0100 Subject: [PATCH 07/12] db: refs #8140 Python venv warn message clarified --- run-playbook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-playbook.sh b/run-playbook.sh index ea7db4e..acfde3e 100755 --- a/run-playbook.sh +++ b/run-playbook.sh @@ -14,7 +14,7 @@ fi if [[ -z "$VIRTUAL_ENV" && -d "$PY_VENV_DIR" ]]; then WARNING="\e[33m" RESET="\e[0m" - echo -e "Warning: ${WARNING}Python virtual environment not enabled! Some tasks might fail.${RESET}" + echo -e "Warning: ${WARNING}Python virtual environment not activated! Some tasks might fail.${RESET}" #export PYTHONPATH=./venv/lib/python3.12/site-packages/ fi -- 2.40.1 From 7af20886b0226018eae1078858d620840b77dd45 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Fri, 14 Feb 2025 09:06:19 +0100 Subject: [PATCH 08/12] db: refs #8140 README debug playbook doc fix --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 7d0991d..d0ac1c4 100644 --- a/README.md +++ b/README.md @@ -105,8 +105,7 @@ ansible-builder build --tag awx-ee:vn1 ## Common playbooks -* **facts.yml**: Collect and display facts from a host -* **ping.yml**: Check that a host is alive and reachable +* **debug.yml**: Debugging Tasks: facts, var, ping... * **awx.yml**: Create and configure AWX user * **debian.yml**: Setup base Debian server -- 2.40.1 From 83ba49156ffa2f74a90d8549776f1ac2dd1aa5dc Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Fri, 14 Feb 2025 09:07:13 +0100 Subject: [PATCH 09/12] db: refs #8140 README debug playbook doc fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d0ac1c4..d278d97 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ ansible-builder build --tag awx-ee:vn1 ## Common playbooks -* **debug.yml**: Debugging Tasks: facts, var, ping... +* **debug.yml**: Debugging tasks: facts, var, ping... * **awx.yml**: Create and configure AWX user * **debian.yml**: Setup base Debian server -- 2.40.1 From 831865a72fa4037c72189fa3c811d27a1de62ece Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Fri, 14 Feb 2025 09:10:12 +0100 Subject: [PATCH 10/12] db: refs #8140 Debug playbook tag fix --- README.md | 2 +- playbooks/debug.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d278d97..f4bf2d9 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ ansible-builder build --tag awx-ee:vn1 ## Common playbooks -* **debug.yml**: Debugging tasks: facts, var, ping... +* **debug.yml**: Debugging tasks: facts, vars, ping... * **awx.yml**: Create and configure AWX user * **debian.yml**: Setup base Debian server diff --git a/playbooks/debug.yml b/playbooks/debug.yml index 9efe37e..91dd062 100644 --- a/playbooks/debug.yml +++ b/playbooks/debug.yml @@ -14,7 +14,7 @@ var: vars - name: Print variable value - tags: var + tags: [var, vars] when: var_name is defined debug: msg: "{{ var_name }}: {{ lookup('vars', var_name, default='undefined') }}" -- 2.40.1 From 178cc5fd13211f17c6804f63a1423c48a44d3d71 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Fri, 14 Feb 2025 09:11:03 +0100 Subject: [PATCH 11/12] db: refs #8140 Debug playbook tag fix --- playbooks/debug.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/debug.yml b/playbooks/debug.yml index 91dd062..0757e5a 100644 --- a/playbooks/debug.yml +++ b/playbooks/debug.yml @@ -14,7 +14,7 @@ var: vars - name: Print variable value - tags: [var, vars] + tags: [facts, vars, var] when: var_name is defined debug: msg: "{{ var_name }}: {{ lookup('vars', var_name, default='undefined') }}" -- 2.40.1 From 7afb02918651aa7934a5a5fc4e017a05757d42eb Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Fri, 14 Feb 2025 09:25:56 +0100 Subject: [PATCH 12/12] db: refs #8140 my.cnf fixes --- roles/db/templates/mariabackup/my.cnf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/db/templates/mariabackup/my.cnf b/roles/db/templates/mariabackup/my.cnf index b1b4ac9..7037bef 100644 --- a/roles/db/templates/mariabackup/my.cnf +++ b/roles/db/templates/mariabackup/my.cnf @@ -3,5 +3,5 @@ host = localhost user = mariabackup password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }} use-memory = 1G -parallel = 2 -stream = mbstream +parallel = 4 +stream = xbstream -- 2.40.1