db: refs #8140 Fix apply SQL, dir structure changes

2025-02-10 13:38:40 +01:00 · 2025-02-10 13:38:40 +01:00 · 4bee560a8d
parent 644e67db87
commit 4bee560a8d
12 changed files with 323 additions and 0 deletions
--- a/roles/db/files/conf/z90-vn.cnf
+++ b/roles/db/files/conf/z90-vn.cnf
@ -0,0 +1,98 @@
+[mysqld]
+# Docs: https://mariadb.com/kb/en/server-system-variables
+
+lc_messages                                 = es_ES
+lc_time_names                               = es_ES
+character-set-server                        = utf8
+collation-server                            = utf8_unicode_ci
+explicit_defaults_for_timestamp             = ON
+datadir                                     = /mnt/mysqldata/mysql
+tmpdir                                      = /mnt/mysqltmp
+log_bin_trust_function_creators             = 1
+sql_mode                                    = NO_ENGINE_SUBSTITUTION
+bind-address                                = 0.0.0.0
+max_password_errors                         = 50
+
+#++++++++++++++++++++++++++++++++++++++++ Threads
+
+thread_stack                                = 512K
+join_buffer_size                            = 2M
+sort_buffer_size                            = 4M
+net_buffer_length                           = 256K
+max_allowed_packet                          = 16M
+read_buffer_size                            = 1M
+read_rnd_buffer_size                        = 512K
+
+#++++++++++++++++++++++++++++++++++++++++ Performance
+
+thread_cache_size                           = 450
+interactive_timeout                         = 1800
+wait_timeout                                = 1800
+open_files_limit                            = 20000
+low_priority_updates                        = 1
+table_open_cache                            = 40000
+table_definition_cache                      = 10000
+table_open_cache_instances                  = 1
+key_buffer_size                             = 256K
+max_heap_table_size                         = 128M
+tmp_table_size                              = 128M
+concurrent_insert                           = ALWAYS
+group_concat_max_len                        = 10000
+max_connect_errors                          = 50
+
+#++++++++++++++++++++++++++++++++++++++++ Binary log
+
+max_binlog_size                             = 1GB
+binlog_cache_size                           = 16M
+binlog_stmt_cache_size                      = 16M
+binlog_row_image                            = noblob
+binlog_format                               = row
+
+#++++++++++++++++++++++++++++++++++++++++ InnoDB
+
+transaction-isolation                       = READ-COMMITTED
+idle_transaction_timeout                    = 60
+innodb_io_capacity                          = 100
+innodb_io_capacity_max                      = 100
+innodb_monitor_enable                       = all
+innodb_read_io_threads                      = 16
+innodb_write_io_threads                     = 16
+innodb_checksum_algorithm                   = crc32
+innodb_adaptive_hash_index                  = 0
+innodb_flush_method                         = O_DIRECT
+innodb_log_buffer_size                      = 32M
+innodb_purge_threads                        = 4
+innodb_buffer_pool_dump_at_shutdown         = ON
+innodb_buffer_pool_load_at_startup          = ON
+
+#++++++++++++++++++++++++++++++++++++++++ Logging
+
+log_error                                   = /var/log/mysql/error.log
+log_output                                  = TABLE
+general_log                                 = OFF
+slow_query_log                              = ON
+long_query_time                             = 2
+min_examined_row_limit                      = 0
+log_slow_admin_statements                   = ON
+log_queries_not_using_indexes               = OFF
+max_error_count                             = 100
+
+#++++++++++++++++++++++++++++++++++++++++ SSL
+
+ssl-ca                                      = /etc/mysql/ca.pem
+ssl-cert                                    = /etc/mysql/cert.pem
+ssl-key                                     = /etc/mysql/key.pem
+
+#++++++++++++++++++++++++++++++++++++++++ Query cache
+
+query_cache_limit                           = 0
+query_cache_type                            = OFF
+query_cache_size                            = 0
+
+#++++++++++++++++++++++++++++++++++++++++ Performance Schema
+
+performance_schema                                         = ON
+performance_schema_digests_size                            = 20000
+performance-schema-consumer-events-statements-history      = ON
+performance_schema_consumer_events_statements_history_long = ON
+userstat                                                   = ON
--- a/roles/db/files/conf/z92-test.cnf
+++ b/roles/db/files/conf/z92-test.cnf
@ -0,0 +1,14 @@
+[mysqld]
+
+port                                        = 3307
+bind-address                                = 0.0.0.0
+innodb_buffer_pool_size                     = 18G
+event-scheduler                             = OFF
+innodb_log_file_size                        = 5G
+log_warnings                                = 2
+
+#++++++++++++++++++++++++++++++++++++++++ Binary log
+
+log-bin                                     = bin.log
+expire_logs_days                            = 1
+relay_log                                   = mysqld-relay-bin
--- a/roles/db/files/conf/z95-production.cnf
+++ b/roles/db/files/conf/z95-production.cnf
@ -0,0 +1,29 @@
+[mysqld]
+
+port                                        = 3306
+innodb_log_file_size                        = 8G
+log_warnings                                = 1
+
+#++++++++++++++++++++++++++++++++++++++++ Binary log
+
+log-bin                                     = /mnt/mysqlbin/binlog/bin.log
+max_connections                             = 1000
+expire_logs_days                            = 7
+innodb_buffer_pool_size                     = 64G
+relay_log                                   = /mnt/mysqlbin/binlog/relay.log
+binlog-ignore-db                            = tmp
+binlog-ignore-db                            = PERCONA_SCHEMA
+
+#++++++++++++++++++++++++++++++++++++++++ Replication
+
+event-scheduler                             = ON
+slave_exec_mode                             = STRICT
+
+replicate-ignore-db                         = tmp
+replicate-ignore-table                      = util.eventLog
+replicate-ignore-table                      = cache.cache_calc
+replicate-ignore-table                      = cache.available
+replicate-ignore-table                      = cache.availableNoRaids
+replicate-ignore-table                      = cache.cache_valid
+replicate-ignore-table                      = cache.stock
+replicate-ignore-table                      = cache.visible
--- a/roles/db/files/mariabackup/apply.development.sql
+++ b/roles/db/files/mariabackup/apply.development.sql
@ -0,0 +1,23 @@
+UPDATE vn2008.tblContadores
+	SET dbproduccion = FALSE;
+
+DELETE FROM util.binlogQueue;
+
+GRANT
+		SELECT,
+		INSERT,
+		UPDATE,
+		DELETE,
+		CREATE,
+		DROP,
+		INDEX,
+		ALTER,
+		CREATE TEMPORARY TABLES,
+		CREATE VIEW,
+		EVENT,
+		TRIGGER,
+		SHOW VIEW,
+		CREATE ROUTINE,
+		ALTER ROUTINE,
+		EXECUTE
+	ON *.* TO developerBoss;
--- a/roles/db/files/mariabackup/apply.test.sql
+++ b/roles/db/files/mariabackup/apply.test.sql
@ -0,0 +1,6 @@
+UPDATE vn2008.tblContadores
+	SET dbproduccion = FALSE;
+
+DELETE FROM util.binlogQueue;
+
+UPDATE `account`.`user` SET `active` = TRUE WHERE `name` = 'mindshore';
--- a/roles/db/files/mariabackup/bacula-after.sh
+++ b/roles/db/files/mariabackup/bacula-after.sh
@ -0,0 +1,51 @@
+#!/bin/bash
+set -e
+
+myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$myDir/config.sh"
+. "$myDir/apply.config.sh"
+
+todayDir=$(date +%Y-%m-%d)
+pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz"
+files=($pattern)
+backupFile="${files[0]}"
+
+"$myDir/restore-backup.sh" "$backupFile"
+rm -r "$baculaDir"
+
+if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then
+	for node in "${dbClusterSiblings[@]}"; do
+		ssh root@$node service mysql stop
+		ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi"
+	done
+	
+	galera_new_cluster
+else
+	service mariadb start
+fi
+
+echo "Applying custom script."
+mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()"
+mysql < "$myDir/apply.sql"
+
+echo "Upgrading tables."
+mysql_upgrade
+
+echo "Applying repository changes."
+curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec"
+
+echo "Waiting for Jenkins job to end."
+jobResult=null
+while [ "$jobResult" = "null" ]; do
+	sleep 10
+	jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result")
+done
+echo "Job result: $jobResult"
+
+echo "Promoting to master."
+"/root/scripts/promote-master.sh"
+
+for node in "${dbClusterSiblings[@]}"; do
+	ssh root@$node service mysql start
+done
+
--- a/roles/db/files/nrpe/95-mariadb.cfg
+++ b/roles/db/files/nrpe/95-mariadb.cfg
@ -0,0 +1,4 @@
+command[check_disk_mysqldata]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/mysqldata
+command[check_disk_mysqlbin]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/mysqlbin
+command[check_disk_backup]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/local-backup
+command[check_mysql_scheduler]=/etc/nagios/plugins/check_mysql_scheduler
--- a/roles/db/tasks/production.yml
+++ b/roles/db/tasks/production.yml
@ -0,0 +1,26 @@
+- name: Ensure production required directories exist
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ item.owner }}"
+    group: "{{ item.group }}"
+    mode: "{{ item.mode }}"
+  loop: "{{ required_directories_master }}"
+
+- name: Set production MariaDB custom configuration
+  copy:
+    src: conf/z95-production.cnf
+    dest: /etc/mysql/mariadb.conf.d/
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify: restart-mariadb
+
+- name: Reminder to check production mount points
+  debug:
+    msg: |
+      Remember to check the following mount points:
+      - /var/lib/mysql
+      - /mnt/mysqlbin
+      - /mnt/local-backup
+      Make sure they are correctly configured and accessible.
--- a/roles/db/tasks/test.yml
+++ b/roles/db/tasks/test.yml
@ -0,0 +1,39 @@
+- name: Set test Mariabackup files
+  copy:
+    src: mariabackup/bacula-after.sh
+    dest: /root/mariabackup/
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=rx
+
+- name: Set test MariaBackup apply config template configuration
+  template:
+    src: mariabackup/apply.config.sh
+    dest: /root/mariabackup/
+    owner: root
+    group: root
+    mode: u=rw,g=,o=
+
+- name: Set test MariaBackup apply SQL template configuration
+  copy:
+    src: mariabackup/apply.{{db.environment}}.sql
+    dest: /root/mariabackup/apply.sql
+    owner: root
+    group: root
+    mode: u=rw,g=,o=
+
+- name: Set test MariaDB custom configuration
+  copy:
+    src: conf/z92-test.cnf
+    dest: /etc/mysql/mariadb.conf.d/
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify: restart-mariadb
+
+- name: Reminder to check test mount points environment
+  debug:
+    msg: |
+      Remember to check the following mount points:
+      - /mnt/mysqltmp
+      Make sure they are correctly configured and accessible.
--- a/roles/db/templates/conf/z99-local.cnf
+++ b/roles/db/templates/conf/z99-local.cnf
@ -0,0 +1,7 @@
+[mysqld]
+
+server-id                                   = {{ serverid }}
+#bind-address                               = 127.0.0.1
+#event-scheduler                            = OFF
+#skip-log-bin
+#skip-slave-start
--- a/roles/db/templates/mariabackup/apply.config.sh
+++ b/roles/db/templates/mariabackup/apply.config.sh
@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Bacula directory for restore
+baculaDir=/mnt/mysqldata/bacula-restore
+
+# Database branch name
+dbBranch={{ db.branch  }}
+
+# Database environment
+dbEnvironment={{ db.environment }}
+
+# MariaDB cluster sibling node hostnames
+dbClusterSiblings=()
+
+# Jenkins authentication string
+jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt_folder).password }}
+
+# Jenkins job URL
+jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ db.branch }}
--- a/roles/db/templates/mariabackup/my.cnf
+++ b/roles/db/templates/mariabackup/my.cnf
@ -0,0 +1,7 @@
+[mariabackup]
+host = localhost
+user = mariabackup
+password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }}
+use-memory = 1G
+parallel = 2
+stream = mbstream