verdnatura/vn-ansible
verdnatura
/
vn-ansible
1
0
Fork 0
Code Issues Pull Requests 2 Packages Projects Releases Wiki Activity

db: refs #8140 Fix apply SQL, dir structure changes

This commit is contained in:
Juan Ferrer 2025-02-10 13:38:45 +01:00
parent 4bee560a8d
commit 15ae174c18
22 changed files with 65 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
playbooks/ceph.yml
View File

@ -5,3 +5,4 @@
name: debian
- import_role:
name: ceph
tags: service

1
playbooks/db.yml
View File

@ -5,3 +5,4 @@
name: debian
- import_role:
name: db
tags: service

1
playbooks/kube.yml
View File

@ -5,3 +5,4 @@
name: debian
- import_role:
name: kube
tags: service

1
playbooks/pve.yml
View File

@ -5,3 +5,4 @@
name: debian
- import_role:
name: pve
tags: service

29
roles/db/defaults/main.yaml
View File

@ -19,24 +19,19 @@ required_directories:
- { path: /root/mariabackup, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
- { path: /mnt/mysqldata/mysql, owner: mysql, group: mysql, mode: 'u=rwx,g=rx,o=rx' }
- { path: /etc/systemd/system/mariadb.service.d, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
required_files_and_mariabackup_files_and_root_scripts:
- { src: "mariadb_override.conf", dest: "/etc/systemd/system/mariadb.service.d/override.conf", mode: "u=rw,g=r,o=r" }
required_mariabackup_files_and_scripts:
- { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" }
- { src: "files/mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/mariabackup/restore-backup.sh", dest: "/root/mariabackup/restore-backup.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/scripts/check-memory.sh", dest: "/root/scripts/check-memory.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/scripts/export-privs.sh", dest: "/root/scripts/export-privs.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/scripts/mysqltuner.pl", dest: "/root/scripts/mysqltuner.pl", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/scripts/promote-master.sh", dest: "/root/scripts/promote-master.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/scripts/promote-slave.sh", dest: "/root/scripts/promote-slave.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/scripts/README.md", dest: "/root/scripts/README.md", mode: "u=rw,g=r,o=r" }
- { src: "files/scripts/scheduler-log.sh", dest: "/root/scripts/scheduler-log.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "files/marianrpe/95-mariadb.cfg", dest: "/etc/nagios/nrpe.d/95-mariadb.cfg", mode: "u=rw,g=r,o=r" }
mariabackup_test_dev_files:
- { src: "mariabackuptest/apply.sql", dest: "/root/mariabackup/", mode: "u=rw,g=r,o=r" }
- { src: "mariabackuptest/bacula-after.sh", dest: "/root/mariabackup/", mode: "u=rwx,g=rx,o=rx" }
- { src: "mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" }
- { src: "mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "mariabackup/restore-backup.sh", dest: "/root/mariabackup/restore-backup.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "scripts/check-memory.sh", dest: "/root/scripts/check-memory.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "scripts/export-privs.sh", dest: "/root/scripts/export-privs.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "scripts/mysqltuner.pl", dest: "/root/scripts/mysqltuner.pl", mode: "u=rwx,g=rx,o=rx" }
- { src: "scripts/promote-master.sh", dest: "/root/scripts/promote-master.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "scripts/promote-slave.sh", dest: "/root/scripts/promote-slave.sh", mode: "u=rwx,g=rx,o=rx" }
- { src: "scripts/README.md", dest: "/root/scripts/README.md", mode: "u=rw,g=r,o=r" }
- { src: "scripts/scheduler-log.sh", dest: "/root/scripts/scheduler-log.sh", mode: "u=rwx,g=rx,o=rx" }
downloads:
- { url: "https://r.mariadb.com/downloads/mariadb_repo_setup", dest: "/tmp/mariadb_repo_setup", mode: "u=rwx,g=rx,o=rx" }
- { url: "https://repo.percona.com/apt/percona-release_latest.generic_all.deb", dest: "/tmp/percona-release_latest.generic_all.deb", mode: "u=rw,g=r,o=r" }

0
roles/db/files/mariabackup/apply.development.sql → roles/db/files/mariabackup/apply.dev.sql
View File

23
roles/db/files/mariabackupdev/apply.sql
View File

@ -1,23 +0,0 @@
UPDATE vn2008.tblContadores
SET dbproduccion = FALSE;
DELETE FROM util.binlogQueue;
GRANT
SELECT,
INSERT,
UPDATE,
DELETE,
CREATE,
DROP,
INDEX,
ALTER,
CREATE TEMPORARY TABLES,
CREATE VIEW,
EVENT,
TRIGGER,
SHOW VIEW,
CREATE ROUTINE,
ALTER ROUTINE,
EXECUTE
ON *.* TO developerBoss;

51
roles/db/files/mariabackupdev/bacula-after.sh
View File

@ -1,51 +0,0 @@
#!/bin/bash
set -e
myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
. "$myDir/config.sh"
. "$myDir/apply.config.sh"
todayDir=$(date +%Y-%m-%d)
pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz"
files=($pattern)
backupFile="${files[0]}"
"$myDir/restore-backup.sh" "$backupFile"
rm -r "$baculaDir"
if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then
for node in "${dbClusterSiblings[@]}"; do
ssh root@$node service mysql stop
ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi"
done
galera_new_cluster
else
service mariadb start
fi
echo "Applying custom script."
mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()"
mysql < "$myDir/apply.sql"
echo "Upgrading tables."
mysql_upgrade
echo "Applying repository changes."
curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec"
echo "Waiting for Jenkins job to end."
jobResult=null
while [ "$jobResult" = "null" ]; do
sleep 10
jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result")
done
echo "Job result: $jobResult"
echo "Promoting to master."
"/root/scripts/promote-master.sh"
for node in "${dbClusterSiblings[@]}"; do
ssh root@$node service mysql start
done

6
roles/db/files/mariabackuptest/apply.sql
View File

@ -1,6 +0,0 @@
UPDATE vn2008.tblContadores
SET dbproduccion = FALSE;
DELETE FROM util.binlogQueue;
UPDATE `account`.`user` SET `active` = TRUE WHERE `name` = 'mindshore';

51
roles/db/files/mariabackuptest/bacula-after.sh
View File

@ -1,51 +0,0 @@
#!/bin/bash
set -e
myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
. "$myDir/config.sh"
. "$myDir/apply.config.sh"
todayDir=$(date +%Y-%m-%d)
pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz"
files=($pattern)
backupFile="${files[0]}"
"$myDir/restore-backup.sh" "$backupFile"
rm -r "$baculaDir"
if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then
for node in "${dbClusterSiblings[@]}"; do
ssh root@$node service mysql stop
ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi"
done
galera_new_cluster
else
service mariadb start
fi
echo "Applying custom script."
mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()"
mysql < "$myDir/apply.sql"
echo "Upgrading tables."
mysql_upgrade
echo "Applying repository changes."
curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec"
echo "Waiting for Jenkins job to end."
jobResult=null
while [ "$jobResult" = "null" ]; do
sleep 10
jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result")
done
echo "Job result: $jobResult"
echo "Promoting to master."
"/root/scripts/promote-master.sh"
for node in "${dbClusterSiblings[@]}"; do
ssh root@$node service mysql start
done

98
roles/db/files/mariaconfigfiles/z90-vn.cnf
View File

@ -1,98 +0,0 @@
[mysqld]
# Docs: https://mariadb.com/kb/en/server-system-variables
lc_messages = es_ES
lc_time_names = es_ES
character-set-server = utf8
collation-server = utf8_unicode_ci
explicit_defaults_for_timestamp = ON
datadir = /mnt/mysqldata/mysql
tmpdir = /mnt/mysqltmp
log_bin_trust_function_creators = 1
sql_mode = NO_ENGINE_SUBSTITUTION
bind-address = 0.0.0.0
max_password_errors = 50
#++++++++++++++++++++++++++++++++++++++++ Threads
thread_stack = 512K
join_buffer_size = 2M
sort_buffer_size = 4M
net_buffer_length = 256K
max_allowed_packet = 16M
read_buffer_size = 1M
read_rnd_buffer_size = 512K
#++++++++++++++++++++++++++++++++++++++++ Performance
thread_cache_size = 450
interactive_timeout = 1800
wait_timeout = 1800
open_files_limit = 20000
low_priority_updates = 1
table_open_cache = 40000
table_definition_cache = 10000
table_open_cache_instances = 1
key_buffer_size = 256K
max_heap_table_size = 128M
tmp_table_size = 128M
concurrent_insert = ALWAYS
group_concat_max_len = 10000
max_connect_errors = 50
#++++++++++++++++++++++++++++++++++++++++ Binary log
max_binlog_size = 1GB
binlog_cache_size = 16M
binlog_stmt_cache_size = 16M
binlog_row_image = noblob
binlog_format = row
#++++++++++++++++++++++++++++++++++++++++ InnoDB
transaction-isolation = READ-COMMITTED
idle_transaction_timeout = 60
innodb_io_capacity = 100
innodb_io_capacity_max = 100
innodb_monitor_enable = all
innodb_read_io_threads = 16
innodb_write_io_threads = 16
innodb_checksum_algorithm = crc32
innodb_adaptive_hash_index = 0
innodb_flush_method = O_DIRECT
innodb_log_buffer_size = 32M
innodb_purge_threads = 4
innodb_buffer_pool_dump_at_shutdown = ON
innodb_buffer_pool_load_at_startup = ON
#++++++++++++++++++++++++++++++++++++++++ Logging
log_error = /var/log/mysql/error.log
log_output = TABLE
general_log = OFF
slow_query_log = ON
long_query_time = 2
min_examined_row_limit = 0
log_slow_admin_statements = ON
log_queries_not_using_indexes = OFF
max_error_count = 100
#++++++++++++++++++++++++++++++++++++++++ SSL
ssl-ca = /etc/mysql/ca.pem
ssl-cert = /etc/mysql/cert.pem
ssl-key = /etc/mysql/key.pem
#++++++++++++++++++++++++++++++++++++++++ Query cache
query_cache_limit = 0
query_cache_type = OFF
query_cache_size = 0
#++++++++++++++++++++++++++++++++++++++++ Performance Schema
performance_schema = ON
performance_schema_digests_size = 20000
performance-schema-consumer-events-statements-history = ON
performance_schema_consumer_events_statements_history_long = ON
userstat = ON

14
roles/db/files/mariaconfigfiles/z92-testing_dev.cnf
View File

@ -1,14 +0,0 @@
[mysqld]
port = 3307
bind-address = 0.0.0.0
innodb_buffer_pool_size = 18G
event-scheduler = OFF
innodb_log_file_size = 5G
log_warnings = 2
#++++++++++++++++++++++++++++++++++++++++ Binary log
log-bin = bin.log
expire_logs_days = 1
relay_log = mysqld-relay-bin

29
roles/db/files/mariaconfigfiles/z95-production.cnf
View File

@ -1,29 +0,0 @@
[mysqld]
port = 3306
innodb_log_file_size = 8G
log_warnings = 1
#++++++++++++++++++++++++++++++++++++++++ Binary log
log-bin = /mnt/mysqlbin/binlog/bin.log
max_connections = 1000
expire_logs_days = 7
innodb_buffer_pool_size = 64G
relay_log = /mnt/mysqlbin/binlog/relay.log
binlog-ignore-db = tmp
binlog-ignore-db = PERCONA_SCHEMA
#++++++++++++++++++++++++++++++++++++++++ Replication
event-scheduler = ON
slave_exec_mode = STRICT
replicate-ignore-db = tmp
replicate-ignore-table = util.eventLog
replicate-ignore-table = cache.cache_calc
replicate-ignore-table = cache.available
replicate-ignore-table = cache.availableNoRaids
replicate-ignore-table = cache.cache_valid
replicate-ignore-table = cache.stock
replicate-ignore-table = cache.visible

4
roles/db/files/marianrpe/95-mariadb.cfg
View File

@ -1,4 +0,0 @@
command[check_disk_mysqldata]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/mysqldata
command[check_disk_mysqlbin]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/mysqlbin
command[check_disk_backup]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/local-backup
command[check_mysql_scheduler]=/etc/nagios/plugins/check_mysql_scheduler

4
roles/db/handlers/main.yml
View File

@ -5,3 +5,7 @@
systemd:
name: mariadb
state: restarted
- name: restart-nrpe
service:
name: nagios-nrpe-server
state: restarted

5
roles/db/tasks/main.yml
View File

@ -1,2 +1,5 @@
- import_tasks: mariadb.yml
tags: service
- when: db.branch == 'master'
import_tasks: production.yml
- when: db.branch in ['dev', 'test']
import_tasks: test.yml

150
roles/db/tasks/mariadb.yml
View File

@ -1,6 +1,3 @@
# Review /root/scripts/check-memory.sh --> It's not optimal to do what this program does
# Also review the cron task /root/scripts/scheduler-log.sh
- name: Ensure Install requirements for MariaDB repository setup script
apt:
name: "{{ mariadb_requeriments }}"
@ -43,7 +40,7 @@
tmpfs /mnt/mysqltmp tmpfs rw,size={{ mysqltmpsize }} 0 0
register: fstab
- name: Set MariaDB Cron to /etc/cron.d
- name: Configure MariaDB memory check CRON
template:
src: check-memory.cron
dest: /etc/cron.d/vn-check-memory
@ -79,115 +76,58 @@
mode: "{{ item.mode }}"
loop: "{{ required_directories }}"
- when: db.branch == 'master'
block:
- name: Copy required MariaBackup files and scripts
copy:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: root
group: root
mode: "{{ item.mode }}"
loop: "{{ required_mariabackup_files_and_scripts }}"
- name: Ensure required directories exist in Production
file:
path: "{{ item.path }}"
state: directory
owner: "{{ item.owner }}"
group: "{{ item.group }}"
mode: "{{ item.mode }}"
loop: "{{ required_directories_master }}"
- name: Set MariaDB custom configuration Production
copy:
src: "{{ item }}"
dest: /etc/mysql/mariadb.conf.d/
owner: root
group: root
mode: u=rw,g=r,o=r
with_items:
- "mariaconfigfiles/z90-vn.cnf"
- "mariaconfigfiles/z95-production.cnf"
notify: restart-mariadb
- name: Reminder to check mount points
debug:
msg: |
Remember to check the following mount points:
- /var/lib/mysql
- /mnt/mysqlbin
- /mnt/local-backup
Make sure they are correctly configured and accessible.
- name: Set MariaDB common configuration
copy:
src: conf/z90-vn.cnf
dest: /etc/mysql/mariadb.conf.d/
owner: root
group: root
mode: u=rw,g=r,o=r
notify: restart-mariadb
- when: db.branch == 'test'
block:
- name: Set MariaBackup custom template configuration TestDB
template:
src: test-db_dev/apply.config.sh
dest: /root/mariabackup/
owner: root
group: root
mode: u=rw,g=,o=
- name: Reminder to check mount points environment TestDB
debug:
msg: |
Remember to check the following mount points:
- /mnt/mysqltmp
Make sure they are correctly configured and accessible.
- when: db.branch == 'dev'
block:
- name: Set MariaBackup custom template configuration DevDB
template:
src: test-db_dev/apply.config.sh
dest: /root/mariabackup/
owner: root
group: root
mode: u=rw,g=,o=
- name: Reminder to check mount points environment DevDB
debug:
msg: |
Remember to check the following mount points:
- /mnt/mysqltmp
Make sure they are correctly configured and accessible.
- when: db.branch == 'dev' or db.branch == 'test'
block:
- name: Set MariaDB custom configuration TestDB or DevDB
copy:
src: "{{ item }}"
dest: /etc/mysql/mariadb.conf.d/
owner: root
group: root
mode: u=rw,g=r,o=r
with_items:
- "files/mariaconfigfiles/z90-vn.cnf"
- "files/mariaconfigfiles/z92-testing_dev.cnf"
notify: restart-mariadb
- name: Set Mariabackup files for TestDB or DevDB
copy:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: root
group: root
mode: "{{ item.mode }}"
loop: "{{ mariabackup_test_dev_files }}"
- name: Set Custom Configuration local template all Environment
- name: Set MariaDB local configuration template
template:
src: z99-local.cnf
src: conf/z99-local.cnf
dest: /etc/mysql/mariadb.conf.d/
owner: root
group: root
mode: u=rw,g=r,o=r
- name: Set MariaBackup custom template configuration all Environment
- name: Set MariaBackup connection configuration
template:
src: "my.cnf"
src: mariabackup/my.cnf
dest: /root/mariabackup/
owner: root
group: root
mode: u=rw,g=,o=
- name: Override MariaDB systemd service configuration
copy:
src: mariadb_override.conf
dest: /etc/systemd/system/mariadb.service.d/override.conf
owner: root
group: root
mode: u=rw,g=r,o=r
notify: reload-systemd
- name: Set MariaDB NRPE configuration
copy:
src: nrpe/95-mariadb.cfg
dest: /etc/nagios/nrpe.d/
owner: root
group: root
mode: u=rw,g=r,o=r
notify: restart-nrpe
- name: Check if /var/lib/mysql/ exists
stat:
path: /var/lib/mysql/
@ -210,16 +150,6 @@
path: /var/lib/mysql/
state: absent
- name: Ensure required custom and Mariabackup files are copied to their destinations and root scripts
copy:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: root
group: root
mode: "{{ item.mode }}"
loop: "{{ required_files_and_mariabackup_files_and_root_scripts }}"
notify: reload-systemd
- name: Mount all filesystems from /etc/fstab
command: mount -a
when: fstab.changed
when: fstab.changed

2
roles/db/tasks/test.yml
View File

@ -16,7 +16,7 @@
- name: Set test MariaBackup apply SQL template configuration
copy:
src: mariabackup/apply.{{db.environment}}.sql
src: mariabackup/apply.{{db.branch}}.sql
dest: /root/mariabackup/apply.sql
owner: root
group: root

7
roles/db/templates/my.cnf
View File

@ -1,7 +0,0 @@
[mariabackup]
host = localhost
user = mariabackup
password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }}
use-memory = 1G
parallel = 2
stream = mbstream

20
roles/db/templates/test-db_dev/apply.config.sh
View File

@ -1,20 +0,0 @@
#!/bin/bash
# Bacula directory for restore
baculaDir=/mnt/mysqldata/bacula-restore
# Database branch name
dbBranch={{ db.branch }}
# Database environment
dbEnvironment={{ db.environment }}
# MariaDB cluster sibling node hostnames
dbClusterSiblings=()
# Jenkins authentication string
jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt_folder).password }}
# Jenkins job URL
jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ db.branch }}

7
roles/db/templates/z99-local.cnf
View File

@ -1,7 +0,0 @@
[mysqld]
server-id = {{ serverid }}
#bind-address = 127.0.0.1
#event-scheduler = OFF
#skip-log-bin
#skip-slave-start

1
roles/pve/handlers/main.yml
View File

@ -2,4 +2,3 @@
service:
name: nagios-nrpe-server
state: restarted