Merge branch 'main' into 8414-Nrpe_Nagios_95-mariadb.cfg

roles/debian-base/defaults/main.yaml

@@ -32,6 +32,7 @@ base_packages:
   - cloud-guest-utils
   - rsync
   - parted
+  - yq
 locales_present:
   - en_US.UTF-8
   - es_ES.UTF-8

roles/debian-base/tasks/bacula.yml

@@ -34,7 +34,7 @@
   register: bacula_config
 - name: Configure master cert
   copy:
-    content: "{{ master_cert_content }}"
+    content: "{{ ca_bacula }}"
     dest: /etc/bacula/master-cert.pem
     owner: root
     group: root

roles/pve/handlers/main.yml

@@ -2,7 +2,4 @@
   service:
     name: nagios-nrpe-server
     state: restarted
-- name: restart-sysctl
+
-  service:
-    name: systemd-sysctl
-    state: restarted

roles/pve/tasks/vhost.yml

@@ -5,4 +5,10 @@
     mode: u=rw,g=r,o=r
     owner: root
     group: root
-  notify: restart-sysctl
+  register: copy_result
+
+- name: Reboot the system if file was copied
+  reboot:
+    reboot_timeout: 600
+  become: true
+  when: copy_result.changed

roles/services/defaults/main.yaml

@@ -1,4 +1,4 @@
-realm: "{{domain}}.{{resolv_domain}}"
+realm: "{{domain}}.{{host_domain}}"
 samba_client_services:  
   - smbd 
   - nmbd 
@@ -23,8 +23,9 @@ mariadb_requeriments:
   - curl
   - apt-transport-https
 certificates:
-  - { content: '{{ ca_mysql }}', dest: '/etc/mysql/ca.pem', mode: 'u=rw,g=r,o=r' }
+  - { content: '{{ ca_company_deprecated }}', dest: '/etc/mysql/ca.pem', mode: 'u=rw,g=r,o=r' }
   - { content: '{{ cert_mysql }}', dest: '/etc/mysql/cert.pem', mode: 'u=rw,g=r,o=r' }
+  - { content: '{{ cert_mysql_key }}', dest: '/etc/mysql/key.pem', mode: 'u=rw,g=,o=' }
 required_directories:
   - { path: /mnt/local-backup, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
   - { path: /mnt/mysqlbin, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
@@ -32,13 +33,15 @@ required_directories:
   - { path: /mnt/mysqlbin/binlog, owner: mysql, group: mysql, mode: 'u=rwx,g=,o=' }
   - { path: /root/scripts, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
   - { path: /root/mariabackup, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }
+  - { path: /mnt/mysqldata/mysql, owner: mysql, group: mysql, mode: 'u=rwx,g=rx,o=rx' }
+  - { path: /etc/systemd/system/mariadb.service.d, owner: root, group: root, mode: 'u=rwx,g=rx,o=rx' }  
+
 required_files_and_mariabackup_files_and_root_scripts:
   - { src: "mariadb_override.conf", dest: "/etc/systemd/system/mariadb.service.d/override.conf", mode: "u=rw,g=r,o=r" }
-  - { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" }
+  - { src: "mysql-flush.sh", dest: "/etc/qemu/fsfreeze-hook.d/mysql-flush.sh", mode: "u=rwx,g=rx,o=rx" }  
   - { src: "files/mariabackup/bacula-before.sh", dest: "/root/mariabackup/bacula-before.sh", mode: "u=rwx,g=rx,o=rx" }
   - { src: "files/mariabackup/config.sh", dest: "/root/mariabackup/config.sh", mode: "u=rwx,g=rx,o=x" }
   - { src: "files/mariabackup/inc-backup.sh", dest: "/root/mariabackup/inc-backup.sh", mode: "u=rwx,g=rx,o=rx" }
-  - { src: "files/mariabackup/my.cnf", dest: "/root/mariabackup/my.cnf", mode: "u=rw,g=,o=" }
   - { src: "files/mariabackup/restore-backup.sh", dest: "/root/mariabackup/restore-backup.sh", mode: "u=rwx,g=rx,o=rx" }
   - { src: "files/scripts/check-memory.sh", dest: "/root/scripts/check-memory.sh", mode: "u=rwx,g=rx,o=rx" }
   - { src: "files/scripts/export-privs.sh", dest: "/root/scripts/export-privs.sh", mode: "u=rwx,g=rx,o=rx" }

roles/services/files/mariabackup/my.cnf

@@ -1,7 +0,0 @@
-[mariabackup]
-host = localhost
-user = {{ user_mariabackup }}
-password = {{ password_user_mariabackup }}
-use-memory = 1G
-parallel = 2
-stream = mbstream

roles/services/files/mariabackupdev/apply.sql

@@ -0,0 +1,23 @@
+UPDATE vn2008.tblContadores
+	SET dbproduccion = FALSE;
+
+DELETE FROM util.binlogQueue;
+
+GRANT
+		SELECT,
+		INSERT,
+		UPDATE,
+		DELETE,
+		CREATE,
+		DROP,
+		INDEX,
+		ALTER,
+		CREATE TEMPORARY TABLES,
+		CREATE VIEW,
+		EVENT,
+		TRIGGER,
+		SHOW VIEW,
+		CREATE ROUTINE,
+		ALTER ROUTINE,
+		EXECUTE
+	ON *.* TO developerBoss;

roles/services/files/mariabackupdev/bacula-after.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -e
+
+myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$myDir/config.sh"
+. "$myDir/apply.config.sh"
+
+todayDir=$(date +%Y-%m-%d)
+pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz"
+files=($pattern)
+backupFile="${files[0]}"
+
+"$myDir/restore-backup.sh" "$backupFile"
+rm -r "$baculaDir"
+
+if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then
+	for node in "${dbClusterSiblings[@]}"; do
+		ssh root@$node service mysql stop
+		ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi"
+	done
+	
+	galera_new_cluster
+else
+	service mariadb start
+fi
+
+echo "Applying custom script."
+mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()"
+mysql < "$myDir/apply.sql"
+
+echo "Upgrading tables."
+mysql_upgrade
+
+echo "Applying repository changes."
+curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec"
+
+echo "Waiting for Jenkins job to end."
+jobResult=null
+while [ "$jobResult" = "null" ]; do
+	sleep 10
+	jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result")
+done
+echo "Job result: $jobResult"
+
+echo "Promoting to master."
+"/root/scripts/promote-master.sh"
+
+for node in "${dbClusterSiblings[@]}"; do
+	ssh root@$node service mysql start
+done
+

roles/services/files/mariabackuptest/apply.sql

@@ -0,0 +1,6 @@
+UPDATE vn2008.tblContadores
+	SET dbproduccion = FALSE;
+
+DELETE FROM util.binlogQueue;
+
+UPDATE `account`.`user` SET `active` = TRUE WHERE `name` = 'mindshore';

roles/services/files/mariabackuptest/bacula-after.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -e
+
+myDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$myDir/config.sh"
+. "$myDir/apply.config.sh"
+
+todayDir=$(date +%Y-%m-%d)
+pattern="$baculaDir/mnt/local-backup/${todayDir}_??-??_full.gz"
+files=($pattern)
+backupFile="${files[0]}"
+
+"$myDir/restore-backup.sh" "$backupFile"
+rm -r "$baculaDir"
+
+if [[ "${#dbClusterSiblings[@]}" -gt "0" ]]; then
+	for node in "${dbClusterSiblings[@]}"; do
+		ssh root@$node service mysql stop
+		ssh root@$node "if pgrep mariadbd; then pkill -9 mariadbd; fi"
+	done
+	
+	galera_new_cluster
+else
+	service mariadb start
+fi
+
+echo "Applying custom script."
+mysql -e "UPDATE util.config SET environment = '$dbEnvironment', lastDump = NOW()"
+mysql < "$myDir/apply.sql"
+
+echo "Upgrading tables."
+mysql_upgrade
+
+echo "Applying repository changes."
+curl --silent --request POST --location --user "$jenkinsAuth" "$jenkinsUrl/build?delay=0sec"
+
+echo "Waiting for Jenkins job to end."
+jobResult=null
+while [ "$jobResult" = "null" ]; do
+	sleep 10
+	jobResult=$(curl --silent --location --user "$jenkinsAuth" "$jenkinsUrl/lastBuild/api/json" | jq --raw-output ".result")
+done
+echo "Job result: $jobResult"
+
+echo "Promoting to master."
+"/root/scripts/promote-master.sh"
+
+for node in "${dbClusterSiblings[@]}"; do
+	ssh root@$node service mysql start
+done
+

roles/services/files/scripts/sync-conf.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-partner=root@db2.static.verdnatura.es
-confDir=/etc/mysql/mariadb.conf.d
-files=(
-	z90-vn.cnf
-	z95-production.cnf
-)
-
-#echo "Reloading service."
-#service mariadb reload
-
-if [ $? -eq "0" ]; then
-	echo "Synchronizing partner configuration."
-	for file in "${files[@]}"; do 
-        	scp "$confDir/$file" $partner:$confDir
-	done
-
-	#echo "Reloading partner service."
-	#ssh $partner service mariadb reload
-fi

roles/services/handlers/main.yml

@@ -2,7 +2,7 @@
   systemd:
     name: chrony
     state: restarted
-- name: reload systemd
+- name: reload-systemd
   command:
     cmd: systemctl daemon-reload
 - name: restart-mariadb

roles/services/tasks/adsamba.yml

@@ -14,71 +14,62 @@
 # apt install ldb-tools
 #
 # samba-tool domain provision --use-rfc2307 --interactive 
+#
+# If we want to go 4.21
+# - name: Add Debian backports repository
+#   apt_repository:
+#     repo: "deb http://deb.debian.org/debian {{ ansible_distribution_release | lower }}-backports main"
+#     state: present
+#
+# - name: Update apt cache
+#   apt:
+#     update_cache: yes
 
-- name: Gather installed packages
+- name: Install adSamba packages
-  package_facts:
+  package:
+    name: "{{ dcsamba_base_packages }}"
+    # default_release: bookworm-backports # If we want to go 4.21
+    state: latest
 
-- name: Check if metadata.tdb exists and is not empty
+- name: Add adsamba host to hosts file
+  blockinfile:
+    path: /etc/hosts
+    marker: "# {mark} ANSIBLE-MANAGED SAMBA DC ENTRY"
+    block: |
+      {{ ip_serverad | default(ansible_default_ipv4.address) }}  {{ ansible_facts['hostname'] }}.{{ domain }}.{{ host_domain }} {{ realm }}
+
+- name: Check if metadata.tdb exists
   stat:
     path: /var/lib/samba/private/sam.ldb.d/metadata.tdb
   register: metadata_tdb
 
-- name: Register domain existence
+- when: metadata_tdb.stat.exists is false
-  set_fact:
-    domain_exists: >-
-      {{
-        ('samba-ad-provision' in ansible_facts.packages or
-        'samba-ad-dc' in ansible_facts.packages) and
-        (metadata_tdb.stat.exists and metadata_tdb.stat.size > 0)
-      }}
-
-# If we want to go 4.21
-#- name: Add Debian backports repository
-#  apt_repository:
-#    repo: "deb http://deb.debian.org/debian {{ ansible_distribution_release | lower }}-backports main"
-#    state: present
-
-#- name: Update apt cache
-#  apt:
-#    update_cache: yes
-
-- when: "not domain_exists"
   block:
-  - name: Install adSamba packages
+
-    package:
-      name: "{{ dcsamba_base_packages }}"
-      # default_release: bookworm-backports # If we want to go 4.21
-      state: latest
-  
-  - name: Add adsamba host to hosts file
-    blockinfile:
-      path: /etc/hosts
-      marker: "# {mark} ANSIBLE-MANAGED SAMBA DC ENTRY"
-      block: |
-        {{ ip_serverad1 }} {{ name_ip_serverad1}}.{{ domain }}.{{ resolv_domain }} {{ realm }}
-  
   - name: Force remove smb.conf file
     file:
       path: /etc/samba/smb.conf
       state: absent
       force: yes
-  
-  - name: Disable Samba client services and mask them
-    systemd:
-      name: "{{ item }}"
-      state: stopped
-      enabled: no
-      masked: yes
-    loop: "{{ samba_client_services }}"
-  
-  - name: Join domain 
-    command:
-      cmd: samba-tool domain provision --realm="{{ realm }}" --domain="{{ domain }}" --dns-backend=SAMBA_INTERNAL --server-role=dc --use-rfc2307
-    register: domain_join
 
-  - name: Show the domain join output with Administrator password
+  - when: main_ad is true
-    debug:
+    block:
-      msg: "{{ domain_join.stderr_lines[-6:] }}"
+    - name: Provision domain 
+      command:
+        cmd: samba-tool domain provision --realm="{{ realm }}" --domain="{{ domain }}" --dns-backend=SAMBA_INTERNAL --server-role=dc --use-rfc2307
+      register: domain_join
+      
+    - name: Show the domain join output with Administrator password
+      debug:
+        msg: "{{ domain_join.stderr_lines[-6:] }}"
+    
+  - when: main_ad is false
+    block:
+
+    - name: Join domain
+      shell: samba-tool domain join "{{ realm }}" DC -U"{{ domain | upper }}\administrator"
+      environment:
+        PASSWD: "{{ lookup(passbolt, 'ad_admin_password', folder_parent_id=passbolt_folder).password }}"
 
   - name: Copy Kerberos configuration
     copy:
@@ -88,18 +79,43 @@
       owner: root
       group: root
       mode: '0644'
-    when: domain_join.changed
 
   - name: Enable and start Samba AD DC service
     systemd:
       name: samba-ad-dc
       state: started
       enabled: yes
-    when: domain_join.changed
+
-  
+- name: Disable Samba client services and mask them
-  # Hay que crear un registro tipo A con el FQDN es decir el GLUE RECORD en tu DNS
+  systemd:
-  # Luego hay que crear la delegación # update add activedirectory.verdnatura.es. 86400 NS dc1-ad.activedirectory.verdnatura.es.
+    name: "{{ item }}"
-  # Bloque de inizialización del dominio ## Hecho
+    state: stopped
-  # Revisar la condicion de domain_join ## Hecho.
+    enabled: no
-  # Cuidado con la copia de KRB5, revisar si lo hace ya el samba-tool. ## Hecho.
+    masked: yes
-  
+  loop: "{{ samba_client_services }}"
+
+- name: Add A record to DNS
+  nsupdate:
+    key_name: 'rndc-key'
+    key_secret: "{{ lookup(passbolt, 'rndc-key', folder_parent_id=passbolt_folder).password }}"
+    key_algorithm: '{{ key_algorithm }}'
+    server: "{{ main_dns_server }}"
+    zone: '{{ host_domain }}'
+    ttl: '{{ ttl }}'
+    type: 'A'
+    record: '{{ inventory_hostname_short }}.{{ realm }}.'
+    value: '{{ ip_serverad }}'
+    state: present
+
+- name: Add NS record to DNS
+  nsupdate:
+    key_name: 'rndc-key'
+    key_secret: "{{ lookup(passbolt, 'rndc-key', folder_parent_id=passbolt_folder).password }}"
+    key_algorithm: '{{ key_algorithm }}'
+    server: '{{ main_dns_server }}'
+    zone: '{{ host_domain }}'
+    ttl: '{{ ttl }}'
+    type: 'NS'
+    record: '{{ realm }}.'
+    value: '{{ inventory_hostname_short }}.{{ realm }}.'
+    state: present

roles/services/tasks/mariadb.yml

@@ -48,14 +48,14 @@
     group: root
     mode: "{{ item.mode }}"
   loop: "{{ required_files_and_mariabackup_files_and_root_scripts }}"
-  notify: reload systemd
+  notify: reload-systemd
 
 - name: Add tmpfs in /etc/fstab
   blockinfile:
     path: /etc/fstab
     marker: "# {mark} ANSIBLE-MANAGED TMPFS ENTRY"
     block: |
-      tmpfs /mnt/mysqltmp         tmpfs rw,size=6144M         0 0
+      tmpfs /mnt/mysqltmp         tmpfs rw,size={{ mysqltmpsize }}         0 0
   register: fstab
 
 - name: Mount all filesystems from /etc/fstab
@@ -71,6 +71,7 @@
     mode: u=rw,g=r,o=r
 
 - name: Insert MySQL certificates
+  no_log: true
   copy:
     content: "{{ item.content }}"
     dest: "{{ item.dest }}"
@@ -80,15 +81,7 @@
   loop: "{{ certificates }}"
   notify: restart-mariadb
 
-- name: Configure MySQL master cert
+- when: db.branch == 'production'
-  copy:
-    content: "{{ lookup(passbolt, 'private_mysql', folder_parent_id=passbolt_folder).description }}"
-    dest: /etc/mysql/key.pem
-    owner: mysql
-    group: mysql
-    mode: u=rw,g=,o=
-
-- when: production is true
   block:
   - name: Set MariaDB custom configuration Production
     copy:
@@ -98,8 +91,8 @@
       group: root
       mode: u=rw,g=r,o=r
     with_items:
-      - "files/mariaconfigfiles/z90-vn.cnf"
+      - "mariaconfigfiles/z90-vn.cnf"
-      - "files/mariaconfigfiles/z95-production"
+      - "mariaconfigfiles/z95-production.cnf"
     notify: restart-mariadb
   
   - name: Reminder to check mount points
@@ -111,9 +104,10 @@
         - /mnt/local-backup
         Make sure they are correctly configured and accessible.
 
-- when: testdb is true
+- when: db.branch == 'test'
+
   block:
-  - name: Set MariaDB custom configuration Test
+  - name: Set MariaDB custom configuration TestDB
     copy:
       src: "{{ item }}"
       dest: /etc/mysql/mariadb.conf.d/
@@ -122,20 +116,111 @@
       mode: u=rw,g=r,o=r
     with_items:
       - "files/mariaconfigfiles/z90-vn.cnf"
-      - "files/mariaconfigfiles/z92-testing.cnf"
+      - "files/mariaconfigfiles/z92-testing_dev.cnf"
     notify: restart-mariadb
+
+  - name: Set MariaBackup custom template configuration TestDB
+    template:
+      src: test-db_dev/apply.config.sh
+      dest: /root/mariabackup/
+      owner: root
+      group: root
+      mode: u=rw,g=,o=
+
+  - name: Set MariaDB local configuration file for TestDB
+    copy:
+      src: "{{ item }}"
+      dest: /root/mariabackup/
+      owner: root
+      group: root
+      mode: u=rw,g=r,o=r
+    with_items:
+      - "mariabackuptest/apply.sql"
+      - "mariabackuptest/bacula-after.sh"
   
-  - name: Reminder to check mount points
+  - name: Reminder to check mount points environment TestDB
     debug:
       msg: |
         Remember to check the following mount points:
         - /mnt/mysqltmp
         Make sure they are correctly configured and accessible.
 
-- name: Set MariaDB local configuration file
+- when: db.branch == 'dev'
+
+  block:
+  - name: Set MariaDB custom configuration DevDB
+    copy:
+      src: "{{ item }}"
+      dest: /etc/mysql/mariadb.conf.d/
+      owner: root
+      group: root
+      mode: u=rw,g=r,o=r
+    with_items:
+      - "files/mariaconfigfiles/z90-vn.cnf"
+      - "files/mariaconfigfiles/z92-testing_dev.cnf"
+    notify: restart-mariadb
+
+  - name: Set MariaBackup custom template configuration DevDB
+    template:
+      src: test-db_dev/apply.config.sh
+      dest: /root/mariabackup/
+      owner: root
+      group: root
+      mode: u=rw,g=,o=
+
+  - name: Set MariaDB local configuration file for DevDB
+    copy:
+      src: "{{ item }}"
+      dest: /root/mariabackup/
+      owner: root
+      group: root
+      mode: u=rw,g=r,o=r
+    with_items:
+      - "mariabackupdev/apply.sql"
+      - "mariabackupdev/bacula-after.sh"
+  
+  - name: Reminder to check mount points environment DevDB
+    debug:
+      msg: |
+        Remember to check the following mount points:
+        - /mnt/mysqltmp
+        Make sure they are correctly configured and accessible.
+
+- name: Set Custom Configuration local template all Environment
   template:
-    src: templates/z99-local.cnf
+    src: z99-local.cnf
     dest: /etc/mysql/mariadb.conf.d/
     owner: root
     group: root
-    mode: u=rw,g=r,o=r
+    mode: u=rw,g=r,o=r
+
+- name: Set MariaBackup custom template configuration all Environment
+  template:
+    src: "my.cnf"
+    dest: /root/mariabackup/
+    owner: root
+    group: root
+    mode: u=rw,g=,o=
+
+- name: Check if /var/lib/mysql/ exists
+  stat:
+    path: /var/lib/mysql/
+  register: mysql_dir
+
+
+- when: mysql_dir.stat.exists
+  block:
+
+  - name: Sync MySQL data directory
+    synchronize:
+      src: /var/lib/mysql/
+      dest: /mnt/mysqldata/mysql/
+      archive: true
+      compress: true
+      recursive: true
+    delegate_to: "{{ inventory_hostname }}"
+  
+  - name: Remove old MySQL data after sync
+    file:
+      path: /var/lib/mysql/
+      state: absent

roles/services/templates/my.cnf

@@ -0,0 +1,7 @@
+[mariabackup]
+host = localhost
+user = mariabackup
+password = {{ lookup(passbolt, 'mariabackup', folder_parent_id=passbolt_folder).password }}
+use-memory = 1G
+parallel = 2
+stream = mbstream

roles/services/templates/test-db_dev/apply.config.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Bacula directory for restore
+baculaDir=/mnt/mysqldata/bacula-restore
+
+# Database branch name
+dbBranch={{ db.branch  }}
+
+# Database environment
+dbEnvironment={{ dbEnvironment }}
+
+# MariaDB cluster sibling node hostnames
+dbClusterSiblings=()
+
+# Jenkins authentication string
+jenkinsAuth=jenkins:{{ lookup(passbolt, 'jenkinsAuth', folder_parent_id=passbolt_folder).password }}
+
+# Jenkins job URL
+jenkinsUrl=https://jenkins.verdnatura.es/job/Scheduler/job/db-apply-changes-{{ db.branch }}
+