From 6984cd1ece8abffb6472f289b4ee109c185f969a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Xavi=20Lle=C3=B3=20Tom=C3=A1s?= <xavi@verdnatura.es>
Date: Fri, 7 Mar 2025 12:30:56 +0100
Subject: [PATCH] db: refs #8414 - add systemd service timmer for collectd data
 and nrpe adjust threshold

---
 roles/db/files/nrpe/95-mariadb.cfg            |  2 +
 .../processes_systemd/process_monitor.service |  6 ++
 .../processes_systemd/process_monitor.timer   | 86 ++++++++++++++++
 roles/db/files/scripts/collect_processes.sh   | 97 +++++++++++++++++++
 roles/db/tasks/mariadb.yml                    | 27 +++++-
 5 files changed, 217 insertions(+), 1 deletion(-)
 create mode 100644 roles/db/files/processes_systemd/process_monitor.service
 create mode 100644 roles/db/files/processes_systemd/process_monitor.timer
 create mode 100644 roles/db/files/scripts/collect_processes.sh

diff --git a/roles/db/files/nrpe/95-mariadb.cfg b/roles/db/files/nrpe/95-mariadb.cfg
index 79e9e36..a918213 100644
--- a/roles/db/files/nrpe/95-mariadb.cfg
+++ b/roles/db/files/nrpe/95-mariadb.cfg
@@ -2,3 +2,5 @@ command[check_disk_mysqldata]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p
 command[check_disk_mysqlbin]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/mysqlbin
 command[check_disk_backup]=/usr/lib/nagios/plugins/check_disk -w 10% -c 5% -p /mnt/local-backup
 command[check_mysql_scheduler]=/etc/nagios/plugins/check_mysql_scheduler
+command[check_total_procs]=/usr/lib/nagios/plugins/check_procs -w 600 -c 700
+
diff --git a/roles/db/files/processes_systemd/process_monitor.service b/roles/db/files/processes_systemd/process_monitor.service
new file mode 100644
index 0000000..a280463
--- /dev/null
+++ b/roles/db/files/processes_systemd/process_monitor.service
@@ -0,0 +1,6 @@
+[Unit]
+Description=Collect system processes and stats
+
+[Service]
+Type=oneshot
+ExecStart=/root/scripts/collect_processes.sh
diff --git a/roles/db/files/processes_systemd/process_monitor.timer b/roles/db/files/processes_systemd/process_monitor.timer
new file mode 100644
index 0000000..24806c5
--- /dev/null
+++ b/roles/db/files/processes_systemd/process_monitor.timer
@@ -0,0 +1,86 @@
+[Unit]
+Description=Timer for collecting system processes and stats
+
+[Timer]
+OnCalendar=*-*-* 00:00:00 Europe/Madrid
+OnCalendar=*-*-* 00:05:00 Europe/Madrid
+OnCalendar=*-*-* 00:10:00 Europe/Madrid
+OnCalendar=*-*-* 00:15:00 Europe/Madrid
+OnCalendar=*-*-* 00:20:00 Europe/Madrid
+OnCalendar=*-*-* 00:25:00 Europe/Madrid
+OnCalendar=*-*-* 00:30:00 Europe/Madrid
+OnCalendar=*-*-* 00:35:00 Europe/Madrid
+OnCalendar=*-*-* 00:40:00 Europe/Madrid
+OnCalendar=*-*-* 00:45:00 Europe/Madrid
+OnCalendar=*-*-* 00:50:00 Europe/Madrid
+OnCalendar=*-*-* 00:55:00 Europe/Madrid
+OnCalendar=*-*-* 01:00:00 Europe/Madrid
+OnCalendar=*-*-* 01:05:00 Europe/Madrid
+OnCalendar=*-*-* 01:10:00 Europe/Madrid
+OnCalendar=*-*-* 01:15:00 Europe/Madrid
+OnCalendar=*-*-* 01:20:00 Europe/Madrid
+OnCalendar=*-*-* 01:25:00 Europe/Madrid
+OnCalendar=*-*-* 01:30:00 Europe/Madrid
+OnCalendar=*-*-* 01:35:00 Europe/Madrid
+OnCalendar=*-*-* 01:40:00 Europe/Madrid
+OnCalendar=*-*-* 01:45:00 Europe/Madrid
+OnCalendar=*-*-* 01:50:00 Europe/Madrid
+OnCalendar=*-*-* 01:55:00 Europe/Madrid
+OnCalendar=*-*-* 02:00:00 Europe/Madrid
+OnCalendar=*-*-* 02:05:00 Europe/Madrid
+OnCalendar=*-*-* 02:10:00 Europe/Madrid
+OnCalendar=*-*-* 02:15:00 Europe/Madrid
+OnCalendar=*-*-* 02:20:00 Europe/Madrid
+OnCalendar=*-*-* 02:25:00 Europe/Madrid
+OnCalendar=*-*-* 02:30:00 Europe/Madrid
+OnCalendar=*-*-* 02:35:00 Europe/Madrid
+OnCalendar=*-*-* 02:40:00 Europe/Madrid
+OnCalendar=*-*-* 02:45:00 Europe/Madrid
+OnCalendar=*-*-* 02:50:00 Europe/Madrid
+OnCalendar=*-*-* 02:55:00 Europe/Madrid
+OnCalendar=*-*-* 03:00:00 Europe/Madrid
+OnCalendar=*-*-* 03:05:00 Europe/Madrid
+OnCalendar=*-*-* 03:10:00 Europe/Madrid
+OnCalendar=*-*-* 03:15:00 Europe/Madrid
+OnCalendar=*-*-* 03:20:00 Europe/Madrid
+OnCalendar=*-*-* 03:25:00 Europe/Madrid
+OnCalendar=*-*-* 03:30:00 Europe/Madrid
+OnCalendar=*-*-* 03:35:00 Europe/Madrid
+OnCalendar=*-*-* 03:40:00 Europe/Madrid
+OnCalendar=*-*-* 03:45:00 Europe/Madrid
+OnCalendar=*-*-* 03:50:00 Europe/Madrid
+OnCalendar=*-*-* 03:55:00 Europe/Madrid
+OnCalendar=*-*-* 04:00:00 Europe/Madrid
+OnCalendar=*-*-* 04:05:00 Europe/Madrid
+OnCalendar=*-*-* 04:10:00 Europe/Madrid
+OnCalendar=*-*-* 04:15:00 Europe/Madrid
+OnCalendar=*-*-* 04:20:00 Europe/Madrid
+OnCalendar=*-*-* 04:25:00 Europe/Madrid
+OnCalendar=*-*-* 04:30:00 Europe/Madrid
+OnCalendar=*-*-* 04:35:00 Europe/Madrid
+OnCalendar=*-*-* 04:40:00 Europe/Madrid
+OnCalendar=*-*-* 04:45:00 Europe/Madrid
+OnCalendar=*-*-* 04:50:00 Europe/Madrid
+OnCalendar=*-*-* 04:55:00 Europe/Madrid
+OnCalendar=*-*-* 05:00:00 Europe/Madrid
+OnCalendar=*-*-* 05:05:00 Europe/Madrid
+OnCalendar=*-*-* 05:10:00 Europe/Madrid
+OnCalendar=*-*-* 05:15:00 Europe/Madrid
+OnCalendar=*-*-* 05:20:00 Europe/Madrid
+OnCalendar=*-*-* 05:25:00 Europe/Madrid
+OnCalendar=*-*-* 05:30:00 Europe/Madrid
+OnCalendar=*-*-* 05:35:00 Europe/Madrid
+OnCalendar=*-*-* 05:40:00 Europe/Madrid
+OnCalendar=*-*-* 05:45:00 Europe/Madrid
+OnCalendar=*-*-* 05:50:00 Europe/Madrid
+OnCalendar=*-*-* 05:55:00 Europe/Madrid
+OnCalendar=*-*-* 06:00:00 Europe/Madrid
+OnCalendar=*-*-* 06:05:00 Europe/Madrid
+OnCalendar=*-*-* 06:10:00 Europe/Madrid
+OnCalendar=*-*-* 06:15:00 Europe/Madrid
+OnCalendar=*-*-* 06:20:00 Europe/Madrid
+OnCalendar=*-*-* 06:25:00 Europe/Madrid
+OnCalendar=*-*-* 06:30:00 Europe/Madrid
+
+[Install]
+WantedBy=timers.target
diff --git a/roles/db/files/scripts/collect_processes.sh b/roles/db/files/scripts/collect_processes.sh
new file mode 100644
index 0000000..19e72e0
--- /dev/null
+++ b/roles/db/files/scripts/collect_processes.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+# Script to collect system and process data for monitoring.
+#
+# Author: Xavi Lleó
+# Copyright (c) 2025 Verdnatura S.L. All rights reserved.
+# Version: 1.0.0
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# Description:
+# This script collects system statistics and process information
+# and logs it to a file in /var/log/processes. It includes:
+#   - CPU and memory usage with `top`
+#   - Process queue statistics with `sar`
+#   - Nagios process count check
+#   - MySQL process list
+#
+# Execution:
+# The script is designed to be executed by a systemd timer every 5 minutes
+# between 02:30 and 06:30 AM daily. The timer is configured to run the
+# script located at /usr/local/bin/monitor_processes.sh.
+#
+# Log Location:
+# The log file will be stored in /var/log/processes with a timestamp in the filename.
+#
+# Example Timer Configuration:
+# The following systemd timer will execute the script every 5 minutes
+# between 02:30 and 06:30 AM:
+# 
+# [Timer]
+# OnCalendar=*-*-* 02:30:00
+# Persistent=true
+# OnBootSec=10min
+# OnUnitActiveSec=5min
+# Until=*-*-* 06:30:00
+#
+# [Service]
+# Type=oneshot
+# ExecStart=/usr/local/bin/monitor_processes.sh
+#
+# Ensure that the systemd timer and service are enabled to run as expected.
+
+LOG_DIR="/var/log/processes"
+mkdir -p "$LOG_DIR"
+
+timestamp() {
+    date +'%Y_%m_%d-%H:%M:%S'
+}
+
+LOG_FILE="$LOG_DIR/processes_$(timestamp).log"
+
+log() {
+    echo "$1" >> "$LOG_FILE"
+}
+
+check_command() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+log "Process and system data collection - $(timestamp)"
+log "-------------------------------------------"
+
+commands=(
+    "/usr/bin/top -b -n 1"
+    "/usr/bin/sar -q 1 10"
+    "/usr/lib/nagios/plugins/check_procs"
+    "/usr/bin/mysql -Bse 'show processlist;'"
+)
+
+for cmd in "${commands[@]}"; do
+    cmd_name=$(echo $cmd | cut -d' ' -f1)
+    
+    if check_command "$cmd_name"; then
+        log "### Executing $cmd_name ###"
+        eval $cmd >> "$LOG_FILE" 2>&1
+        if [ $? -ne 0 ]; then
+            log "Error executing '$cmd_name'"
+        fi
+    else
+        log "Skipping execution of '$cmd_name' due to missing command"
+    fi
+done
+
+log "-------------------------------------------"
+log "Log finished - $(timestamp)"
+
diff --git a/roles/db/tasks/mariadb.yml b/roles/db/tasks/mariadb.yml
index a3980ce..21d4a57 100644
--- a/roles/db/tasks/mariadb.yml
+++ b/roles/db/tasks/mariadb.yml
@@ -135,7 +135,7 @@
 
 - when: mysql_dir.stat.exists
   block:
-
+  
   - name: Sync MySQL data directory
     synchronize:
       src: /var/lib/mysql/
@@ -153,3 +153,28 @@
 - name: Mount all filesystems from /etc/fstab
   command: mount -a
   when: fstab.changed
+
+- name: Copy systemd service and timer files
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: root
+    group: root
+    mode: '0644'
+  loop:
+    - { src: 'processes_systemd/process_monitor.service', dest: '/lib/systemd/system/process_monitor.service' }
+    - { src: 'processes_systemd/process_monitor.timer', dest: '/lib/systemd/system/process_monitor.timer' }
+  notify: reload-systemd
+
+- name: Copy collect_processes script to root path scripts
+  copy:
+    src: scripts/collect_processes.sh
+    dest: /root/scripts/collect_processes.sh
+    owner: root
+    group: root
+    mode: '0755'
+- name: Enable process_monitor timer without starting it
+  systemd:
+    name: process_monitor.timer
+    enabled: true
+    state: stopped