WIP: feat: refs #7144 diagnoseProductionFailure #2499

Draft
guillermo wants to merge 6 commits from 7144-diagnoseProductionFailure into dev
4 changed files with 73 additions and 21 deletions
Showing only changes of commit cb32be3814 - Show all commits

View File

@ -1,5 +1,7 @@
DELIMITER $$ DELIMITER $$
CREATE OR REPLACE DEFINER=`root`@`localhost` PROCEDURE `vn`.`diagnoseProductionFailure_dbFreeze`() CREATE OR REPLACE DEFINER=`root`@`localhost` FUNCTION `vn`.`diagnoseProductionFailure_dbFreeze`()
RETURNS int(11)
DETERMINISTIC
BEGIN BEGIN
/** /**
* Comprueba si la db está congelada. * Comprueba si la db está congelada.
@ -8,12 +10,11 @@ BEGIN
*/ */
DECLARE vHasError BOOL; DECLARE vHasError BOOL;
SELECT IF(COUNT(ID) > 5, TRUE, FALSE) INTO vHasError # Variable en tabla SELECT IF(COUNT(ID) > 5, TRUE, FALSE) INTO vHasError
FROM information_schema.PROCESSLIST FROM information_schema.PROCESSLIST
WHERE COMMAND = 'Query' WHERE COMMAND = 'Query'
AND `time` > 60; # Variable en tabla AND `time` > 60; # Variable en tabla
INSERT INTO tmp.diagnoseProductionFailure RETURN vHasError;
SELECT 'Base de datos congelada', vHasError;
END$$ END$$
DELIMITER ; DELIMITER ;

View File

@ -1,5 +1,7 @@
DELIMITER $$ DELIMITER $$
CREATE OR REPLACE DEFINER=`root`@`localhost` PROCEDURE `vn`.`diagnoseProductionFailure_dbGetLock`() CREATE OR REPLACE DEFINER=`root`@`localhost` FUNCTION `vn`.`diagnoseProductionFailure_dbGetLock`()
RETURNS int(11)
DETERMINISTIC
BEGIN BEGIN
/** /**
* Comprueba si la db está congelada. * Comprueba si la db está congelada.
@ -7,14 +9,13 @@ BEGIN
* @table tmp.diagnoseProductionFailure(message, hasError) * @table tmp.diagnoseProductionFailure(message, hasError)
*/ */
DECLARE vHasError BOOL; DECLARE vHasError BOOL;
/* Bug: En local la tabla METADATA_LOCK_INFO no existe...
SELECT COUNT(*) INTO vHasError SELECT COUNT(*) INTO vHasError
FROM information_schema.METADATA_LOCK_INFO mli FROM information_schema.METADATA_LOCK_INFO mli
JOIN information_schema.PROCESSLIST pl ON pl.ID = mli.THREAD_ID JOIN information_schema.PROCESSLIST pl ON pl.ID = mli.THREAD_ID
WHERE mli.LOCK_MODE = 'MDL_SHARED_NO_WRITE' WHERE mli.LOCK_MODE = 'MDL_SHARED_NO_WRITE'
AND pl.`time` > 30; # Variable en tabla AND pl.`time` > 30; # Variable en tabla
*/
INSERT INTO tmp.diagnoseProductionFailure RETURN true; -- vHasError;
SELECT 'Get lock bloqueando para asignar colecciones', vHasError;
END$$ END$$
DELIMITER ; DELIMITER ;

View File

@ -6,17 +6,61 @@ BEGIN
/** /**
* Comprueba si hay fallos en producción. * Comprueba si hay fallos en producción.
*/ */
DECLARE vDynSql VARCHAR(125);
DECLARE vSchema VARCHAR(64);
DECLARE vFunction VARCHAR(64);
DECLARE vParams VARCHAR(255);
DECLARE vMessage TEXT;
DECLARE vResolution TEXT;
DECLARE vDone BOOL;
DECLARE cFunctions CURSOR FOR
SELECT `schema`,
`function`,
params,
`message`,
resolution
FROM diagnoseProductionFailureQueue
ORDER BY `order`;
DECLARE CONTINUE HANDLER FOR NOT FOUND SET vDone = TRUE;
CREATE OR REPLACE TEMPORARY TABLE tmp.diagnoseProductionFailure ( CREATE OR REPLACE TEMPORARY TABLE tmp.diagnoseProductionFailure (
`message` TEXT NOT NULL, `message` TEXT NOT NULL,
hasError TINYINT(1) NOT NULL hasError TINYINT(1) NOT NULL,
`resolution` TEXT
); );
CALL vn.diagnoseProductionFailure_dbFreeze(); OPEN cFunctions;
l: LOOP
SET vDone = FALSE;
FETCH cFunctions INTO
vSchema,
vFunction,
vParams,
vMessage,
vResolution;
CALL vn.diagnoseProductionFailure_dbGetLock(); IF vDone THEN
LEAVE l;
END IF;
SET vDynSql = CONCAT('SELECT ', vSchema, '.', vFunction, '() INTO @vHasError');
PREPARE stmt FROM vDynSql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
INSERT INTO tmp.diagnoseProductionFailure
SELECT vMessage,
@vHasError,
IF (@vHasError,
vResolution,
NULL
);
END LOOP;
CLOSE cFunctions;
SELECT * FROM tmp.diagnoseProductionFailure; SELECT * FROM tmp.diagnoseProductionFailure;
DROP TEMPORARY TABLE tmp.diagnoseProductionFailure; DROP TEMPORARY TABLE tmp.diagnoseProductionFailure;
END$$ END$$
DELIMITER ; DELIMITER ;

View File

@ -1,14 +1,15 @@
CREATE TABLE `vn`.`diagnoseProductionFailureQueue` ( CREATE TABLE `vn`.`diagnoseProductionFailureQueue` (
`schema` varchar(64) NOT NULL, `schema` varchar(64) NOT NULL,
`procedure` varchar(64) NOT NULL, `function` varchar(64) NOT NULL,
`params` varchar(255) DEFAULT NULL, `params` varchar(255) DEFAULT NULL,
`order` int(11) DEFAULT NULL, `order` int(11) DEFAULT NULL,
`message` text NOT NULL, `message` text NOT NULL,
`resolution` text DEFAULT NULL, `resolution` text DEFAULT NULL,
PRIMARY KEY (`schema`,`procedure`) PRIMARY KEY (`schema`,`function`),
UNIQUE KEY `diagnoseProductionFailureQueue_unique` (`order`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
CREATE TABLE `diagnoseProductionFailureConfig` ( CREATE TABLE `vn`.`diagnoseProductionFailureConfig` (
`id` int(11) NOT NULL, `id` int(11) NOT NULL,
`dbFreezeMaxThreads` int(11) NOT NULL, `dbFreezeMaxThreads` int(11) NOT NULL,
`dbFreezeMaxTime` int(11) NOT NULL, `dbFreezeMaxTime` int(11) NOT NULL,
@ -19,3 +20,8 @@ CREATE TABLE `diagnoseProductionFailureConfig` (
INSERT IGNORE INTO vn.diagnoseProductionFailureConfig (id,dbFreezeMaxThreads,dbFreezeMaxTime,dbGetLockMaxTime) INSERT IGNORE INTO vn.diagnoseProductionFailureConfig (id,dbFreezeMaxThreads,dbFreezeMaxTime,dbGetLockMaxTime)
VALUES (1, 5, 60, 30); VALUES (1, 5, 60, 30);
INSERT IGNORE INTO vn.diagnoseProductionFailureQueue (`schema`,`function`,`order`,message,resolution) VALUES
('vn','diagnoseProductionFailure_dbFreeze',1,'Base de datos congelada','Esto es una prueba 1'),
('vn','diagnoseProductionFailure_dbGetLock',2,'Get lock bloqueando para asignar colecciones','Esto es una prueba 2');