feat: refs #7562 deleteDeprecatedObjects #6

Merged
guillermo merged 7 commits from 7562-deleteDeprecatedObjects into master 2024-09-12 08:06:15 +00:00
9 changed files with 132 additions and 15 deletions

View File

@ -6,6 +6,11 @@ mockFunctions:
- mockTime - mockTime
- mockUtcTime - mockUtcTime
sumViews: true sumViews: true
defaultDefiner: root@localhost
deprecMarkRegex: __$
deprecCommentRegex: ^@deprecated [0-9]{4}-[0-9]{2}-[0-9]{2}
deprecDateRegex: '[0-9]{4}-[0-9]{2}-[0-9]{2}'
deprecRetentionPeriod: 60 # Days
privileges: privileges:
userTable: global_priv userTable: global_priv
userWhere: >- userWhere: >-

View File

@ -10,7 +10,7 @@ class Clean extends Command {
static usage = { static usage = {
description: 'Cleans old applied versions', description: 'Cleans old applied versions',
params: { params: {
purge: 'Wether to remove non-existent scripts from DB log' purge: 'Whether to remove non-existent scripts from DB log'
} }
}; };

View File

@ -43,7 +43,7 @@ class Create extends Command {
const params = { const params = {
schema, schema,
name, name,
definer: 'root@localhost' definer: opts.defaultDefiner
}; };
switch (opts.type) { switch (opts.type) {

View File

@ -9,7 +9,7 @@ class Dump extends Command {
description: 'Dumps structure and fixtures from remote', description: 'Dumps structure and fixtures from remote',
params: { params: {
lock: 'Whether to lock tables on dump', lock: 'Whether to lock tables on dump',
triggers: 'Wether to include triggers into dump' triggers: 'Whether to include triggers into dump'
}, },
operand: 'remote' operand: 'remote'
}; };

View File

@ -15,9 +15,9 @@ class Push extends Command {
description: 'Apply changes into database', description: 'Apply changes into database',
params: { params: {
force: 'Answer yes to all questions', force: 'Answer yes to all questions',
commit: 'Wether to save the commit SHA into database', commit: 'Whether to save the commit SHA into database',
sums: 'Save SHA sums of pushed objects', sums: 'Save SHA sums of pushed objects',
triggers: 'Wether to exclude triggers, used to generate local DB' triggers: 'Whether to exclude triggers, used to generate local DB'
}, },
operand: 'remote' operand: 'remote'
}; };

View File

@ -1,6 +1,7 @@
const Myt = require('./myt'); const Myt = require('./myt');
const Command = require('./lib/command'); const Command = require('./lib/command');
const fs = require('fs-extra'); const fs = require('fs-extra');
const SqlString = require('sqlstring');
/** /**
* Creates a new version. * Creates a new version.
@ -9,18 +10,23 @@ class Version extends Command {
static usage = { static usage = {
description: 'Creates a new version', description: 'Creates a new version',
params: { params: {
name: 'Name for the new version' name: 'Name for the new version',
deprecate: 'Whether to generate sql to delete deprecated objects'
}, },
operand: 'name' operand: 'name'
}; };
static opts = { static opts = {
alias: { alias: {
name: 'n' name: 'n',
guillermo marked this conversation as resolved
Review

Los alias solo tienen que tener una letra y hacer referencia a una palabra inglesa, ej: -p

Los alias solo tienen que tener una letra y hacer referencia a una palabra inglesa, ej: `-p`
deprecate: 'p'
}, },
string: [ string: [
'name' 'name'
], ],
boolean: [
'deprecate'
],
default: { default: {
remote: 'production' remote: 'production'
} }
@ -36,7 +42,8 @@ class Version extends Command {
}, },
versionCreated: function(versionName) { versionCreated: function(versionName) {
console.log(`New version created: ${versionName}`); console.log(`New version created: ${versionName}`);
} },
deprecate: 'Generating SQL for deprecated objects deletion.'
guillermo marked this conversation as resolved Outdated
Outdated
Review

Generating SQL for deprecated objects deletion

_Generating SQL for deprecated objects deletion_
}; };
async run(myt, opts) { async run(myt, opts) {
@ -121,10 +128,16 @@ class Version extends Command {
[opts.code, newVersion] [opts.code, newVersion]
); );
await fs.mkdir(newVersionDir); await fs.mkdir(newVersionDir);
if (opts.deprecate) {
this.emit('deprecate');
await deprecate(conn, opts, newVersionDir);
} else
await fs.writeFile( await fs.writeFile(
guillermo marked this conversation as resolved
Review

else y } deben estar en la misma linea: } else

`else` y `}` deben estar en la misma linea: `} else`
`${newVersionDir}/00-firstScript.sql`, `${newVersionDir}/00-firstScript.sql`,
'-- Place your SQL code here\n' '-- Place your SQL code here\n'
); );
this.emit('versionCreated', versionFolder); this.emit('versionCreated', versionFolder);
await conn.query('COMMIT'); await conn.query('COMMIT');
@ -137,6 +150,105 @@ class Version extends Command {
} }
} }
async function deprecate(conn, opts, newVersionDir) {
const now = new Date();
const minDeprecDate = new Date(now.getTime() - opts.deprecRetentionPeriod * 24 * 60 * 60 * 1000);
const deprecMarkRegex = opts.deprecMarkRegex;
guillermo marked this conversation as resolved Outdated
Outdated
Review

Usar el fichero de configuración de Myt para estos parámetros

Usar el fichero de configuración de Myt para estos parámetros
const deprecCommentRegex = opts.deprecCommentRegex;
const deprecDateRegex = opts.deprecDateRegex;
const filePath = `${newVersionDir}/00-deprecate.sql`;
// Generate the drops of the primary keys
const [primaryKeys] = await conn.query(`
SELECT c.TABLE_SCHEMA 'schema', c.TABLE_NAME 'table'
FROM information_schema.COLUMNS c
LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA
AND v.TABLE_NAME = c.TABLE_NAME
JOIN information_schema.STATISTICS s ON s.TABLE_SCHEMA = c.TABLE_SCHEMA
AND s.TABLE_NAME = c.TABLE_NAME
AND s.COLUMN_NAME = c.COLUMN_NAME
WHERE c.COLUMN_NAME REGEXP ? COLLATE utf8mb4_unicode_ci
AND c.COLUMN_COMMENT REGEXP ? COLLATE utf8mb4_unicode_ci
AND REGEXP_SUBSTR(c.COLUMN_COMMENT, ? COLLATE utf8mb4_unicode_ci) < ?
AND v.TABLE_NAME IS NULL
AND s.INDEX_NAME = 'PRIMARY'
`, [deprecMarkRegex, deprecCommentRegex, deprecDateRegex, minDeprecDate]);
primaryKeys.map(async row => {
await fs.appendFile(
filePath,
'ALTER TABLE ' + SqlString.escapeId(row.schema, true) + '.' +
SqlString.escapeId(row.table, true) + ' DROP PRIMARY KEY;\n'
);
});
// Generate the drops of the foreign keys
const [foreignKeys] = await conn.query(`
SELECT c.TABLE_SCHEMA 'schema', c.TABLE_NAME 'table', kcu.CONSTRAINT_NAME 'constraint'
FROM information_schema.COLUMNS c
LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA
AND v.TABLE_NAME = c.TABLE_NAME
JOIN information_schema.KEY_COLUMN_USAGE kcu ON kcu.TABLE_SCHEMA = c.TABLE_SCHEMA
AND kcu.TABLE_NAME = c.TABLE_NAME
AND kcu.COLUMN_NAME = c.COLUMN_NAME
WHERE c.COLUMN_NAME REGEXP ? COLLATE utf8mb4_unicode_ci
AND c.COLUMN_COMMENT REGEXP ? COLLATE utf8mb4_unicode_ci
AND REGEXP_SUBSTR(c.COLUMN_COMMENT, ? COLLATE utf8mb4_unicode_ci) < ?
AND v.TABLE_NAME IS NULL
AND kcu.REFERENCED_COLUMN_NAME IS NOT NULL
`, [deprecMarkRegex, deprecCommentRegex, deprecDateRegex, minDeprecDate]);
foreignKeys.map(async row => {
await fs.appendFile(
filePath,
'ALTER TABLE ' + SqlString.escapeId(row.schema, true) + '.' +
SqlString.escapeId(row.table, true) + ' DROP FOREIGN KEY ' +
SqlString.escapeId(row.constraint, true) + ';\n'
);
});
guillermo marked this conversation as resolved Outdated
Outdated
Review
  • Generar el SQL con Javascript en lugar de usar CONCATs SQL
  • Escapar identificadores (nombres de esquemas, tablas, columnas...)
  • Hacer comprobación estricta del patron @deprecated YYYY-MM-DD
- Generar el SQL con Javascript en lugar de usar CONCATs SQL - Escapar identificadores (nombres de esquemas, tablas, columnas...) - Hacer comprobación **estricta** del patron `@deprecated YYYY-MM-DD`
// Generate the drops of the columns
const [columns] = await conn.query(`
SELECT c.TABLE_SCHEMA 'schema', c.TABLE_NAME 'table', c.COLUMN_NAME 'column'
FROM information_schema.COLUMNS c
LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA
AND v.TABLE_NAME = c.TABLE_NAME
LEFT JOIN information_schema.KEY_COLUMN_USAGE kcu ON kcu.TABLE_SCHEMA = c.TABLE_SCHEMA
AND kcu.TABLE_NAME = c.TABLE_NAME
AND kcu.COLUMN_NAME = c.COLUMN_NAME
WHERE c.COLUMN_NAME REGEXP ? COLLATE utf8mb4_unicode_ci
AND c.COLUMN_COMMENT REGEXP ? COLLATE utf8mb4_unicode_ci
AND REGEXP_SUBSTR(c.COLUMN_COMMENT, ? COLLATE utf8mb4_unicode_ci) <?
AND v.TABLE_NAME IS NULL
`, [deprecMarkRegex, deprecCommentRegex, deprecDateRegex, minDeprecDate]);
columns.map(async row => {
await fs.appendFile(
filePath,
'ALTER TABLE ' + SqlString.escapeId(row.schema, true) + '.' +
SqlString.escapeId(row.table, true) + ' DROP COLUMN ' +
SqlString.escapeId(row.column, true) + ';\n'
);
});
// Generate the drops of the tables
const [tables] = await conn.query(`
SELECT TABLE_SCHEMA 'schema', TABLE_NAME 'table'
FROM information_schema.TABLES
WHERE TABLE_NAME REGEXP ? COLLATE utf8mb4_unicode_ci
AND TABLE_COMMENT REGEXP ? COLLATE utf8mb4_unicode_ci
AND REGEXP_SUBSTR(TABLE_COMMENT, ? COLLATE utf8mb4_unicode_ci) < ?
`, [deprecMarkRegex, deprecCommentRegex, deprecDateRegex, minDeprecDate]);
tables.map(async row => {
await fs.appendFile(
filePath,
'DROP TABLE ' + SqlString.escapeId(row.schema, true) + '.' +
SqlString.escapeId(row.table, true) + ';\n'
);
});
}
function randomName() { function randomName() {
const color = random(colors); const color = random(colors);
let plant = random(plants); let plant = random(plants);

2
myt.js
View File

@ -19,7 +19,7 @@ class Myt {
params: { params: {
remote: 'Name of remote to use', remote: 'Name of remote to use',
workspace: 'The base directory of the project', workspace: 'The base directory of the project',
debug: 'Wether to enable debug mode', debug: 'Whether to enable debug mode',
version: 'Display the version number and exit', version: 'Display the version number and exit',
help: 'Display this help message' help: 'Display this help message'
} }

View File

@ -1,6 +1,6 @@
{ {
"name": "@verdnatura/myt", "name": "@verdnatura/myt",
"version": "1.6.9", "version": "1.6.10",
"author": "Verdnatura Levante SL", "author": "Verdnatura Levante SL",
"description": "MySQL version control", "description": "MySQL version control",
"license": "GPL-3.0", "license": "GPL-3.0",