From 912719cc08de732f9e52c4132606282073a6c622 Mon Sep 17 00:00:00 2001 From: guillermo Date: Wed, 4 Sep 2024 14:50:27 +0200 Subject: [PATCH 1/7] feat: refs #7562 Added deprecate funcion --- myt-clean.js | 2 +- myt-dump.js | 2 +- myt-push.js | 4 +-- myt-version.js | 76 +++++++++++++++++++++++++++++++++++++++++++++++--- myt.js | 2 +- 5 files changed, 77 insertions(+), 9 deletions(-) diff --git a/myt-clean.js b/myt-clean.js index db37150..a83c2ec 100644 --- a/myt-clean.js +++ b/myt-clean.js @@ -10,7 +10,7 @@ class Clean extends Command { static usage = { description: 'Cleans old applied versions', params: { - purge: 'Wether to remove non-existent scripts from DB log' + purge: 'Whether to remove non-existent scripts from DB log' } }; diff --git a/myt-dump.js b/myt-dump.js index 19b13f0..4c447c1 100644 --- a/myt-dump.js +++ b/myt-dump.js @@ -9,7 +9,7 @@ class Dump extends Command { description: 'Dumps structure and fixtures from remote', params: { lock: 'Whether to lock tables on dump', - triggers: 'Wether to include triggers into dump' + triggers: 'Whether to include triggers into dump' }, operand: 'remote' }; diff --git a/myt-push.js b/myt-push.js index 041baf7..f87b9d4 100644 --- a/myt-push.js +++ b/myt-push.js @@ -15,9 +15,9 @@ class Push extends Command { description: 'Apply changes into database', params: { force: 'Answer yes to all questions', - commit: 'Wether to save the commit SHA into database', + commit: 'Whether to save the commit SHA into database', sums: 'Save SHA sums of pushed objects', - triggers: 'Wether to exclude triggers, used to generate local DB' + triggers: 'Whether to exclude triggers, used to generate local DB' }, operand: 'remote' }; diff --git a/myt-version.js b/myt-version.js index 219bcaf..8e2677f 100644 --- a/myt-version.js +++ b/myt-version.js @@ -9,18 +9,23 @@ class Version extends Command { static usage = { description: 'Creates a new version', params: { - name: 'Name for the new version' + name: 'Name for the new version', + deprecate: 'Whether to generate sql to delete deprecated objects' }, operand: 'name' }; static opts = { alias: { - name: 'n' + name: 'n', + deprecate: 'kk' }, string: [ 'name' ], + boolean: [ + 'deprecate' + ], default: { remote: 'production' } @@ -36,7 +41,8 @@ class Version extends Command { }, versionCreated: function(versionName) { console.log(`New version created: ${versionName}`); - } + }, + deprecate: 'Generating sql.' }; async run(myt, opts) { @@ -123,7 +129,9 @@ class Version extends Command { await fs.mkdir(newVersionDir); await fs.writeFile( `${newVersionDir}/00-firstScript.sql`, - '-- Place your SQL code here\n' + opts.deprecate + ? this.emit('deprecate') && await deprecate() + : '-- Place your SQL code here\n' ); this.emit('versionCreated', versionFolder); @@ -137,6 +145,66 @@ class Version extends Command { } } +async function deprecate() { + const [[config]] = await conn.query(` + SELECT dateRegex, + deprecatedMarkRegex, + VN_CURDATE() - INTERVAL daysKeepDeprecatedObjects DAY dated + FROM config + `); + + const sql = await conn.query(` + WITH variables AS ( + SELECT ? markRegex, ? dateRegex, ? dated + ) + SELECT CONCAT('ALTER TABLE ', c.TABLE_SCHEMA, '.', c.TABLE_NAME, ' DROP PRIMARY KEY;') sql + FROM information_schema.COLUMNS c + LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA + AND v.TABLE_NAME = c.TABLE_NAME + JOIN information_schema.STATISTICS s ON s.TABLE_SCHEMA = c.TABLE_SCHEMA + AND s.TABLE_NAME = c.TABLE_NAME + AND s.COLUMN_NAME = c.COLUMN_NAME + JOIN variables var + WHERE c.COLUMN_NAME REGEXP var.markRegex COLLATE utf8mb4_unicode_ci + AND REGEXP_SUBSTR(c.COLUMN_COMMENT, var.dateRegex COLLATE utf8mb4_unicode_ci) < var.dated + AND v.TABLE_NAME IS NULL + AND s.INDEX_NAME = 'PRIMARY' + UNION + SELECT CONCAT('ALTER TABLE ', c.TABLE_SCHEMA, '.', c.TABLE_NAME, ' DROP FOREIGN KEY ', kcu.CONSTRAINT_NAME, ';') + FROM information_schema.COLUMNS c + LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA + AND v.TABLE_NAME = c.TABLE_NAME + JOIN information_schema.KEY_COLUMN_USAGE kcu ON kcu.TABLE_SCHEMA = c.TABLE_SCHEMA + AND kcu.TABLE_NAME = c.TABLE_NAME + AND kcu.COLUMN_NAME = c.COLUMN_NAME + JOIN variables var + WHERE c.COLUMN_NAME REGEXP var.markRegex COLLATE utf8mb4_unicode_ci + AND REGEXP_SUBSTR(c.COLUMN_COMMENT, var.dateRegex COLLATE utf8mb4_unicode_ci) < var.dated + AND v.TABLE_NAME IS NULL + AND kcu.REFERENCED_COLUMN_NAME IS NOT NULL + UNION + SELECT CONCAT('ALTER TABLE ', c.TABLE_SCHEMA, '.', c.TABLE_NAME, ' DROP COLUMN ', c.COLUMN_NAME, ';') + FROM information_schema.COLUMNS c + LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA + AND v.TABLE_NAME = c.TABLE_NAME + LEFT JOIN information_schema.KEY_COLUMN_USAGE kcu ON kcu.TABLE_SCHEMA = c.TABLE_SCHEMA + AND kcu.TABLE_NAME = c.TABLE_NAME + AND kcu.COLUMN_NAME = c.COLUMN_NAME + JOIN variables var + WHERE c.COLUMN_NAME REGEXP var.markRegex COLLATE utf8mb4_unicode_ci + AND REGEXP_SUBSTR(c.COLUMN_COMMENT, var.dateRegex COLLATE utf8mb4_unicode_ci) < var.dated + AND v.TABLE_NAME IS NULL + UNION + SELECT CONCAT('DROP TABLE ', t.TABLE_SCHEMA, '.', t.TABLE_NAME, ';') + FROM information_schema.TABLES t + JOIN variables var + WHERE t.TABLE_NAME REGEXP var.markRegex COLLATE utf8mb4_unicode_ci + AND REGEXP_SUBSTR(t.TABLE_COMMENT, var.dateRegex COLLATE utf8mb4_unicode_ci) < var.dated + `, [config.deprecatedMarkRegex, config.dateRegex, config.dated]); + + return sql.map(row => row.sql).join('\n'); +} + function randomName() { const color = random(colors); let plant = random(plants); diff --git a/myt.js b/myt.js index 128c3b8..5b4007c 100755 --- a/myt.js +++ b/myt.js @@ -19,7 +19,7 @@ class Myt { params: { remote: 'Name of remote to use', workspace: 'The base directory of the project', - debug: 'Wether to enable debug mode', + debug: 'Whether to enable debug mode', version: 'Display the version number and exit', help: 'Display this help message' } From a27afbaa53b04a240c3273a4eb051aedb99b4f84 Mon Sep 17 00:00:00 2001 From: guillermo Date: Wed, 4 Sep 2024 15:04:45 +0200 Subject: [PATCH 2/7] feat: refs #7562 Fixes --- myt-version.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/myt-version.js b/myt-version.js index 8e2677f..45b7765 100644 --- a/myt-version.js +++ b/myt-version.js @@ -127,12 +127,19 @@ class Version extends Command { [opts.code, newVersion] ); await fs.mkdir(newVersionDir); + + let content; + if (opts.deprecate) { + this.emit('deprecate'); + content = await deprecate(); + } else + content = '-- Place your SQL code here\n' + await fs.writeFile( `${newVersionDir}/00-firstScript.sql`, - opts.deprecate - ? this.emit('deprecate') && await deprecate() - : '-- Place your SQL code here\n' + content ); + this.emit('versionCreated', versionFolder); await conn.query('COMMIT'); From a38cda0ba3237b2293a3aa0e203a7ca6449a0e5a Mon Sep 17 00:00:00 2001 From: guillermo Date: Thu, 5 Sep 2024 08:18:18 +0200 Subject: [PATCH 3/7] feat: refs #7562 Fixes --- myt-version.js | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/myt-version.js b/myt-version.js index 45b7765..4fc74ce 100644 --- a/myt-version.js +++ b/myt-version.js @@ -42,7 +42,7 @@ class Version extends Command { versionCreated: function(versionName) { console.log(`New version created: ${versionName}`); }, - deprecate: 'Generating sql.' + deprecate: 'Generating SQL for deprecate.' }; async run(myt, opts) { @@ -128,16 +128,11 @@ class Version extends Command { ); await fs.mkdir(newVersionDir); - let content; - if (opts.deprecate) { - this.emit('deprecate'); - content = await deprecate(); - } else - content = '-- Place your SQL code here\n' - await fs.writeFile( `${newVersionDir}/00-firstScript.sql`, - content + opts.deprecate + ? (this.emit('deprecate'), await deprecate(conn)) + : '-- Place your SQL code here\n' ); this.emit('versionCreated', versionFolder); @@ -152,7 +147,7 @@ class Version extends Command { } } -async function deprecate() { +async function deprecate(conn) { const [[config]] = await conn.query(` SELECT dateRegex, deprecatedMarkRegex, @@ -160,11 +155,14 @@ async function deprecate() { FROM config `); - const sql = await conn.query(` + if (!config || Object.values(config).some(value => value == null || value === '')) + throw new Error('missing configuration variables'); + + const [sql] = await conn.query(` WITH variables AS ( SELECT ? markRegex, ? dateRegex, ? dated ) - SELECT CONCAT('ALTER TABLE ', c.TABLE_SCHEMA, '.', c.TABLE_NAME, ' DROP PRIMARY KEY;') sql + SELECT CONCAT('ALTER TABLE ', c.TABLE_SCHEMA, '.', c.TABLE_NAME, ' DROP PRIMARY KEY;') "sql" FROM information_schema.COLUMNS c LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA AND v.TABLE_NAME = c.TABLE_NAME From 5ac41532d99a49d2934815488a44950feefb5a4c Mon Sep 17 00:00:00 2001 From: guillermo Date: Fri, 6 Sep 2024 09:46:24 +0200 Subject: [PATCH 4/7] feat: refs #7562 No sql concat and more --- .gitignore | 2 + assets/myt.default.yml | 5 ++ myt-create.js | 2 +- myt-version.js | 126 +++++++++++++++++++++++++++-------------- 4 files changed, 91 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index 9daa824..4dab68d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .DS_Store node_modules +db +myt.config.yml \ No newline at end of file diff --git a/assets/myt.default.yml b/assets/myt.default.yml index ca5ac48..008abb8 100755 --- a/assets/myt.default.yml +++ b/assets/myt.default.yml @@ -6,6 +6,11 @@ mockFunctions: - mockTime - mockUtcTime sumViews: true +defaultDefiner: root@localhost +deprecMarkRegex: __$ +deprecCommentRegex: ^@deprecated [0-9]{4}-[0-9]{2}-[0-9]{2} +deprecDateRegex: '[0-9]{4}-[0-9]{2}-[0-9]{2}' +deprecRetentionPeriod: 60 # Days privileges: userTable: global_priv userWhere: >- diff --git a/myt-create.js b/myt-create.js index 1754d9a..84f282b 100755 --- a/myt-create.js +++ b/myt-create.js @@ -43,7 +43,7 @@ class Create extends Command { const params = { schema, name, - definer: 'root@localhost' + definer: opts.defaultDefiner }; switch (opts.type) { diff --git a/myt-version.js b/myt-version.js index 4fc74ce..9dba08f 100644 --- a/myt-version.js +++ b/myt-version.js @@ -1,6 +1,7 @@ const Myt = require('./myt'); const Command = require('./lib/command'); const fs = require('fs-extra'); +const SqlString = require('sqlstring'); /** * Creates a new version. @@ -18,7 +19,7 @@ class Version extends Command { static opts = { alias: { name: 'n', - deprecate: 'kk' + deprecate: 'p' }, string: [ 'name' @@ -128,12 +129,15 @@ class Version extends Command { ); await fs.mkdir(newVersionDir); - await fs.writeFile( - `${newVersionDir}/00-firstScript.sql`, - opts.deprecate - ? (this.emit('deprecate'), await deprecate(conn)) - : '-- Place your SQL code here\n' - ); + if (opts.deprecate) { + this.emit('deprecate'); + await deprecate(conn, opts, newVersionDir); + } + else + await fs.writeFile( + `${newVersionDir}/00-firstScript.sql`, + '-- Place your SQL code here\n' + ); this.emit('versionCreated', versionFolder); @@ -147,67 +151,103 @@ class Version extends Command { } } -async function deprecate(conn) { - const [[config]] = await conn.query(` - SELECT dateRegex, - deprecatedMarkRegex, - VN_CURDATE() - INTERVAL daysKeepDeprecatedObjects DAY dated - FROM config - `); +async function deprecate(conn, opts, newVersionDir) { + const now = new Date(); + const minDeprecDate = new Date(now.getTime() - opts.deprecRetentionPeriod * 24 * 60 * 60 * 1000); + const deprecMarkRegex = opts.deprecMarkRegex; + const deprecCommentRegex = opts.deprecCommentRegex; + const deprecDateRegex = opts.deprecDateRegex; + const filePath = `${newVersionDir}/00-deprecate.sql`; - if (!config || Object.values(config).some(value => value == null || value === '')) - throw new Error('missing configuration variables'); - - const [sql] = await conn.query(` - WITH variables AS ( - SELECT ? markRegex, ? dateRegex, ? dated - ) - SELECT CONCAT('ALTER TABLE ', c.TABLE_SCHEMA, '.', c.TABLE_NAME, ' DROP PRIMARY KEY;') "sql" + // Generate the drops of the primary keys + const [primaryKeys] = await conn.query(` + SELECT c.TABLE_SCHEMA 'schema', c.TABLE_NAME 'table' FROM information_schema.COLUMNS c LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA AND v.TABLE_NAME = c.TABLE_NAME JOIN information_schema.STATISTICS s ON s.TABLE_SCHEMA = c.TABLE_SCHEMA AND s.TABLE_NAME = c.TABLE_NAME AND s.COLUMN_NAME = c.COLUMN_NAME - JOIN variables var - WHERE c.COLUMN_NAME REGEXP var.markRegex COLLATE utf8mb4_unicode_ci - AND REGEXP_SUBSTR(c.COLUMN_COMMENT, var.dateRegex COLLATE utf8mb4_unicode_ci) < var.dated + WHERE c.COLUMN_NAME REGEXP ? COLLATE utf8mb4_unicode_ci + AND c.COLUMN_COMMENT REGEXP ? COLLATE utf8mb4_unicode_ci + AND REGEXP_SUBSTR(c.COLUMN_COMMENT, ? COLLATE utf8mb4_unicode_ci) < ? AND v.TABLE_NAME IS NULL AND s.INDEX_NAME = 'PRIMARY' - UNION - SELECT CONCAT('ALTER TABLE ', c.TABLE_SCHEMA, '.', c.TABLE_NAME, ' DROP FOREIGN KEY ', kcu.CONSTRAINT_NAME, ';') + `, [deprecMarkRegex, deprecCommentRegex, deprecDateRegex, minDeprecDate]); + + primaryKeys.map(async row => { + await fs.appendFile( + filePath, + 'ALTER TABLE ' + SqlString.escapeId(row.schema, true) + '.' + + SqlString.escapeId(row.table, true) + ' DROP PRIMARY KEY;\n' + ); + }); + + // Generate the drops of the foreign keys + const [foreignKeys] = await conn.query(` + SELECT c.TABLE_SCHEMA 'schema', c.TABLE_NAME 'table', kcu.CONSTRAINT_NAME 'constraint' FROM information_schema.COLUMNS c LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA AND v.TABLE_NAME = c.TABLE_NAME JOIN information_schema.KEY_COLUMN_USAGE kcu ON kcu.TABLE_SCHEMA = c.TABLE_SCHEMA AND kcu.TABLE_NAME = c.TABLE_NAME AND kcu.COLUMN_NAME = c.COLUMN_NAME - JOIN variables var - WHERE c.COLUMN_NAME REGEXP var.markRegex COLLATE utf8mb4_unicode_ci - AND REGEXP_SUBSTR(c.COLUMN_COMMENT, var.dateRegex COLLATE utf8mb4_unicode_ci) < var.dated + WHERE c.COLUMN_NAME REGEXP ? COLLATE utf8mb4_unicode_ci + AND c.COLUMN_COMMENT REGEXP ? COLLATE utf8mb4_unicode_ci + AND REGEXP_SUBSTR(c.COLUMN_COMMENT, ? COLLATE utf8mb4_unicode_ci) < ? AND v.TABLE_NAME IS NULL AND kcu.REFERENCED_COLUMN_NAME IS NOT NULL - UNION - SELECT CONCAT('ALTER TABLE ', c.TABLE_SCHEMA, '.', c.TABLE_NAME, ' DROP COLUMN ', c.COLUMN_NAME, ';') + `, [deprecMarkRegex, deprecCommentRegex, deprecDateRegex, minDeprecDate]); + + foreignKeys.map(async row => { + await fs.appendFile( + filePath, + 'ALTER TABLE ' + SqlString.escapeId(row.schema, true) + '.' + + SqlString.escapeId(row.table, true) + ' DROP FOREIGN KEY ' + + SqlString.escapeId(row.constraint, true) + ';\n' + ); + }); + + // Generate the drops of the columns + const [columns] = await conn.query(` + SELECT c.TABLE_SCHEMA 'schema', c.TABLE_NAME 'table', c.COLUMN_NAME 'column' FROM information_schema.COLUMNS c LEFT JOIN information_schema.VIEWS v ON v.TABLE_SCHEMA = c.TABLE_SCHEMA AND v.TABLE_NAME = c.TABLE_NAME LEFT JOIN information_schema.KEY_COLUMN_USAGE kcu ON kcu.TABLE_SCHEMA = c.TABLE_SCHEMA AND kcu.TABLE_NAME = c.TABLE_NAME AND kcu.COLUMN_NAME = c.COLUMN_NAME - JOIN variables var - WHERE c.COLUMN_NAME REGEXP var.markRegex COLLATE utf8mb4_unicode_ci - AND REGEXP_SUBSTR(c.COLUMN_COMMENT, var.dateRegex COLLATE utf8mb4_unicode_ci) < var.dated + WHERE c.COLUMN_NAME REGEXP ? COLLATE utf8mb4_unicode_ci + AND c.COLUMN_COMMENT REGEXP ? COLLATE utf8mb4_unicode_ci + AND REGEXP_SUBSTR(c.COLUMN_COMMENT, ? COLLATE utf8mb4_unicode_ci) row.sql).join('\n'); + columns.map(async row => { + await fs.appendFile( + filePath, + 'ALTER TABLE ' + SqlString.escapeId(row.schema, true) + '.' + + SqlString.escapeId(row.table, true) + ' DROP COLUMN ' + + SqlString.escapeId(row.column, true) + ';\n' + ); + }); + + // Generate the drops of the tables + const [tables] = await conn.query(` + SELECT TABLE_SCHEMA 'schema', TABLE_NAME 'table' + FROM information_schema.TABLES + WHERE TABLE_NAME REGEXP ? COLLATE utf8mb4_unicode_ci + AND TABLE_COMMENT REGEXP ? COLLATE utf8mb4_unicode_ci + AND REGEXP_SUBSTR(TABLE_COMMENT, ? COLLATE utf8mb4_unicode_ci) < ? + `, [deprecMarkRegex, deprecCommentRegex, deprecDateRegex, minDeprecDate]); + + tables.map(async row => { + await fs.appendFile( + filePath, + 'DROP TABLE ' + SqlString.escapeId(row.schema, true) + '.' + + SqlString.escapeId(row.table, true) + ';\n' + ); + }); } function randomName() { From ea42017e4f293285260f344f5a088c3279a06507 Mon Sep 17 00:00:00 2001 From: guillermo Date: Thu, 12 Sep 2024 09:42:10 +0200 Subject: [PATCH 5/7] feat: refs #7562 Requested changes --- myt-version.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/myt-version.js b/myt-version.js index 9dba08f..2b3ae16 100644 --- a/myt-version.js +++ b/myt-version.js @@ -43,7 +43,7 @@ class Version extends Command { versionCreated: function(versionName) { console.log(`New version created: ${versionName}`); }, - deprecate: 'Generating SQL for deprecate.' + deprecate: 'Generating SQL for deprecated objects deletion.' }; async run(myt, opts) { @@ -132,8 +132,7 @@ class Version extends Command { if (opts.deprecate) { this.emit('deprecate'); await deprecate(conn, opts, newVersionDir); - } - else + } else await fs.writeFile( `${newVersionDir}/00-firstScript.sql`, '-- Place your SQL code here\n' From d92be7533cc0821f199b388f4b43887d8f04bc12 Mon Sep 17 00:00:00 2001 From: guillermo Date: Thu, 12 Sep 2024 09:46:55 +0200 Subject: [PATCH 6/7] feat: refs #7562 Requested changes --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 4dab68d..91dfed8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,2 @@ .DS_Store -node_modules -db -myt.config.yml \ No newline at end of file +node_modules \ No newline at end of file From 5bc8fb083980debef744f117ede3c8669e198c7a Mon Sep 17 00:00:00 2001 From: guillermo Date: Thu, 12 Sep 2024 09:48:38 +0200 Subject: [PATCH 7/7] feat: refs #7562 Version increased --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8171d65..b849b3c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@verdnatura/myt", - "version": "1.6.9", + "version": "1.6.10", "author": "Verdnatura Levante SL", "description": "MySQL version control", "license": "GPL-3.0",