From 55a47ec99c0f6d49ff05483115d3f8b39a6fbb1e Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Sun, 14 Jan 2024 15:49:49 +0100 Subject: [PATCH 1/4] fix: refs #6661 #5123 #5483 clean, run triggers, push logging, code clean --- myt-clean.js | 17 ++++--- myt-push.js | 127 ++++++++++++++++++---------------------------- myt-run.js | 2 +- myt-version.js | 6 +-- myt.js | 59 +++++++++++++++++++++ package-lock.json | 4 +- package.json | 2 +- 7 files changed, 125 insertions(+), 92 deletions(-) diff --git a/myt-clean.js b/myt-clean.js index 43d0b31..93dbca2 100644 --- a/myt-clean.js +++ b/myt-clean.js @@ -19,16 +19,19 @@ class Clean extends Command { async run(myt, opts) { await myt.dbConnect(); - const version = await myt.fetchDbVersion() || {}; - const number = version.number; + const dbVersion = await myt.fetchDbVersion() || {}; + const number = parseInt(dbVersion.number); const oldVersions = []; const versionDirs = await fs.readdir(opts.versionsDir); for (const versionDir of versionDirs) { - const dirVersion = myt.parseVersionDir(versionDir); - if (!dirVersion) continue; + const version = await myt.loadVersion(versionDir); + const shouldArchive = version + && version.matchRegex + && !version.apply + && parseInt(version.number) < number; - if (parseInt(dirVersion.number) < parseInt(number)) + if (shouldArchive) oldVersions.push(versionDir); } @@ -46,9 +49,9 @@ class Clean extends Command { path.join(archiveDir, oldVersion) ); - console.log(`Old versions deleted: ${oldVersions.length}`); + console.log(`Old versions archived: ${oldVersions.length}`); } else - console.log(`No versions to delete.`); + console.log(`No versions to archive.`); } } diff --git a/myt-push.js b/myt-push.js index ec96400..664b606 100644 --- a/myt-push.js +++ b/myt-push.js @@ -90,17 +90,17 @@ class Push extends Command { // Get database version - const version = await myt.fetchDbVersion() || {}; + const dbVersion = await myt.fetchDbVersion() || {}; console.log( `Database information:` - + `\n -> Version: ${version.number}` - + `\n -> Commit: ${version.gitCommit}` + + `\n -> Version: ${dbVersion.number}` + + `\n -> Commit: ${dbVersion.gitCommit}` ); - if (!version.number) - version.number = String('0').padStart(opts.versionDigits, '0'); - if (!/^[0-9]*$/.test(version.number)) + if (!dbVersion.number) + dbVersion.number = String('0').padStart(opts.versionDigits, '0'); + if (!/^[0-9]*$/.test(dbVersion.number)) throw new Error('Wrong database version'); // Prevent push to production by mistake @@ -142,16 +142,25 @@ class Push extends Command { let silent = true; const versionsDir = opts.versionsDir; - function logVersion(version, name, error) { - console.log('', version.bold, name); + function logVersion(version, name, action, error) { + let actionMsg; + switch(action) { + case 'apply': + actionMsg = '[A]'.green; + break; + case 'ignore': + actionMsg = '[I]'.blue; + break; + default: + actionMsg = '[W]'.yellow; + } + + console.log('', (actionMsg + version).bold, name); } function logScript(type, message, error) { console.log(' ', type.bold, message); } - function isUndoScript(script) { - return /\.undo\.sql$/.test(script); - } - + const skipFiles = new Set([ 'README.md', '.archive' @@ -159,87 +168,49 @@ class Push extends Command { if (await fs.pathExists(versionsDir)) { const versionDirs = await fs.readdir(versionsDir); - const [[realm]] = await this.conn.query( - `SELECT realm - FROM versionConfig` - ); - for (const versionDir of versionDirs) { if (skipFiles.has(versionDir)) continue; + const version = await myt.loadVersion(versionDir); - const dirVersion = myt.parseVersionDir(versionDir); - if (!dirVersion) { - logVersion('[?????]'.yellow, versionDir, + if (!version) { + logVersion('[?????]'.yellow, versionDir, 'warn', `Wrong directory name.` ); continue; } - - const versionNumber = dirVersion.number; - const versionName = dirVersion.name; - - if (versionNumber.length != version.number.length) { - logVersion('[*****]'.gray, versionDir, - `Bad version length, should have ${version.number.length} characters.` + if (version.number.length != dbVersion.number.length) { + logVersion('[*****]'.gray, versionDir, 'warn' + `Bad version length, should have ${dbVersion.number.length} characters.` ); continue; } - const scriptsDir = `${versionsDir}/${versionDir}`; - const scripts = await fs.readdir(scriptsDir); - - const [versionLog] = await conn.query( - `SELECT file FROM versionLog - WHERE code = ? - AND number = ? - AND errorNumber IS NULL`, - [opts.code, versionNumber] - ); - - for (const script of scripts) - if (!isUndoScript(script) - && versionLog.findIndex(x => x.file == script) === -1) { - silent = false; - break; - } - + const {apply} = version; + if (apply) silent = false; if (silent) continue; - logVersion(`[${versionNumber}]`.cyan, versionName); - for (const script of scripts) { - const match = script.match(/^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(?:\.undo)?\.sql$/); + const action = apply ? 'apply' : 'ignore'; + logVersion(`[${version.number}]`.cyan, version.name, action); + + if (!apply) continue; + + for (const script of version.scripts) { + const scriptFile = script.file; - if (!match) { - logScript('[W]'.yellow, script, `Wrong file name.`); + if (!script.matchRegex) { + logScript('[W]'.yellow, scriptFile, `Wrong file name.`); continue; } - const skipRealm = match[1] && match[1] !== realm; - - if (isUndoScript(script) || skipRealm) - continue; - - const [[row]] = await conn.query( - `SELECT errorNumber FROM versionLog - WHERE code = ? - AND number = ? - AND file = ?`, - [ - opts.code, - versionNumber, - script - ] - ); - const apply = !row || row.errorNumber; - const actionMsg = apply ? '[+]'.green : '[I]'.blue; - - logScript(actionMsg, script); - if (!apply) continue; + const actionMsg = script.apply ? '[+]'.green : '[I]'.blue; + logScript(actionMsg, scriptFile); + + if (!script.apply) continue; let err; try { await connExt.queryFromFile(pushConn, - `${scriptsDir}/${script}`); + `${versionsDir}/${versionDir}/${scriptFile}`); } catch (e) { err = e; } @@ -260,8 +231,8 @@ class Push extends Command { errorMessage = VALUES(errorMessage)`, [ opts.code, - versionNumber, - script, + version.number, + scriptFile, err && err.errno, err && err.message ] @@ -271,7 +242,7 @@ class Push extends Command { nChanges++; } - await this.updateVersion('number', versionNumber); + await this.updateVersion('number', version.number); } } @@ -282,7 +253,7 @@ class Push extends Command { const gitExists = await fs.pathExists(`${opts.workspace}/.git`); let nRoutines = 0; - const changes = await this.changedRoutines(version.gitCommit); + const changes = await this.changedRoutines(dbVersion.gitCommit); const routines = []; for (const change of changes) @@ -359,7 +330,7 @@ class Push extends Command { const typeMsg = `[${change.type.abbr}]`[change.type.color]; console.log('', - (statusMsg + actionMsg).bold, + (actionMsg + statusMsg).bold, typeMsg.bold, change.fullName ); @@ -419,7 +390,7 @@ class Push extends Command { const repo = await nodegit.Repository.open(this.opts.workspace); const head = await repo.getHeadCommit(); - if (head && version.gitCommit !== head.sha()) + if (head && dbVersion.gitCommit !== head.sha()) await this.updateVersion('gitCommit', head.sha()); } diff --git a/myt-run.js b/myt-run.js index b78932c..9af22e1 100644 --- a/myt-run.js +++ b/myt-run.js @@ -140,7 +140,7 @@ class Run extends Command { const hasTriggers = await fs.exists(`${dumpDataDir}/triggers.sql`); Object.assign(opts, { - triggers: hasTriggers, + triggers: !hasTriggers, commit: true, dbConfig }); diff --git a/myt-version.js b/myt-version.js index cd4472d..909525e 100644 --- a/myt-version.js +++ b/myt-version.js @@ -74,9 +74,9 @@ class Version extends Command { const versionNames = new Set(); const versionDirs = await fs.readdir(opts.versionsDir); for (const versionDir of versionDirs) { - const dirVersion = myt.parseVersionDir(versionDir); - if (!dirVersion) continue; - versionNames.add(dirVersion.name); + const version = myt.parseVersionDir(versionDir); + if (!version) continue; + versionNames.add(version.name); } if (!versionName) { diff --git a/myt.js b/myt.js index 15b21d7..e4c5f52 100755 --- a/myt.js +++ b/myt.js @@ -11,6 +11,8 @@ const mysql = require('mysql2/promise'); const nodegit = require('nodegit'); const camelToSnake = require('./lib/util').camelToSnake; +const scriptRegex = /^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(\.undo)?\.sql$/; + class Myt { static usage = { description: 'Utility for database versioning', @@ -305,6 +307,11 @@ class Myt { `${__dirname}/assets/structure.sql`, 'utf8'); await conn.query(structure); } + + const [[realm]] = await conn.query( + `SELECT realm FROM versionConfig` + ); + this.realm = realm; } return this.conn; @@ -332,6 +339,58 @@ class Myt { }; } + async loadVersion(versionDir) { + const {opts} = this; + + const info = this.parseVersionDir(versionDir); + if (!info) return null; + + const versionsDir = opts.versionsDir; + const scriptsDir = `${versionsDir}/${versionDir}`; + const scriptList = await fs.readdir(scriptsDir); + + const [res] = await this.conn.query( + `SELECT file, errorNumber IS NOT NULL hasError + FROM versionLog + WHERE code = ? + AND number = ?`, + [opts.code, info.number] + ); + const versionLog = new Map(); + res.map(x => versionLog.set(x.file, x)); + + let applyVersion = false; + const scripts = []; + + for (const file of scriptList) { + const match = file.match(scriptRegex); + if (match) { + const scriptRealm = match[1]; + const isUndo = !!match[2]; + + if ((scriptRealm && scriptRealm !== this.realm) || isUndo) + continue; + } + + const logInfo = versionLog.get(file); + const apply = !logInfo || logInfo.hasError; + if (apply) applyVersion = true; + + scripts.push({ + file, + matchRegex: !!match, + apply + }); + } + + return { + number: info.number, + name: info.name, + scripts, + apply: applyVersion + }; + } + async openRepo() { const {opts} = this; diff --git a/package-lock.json b/package-lock.json index 40bcf92..7df89b6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@verdnatura/myt", - "version": "1.5.22", + "version": "1.5.24", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@verdnatura/myt", - "version": "1.5.22", + "version": "1.5.24", "license": "GPL-3.0", "dependencies": { "@sqltools/formatter": "^1.2.5", diff --git a/package.json b/package.json index a096914..48d2b72 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@verdnatura/myt", - "version": "1.5.23", + "version": "1.5.24", "author": "Verdnatura Levante SL", "description": "MySQL version control", "license": "GPL-3.0", From 85af498e15c5bd13c70eeef6f9f9c5e039238f99 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Wed, 17 Jan 2024 15:14:35 +0100 Subject: [PATCH 2/4] feat: refs #5123 clean fixes, purge option added --- README.md | 5 ++-- myt-clean.js | 73 +++++++++++++++++++++++++++++++++++++++++++---- myt-run.js | 7 ++--- package-lock.json | 4 +-- package.json | 2 +- 5 files changed, 76 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 1f10be3..1ad0035 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ $ myt create [-t ] . Cleans all already applied versions older than *maxOldVersions*. ```text -$ myt clean +$ myt clean [-p|--purge] ``` ## Local server commands @@ -276,8 +276,7 @@ $ myt fixtures [] ### run Builds and starts local database server container. It only rebuilds the image -when fixtures have been modified or when the day on which the image was built -is different to today. +dump has been modified. ```text $ myt run [-c|--ci] [-r|--random] diff --git a/myt-clean.js b/myt-clean.js index 93dbca2..03ef6cc 100644 --- a/myt-clean.js +++ b/myt-clean.js @@ -8,26 +8,36 @@ const path = require('path'); */ class Clean extends Command { static usage = { - description: 'Cleans old applied versions' + description: 'Cleans old applied versions', + params: { + purge: 'Wether to remove non-existent scripts from DB log' + } }; static opts = { + alias: { + purge: 'p' + }, + boolean: [ + 'purge' + ], default: { remote: 'production' } }; async run(myt, opts) { - await myt.dbConnect(); + const conn = await myt.dbConnect(); + const versionDirs = await fs.readdir(opts.versionsDir); + const archiveDir = path.join(opts.versionsDir, '.archive'); + const dbVersion = await myt.fetchDbVersion() || {}; const number = parseInt(dbVersion.number); const oldVersions = []; - const versionDirs = await fs.readdir(opts.versionsDir); for (const versionDir of versionDirs) { const version = await myt.loadVersion(versionDir); const shouldArchive = version - && version.matchRegex && !version.apply && parseInt(version.number) < number; @@ -39,7 +49,6 @@ class Clean extends Command { && oldVersions.length > opts.maxOldVersions) { oldVersions.splice(-opts.maxOldVersions); - const archiveDir = path.join(opts.versionsDir, '.archive'); if (!await fs.pathExists(archiveDir)) await fs.mkdir(archiveDir); @@ -52,6 +61,60 @@ class Clean extends Command { console.log(`Old versions archived: ${oldVersions.length}`); } else console.log(`No versions to archive.`); + + if (opts.purge) { + const versionDb = new VersionDb(myt, opts.versionsDir); + versionDb.load(); + + const archiveDb = new VersionDb(myt, archiveDir); + archiveDb.load(); + + const [res] = await conn.query( + `SELECT number, file FROM versionLog + WHERE code = ? + ORDER BY number, file`, + [opts.code] + ); + + for (const script of res) { + const hasVersion = await versionDb.hasScript(script); + const hasArchive = await archiveDb.hasScript(script); + + if (!hasVersion && !hasArchive) { + await conn.query( + `DELETE FROM versionLog + WHERE code = ? AND number = ? AND file = ?`, + [opts.code, script.number, script.file] + ); + } + } + } + } +} + +class VersionDb { + constructor(myt, baseDir) { + Object.assign(this, {myt, baseDir}); + } + + async load() { + const versionMap = this.versionMap = new Map(); + if (await fs.pathExists(this.baseDir)) { + const dirs = await fs.readdir(this.baseDir); + for (const dir of dirs) { + const version = this.myt.parseVersionDir(dir); + if (!version) continue; + versionMap.set(version.number, dir); + } + } + return versionMap; + } + + async hasScript(script) { + const dir = this.versionMap.get(script.number); + if (!dir) return false; + const scriptPath = path.join(this.baseDir, dir, script.file); + return await fs.pathExists(scriptPath); } } diff --git a/myt-run.js b/myt-run.js index 9af22e1..2915949 100644 --- a/myt-run.js +++ b/myt-run.js @@ -9,10 +9,9 @@ const connExt = require('./lib/conn'); const SqlString = require('sqlstring'); /** - * Builds the database image and runs a container. It only rebuilds the - * image when fixtures have been modified or when the day on which the - * image was built is different to today. Some workarounds have been used - * to avoid a bug with OverlayFS driver on MacOS. + * Builds the database image and runs a container. It only rebuilds the image + * when dump has been modified. Some workarounds have been used to avoid a bug + * with OverlayFS driver on MacOS. */ class Run extends Command { static usage = { diff --git a/package-lock.json b/package-lock.json index 7df89b6..795db60 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@verdnatura/myt", - "version": "1.5.24", + "version": "1.5.25", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@verdnatura/myt", - "version": "1.5.24", + "version": "1.5.25", "license": "GPL-3.0", "dependencies": { "@sqltools/formatter": "^1.2.5", diff --git a/package.json b/package.json index 48d2b72..95ece9d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@verdnatura/myt", - "version": "1.5.24", + "version": "1.5.25", "author": "Verdnatura Levante SL", "description": "MySQL version control", "license": "GPL-3.0", From 7208a8284ff4103d2ae3078a2a53b63a92c99593 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Wed, 17 Jan 2024 17:53:48 +0100 Subject: [PATCH 3/4] fix: refs #5123 clean now merges directories when archive exists --- myt-clean.js | 23 ++++++++++++++++++----- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/myt-clean.js b/myt-clean.js index 03ef6cc..825f003 100644 --- a/myt-clean.js +++ b/myt-clean.js @@ -52,11 +52,24 @@ class Clean extends Command { if (!await fs.pathExists(archiveDir)) await fs.mkdir(archiveDir); - for (const oldVersion of oldVersions) - await fs.move( - path.join(opts.versionsDir, oldVersion), - path.join(archiveDir, oldVersion) - ); + for (const oldVersion of oldVersions) { + const srcDir = path.join(opts.versionsDir, oldVersion); + const dstDir = path.join(archiveDir, oldVersion); + + if (!await fs.pathExists(dstDir)) + await fs.mkdir(dstDir); + + const scripts = await fs.readdir(srcDir); + for (const script of scripts) { + await fs.move( + path.join(srcDir, script), + path.join(dstDir, script), + {overwrite: true} + ); + } + + await fs.rmdir(srcDir); + } console.log(`Old versions archived: ${oldVersions.length}`); } else diff --git a/package-lock.json b/package-lock.json index 795db60..65e2ce1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@verdnatura/myt", - "version": "1.5.25", + "version": "1.5.26", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@verdnatura/myt", - "version": "1.5.25", + "version": "1.5.26", "license": "GPL-3.0", "dependencies": { "@sqltools/formatter": "^1.2.5", diff --git a/package.json b/package.json index 95ece9d..af019fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@verdnatura/myt", - "version": "1.5.25", + "version": "1.5.26", "author": "Verdnatura Levante SL", "description": "MySQL version control", "license": "GPL-3.0", From 9ab0c0b09cba44f37e0497bbed0fb3cd35553b93 Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Fri, 19 Jan 2024 10:16:47 +0100 Subject: [PATCH 4/4] fix(clean): refs #5123 handle numbers with more than one name --- myt-clean.js | 32 ++++++++++++++++++++++---------- myt-push.js | 12 +++++++++--- package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/myt-clean.js b/myt-clean.js index 825f003..fed9bc1 100644 --- a/myt-clean.js +++ b/myt-clean.js @@ -28,13 +28,13 @@ class Clean extends Command { async run(myt, opts) { const conn = await myt.dbConnect(); - const versionDirs = await fs.readdir(opts.versionsDir); const archiveDir = path.join(opts.versionsDir, '.archive'); const dbVersion = await myt.fetchDbVersion() || {}; const number = parseInt(dbVersion.number); const oldVersions = []; + const versionDirs = await fs.readdir(opts.versionsDir); for (const versionDir of versionDirs) { const version = await myt.loadVersion(versionDir); const shouldArchive = version @@ -71,9 +71,9 @@ class Clean extends Command { await fs.rmdir(srcDir); } - console.log(`Old versions archived: ${oldVersions.length}`); + console.log(` -> ${oldVersions.length} versions archived.`); } else - console.log(`No versions to archive.`); + console.log(` -> No versions archived.`); if (opts.purge) { const versionDb = new VersionDb(myt, opts.versionsDir); @@ -89,6 +89,7 @@ class Clean extends Command { [opts.code] ); + let nPurged = 0; for (const script of res) { const hasVersion = await versionDb.hasScript(script); const hasArchive = await archiveDb.hasScript(script); @@ -99,8 +100,14 @@ class Clean extends Command { WHERE code = ? AND number = ? AND file = ?`, [opts.code, script.number, script.file] ); + nPurged++; } } + + if (nPurged) + console.log(` -> ${nPurged} versions purged from log.`); + else + console.log(` -> No versions purged from log.`); } } } @@ -111,23 +118,28 @@ class VersionDb { } async load() { - const versionMap = this.versionMap = new Map(); + const map = this.map = new Map(); if (await fs.pathExists(this.baseDir)) { const dirs = await fs.readdir(this.baseDir); for (const dir of dirs) { const version = this.myt.parseVersionDir(dir); if (!version) continue; - versionMap.set(version.number, dir); + let subdirs = map.get(version.number); + if (!subdirs) map.set(version.number, subdirs = []); + subdirs.push(dir); } } - return versionMap; + return map; } async hasScript(script) { - const dir = this.versionMap.get(script.number); - if (!dir) return false; - const scriptPath = path.join(this.baseDir, dir, script.file); - return await fs.pathExists(scriptPath); + const dirs = this.map.get(script.number); + if (dirs) + for (const dir of dirs) { + const scriptPath = path.join(this.baseDir, dir, script.file); + if (await fs.pathExists(scriptPath)) return true; + } + return false; } } diff --git a/myt-push.js b/myt-push.js index 664b606..c0fac46 100644 --- a/myt-push.js +++ b/myt-push.js @@ -138,6 +138,7 @@ class Push extends Command { console.log('Applying versions.'); + let nVersions = 0; let nChanges = 0; let silent = true; const versionsDir = opts.versionsDir; @@ -191,7 +192,6 @@ class Push extends Command { const action = apply ? 'apply' : 'ignore'; logVersion(`[${version.number}]`.cyan, version.name, action); - if (!apply) continue; for (const script of version.scripts) { @@ -243,9 +243,15 @@ class Push extends Command { } await this.updateVersion('number', version.number); + nVersions++; } } + if (nVersions) { + console.log(` -> ${nVersions} versions with ${nChanges} changes applied.`); + } else + console.log(` -> No versions applied.`); + // Apply routines console.log('Applying changed routines.'); @@ -381,8 +387,8 @@ class Push extends Command { await finalize(); - if (nRoutines > 0) { - console.log(` -> ${nRoutines} routines have changed.`); + if (nRoutines) { + console.log(` -> ${nRoutines} routines changed.`); } else console.log(` -> No routines changed.`); diff --git a/package-lock.json b/package-lock.json index 65e2ce1..638072a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@verdnatura/myt", - "version": "1.5.26", + "version": "1.5.27", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@verdnatura/myt", - "version": "1.5.26", + "version": "1.5.27", "license": "GPL-3.0", "dependencies": { "@sqltools/formatter": "^1.2.5", diff --git a/package.json b/package.json index af019fd..069f070 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@verdnatura/myt", - "version": "1.5.26", + "version": "1.5.27", "author": "Verdnatura Levante SL", "description": "MySQL version control", "license": "GPL-3.0",