diff --git a/README.md b/README.md index e4573ce..d70155b 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ $ myt create [-t ] . Cleans all already applied versions older than *maxOldVersions*. ```text -$ myt clean +$ myt clean [-p|--purge] ``` ## Local server commands diff --git a/myt-clean.js b/myt-clean.js index b1cb8d3..93801d3 100644 --- a/myt-clean.js +++ b/myt-clean.js @@ -8,34 +8,55 @@ const path = require('path'); */ class Clean extends Command { static usage = { - description: 'Cleans old applied versions' + description: 'Cleans old applied versions', + params: { + purge: 'Wether to remove non-existent scripts from DB log' + } }; static opts = { + alias: { + purge: 'p' + }, + boolean: [ + 'purge' + ], default: { remote: 'production' } }; static reporter = { - versionsDeleted: function(nVersions) { - console.log(`Old versions deleted: ${nVersions}`); + versionsArchived: function(nVersions) { + if (nVersions) + console.log(` -> ${oldVersions.length} versions archived.`); + else + console.log(` -> No versions archived.`); }, - noVersionsDeleted: 'No versions to delete.' + versionLogPurged: function(nPurged) { + if (nPurged) + console.log(` -> ${nPurged} changes purged from log.`); + else + console.log(` -> No logs purged.`); + } }; async run(myt, opts) { - await myt.dbConnect(); - const version = await myt.fetchDbVersion() || {}; - const number = version.number; + const conn = await myt.dbConnect(); + const archiveDir = path.join(opts.versionsDir, '.archive'); + + const dbVersion = await myt.fetchDbVersion() || {}; + const number = parseInt(dbVersion.number); const oldVersions = []; const versionDirs = await fs.readdir(opts.versionsDir); for (const versionDir of versionDirs) { - const dirVersion = myt.parseVersionDir(versionDir); - if (!dirVersion) continue; + const version = await myt.loadVersion(versionDir); + const shouldArchive = version + && !version.apply + && parseInt(version.number) < number; - if (parseInt(dirVersion.number) < parseInt(number)) + if (shouldArchive) oldVersions.push(versionDir); } @@ -43,19 +64,94 @@ class Clean extends Command { && oldVersions.length > opts.maxOldVersions) { oldVersions.splice(-opts.maxOldVersions); - const archiveDir = path.join(opts.versionsDir, '.archive'); if (!await fs.pathExists(archiveDir)) await fs.mkdir(archiveDir); - for (const oldVersion of oldVersions) - await fs.move( - path.join(opts.versionsDir, oldVersion), - path.join(archiveDir, oldVersion) - ); + for (const oldVersion of oldVersions) { + const srcDir = path.join(opts.versionsDir, oldVersion); + const dstDir = path.join(archiveDir, oldVersion); - this.emit('versionsDeleted', oldVersions.length); + if (!await fs.pathExists(dstDir)) + await fs.mkdir(dstDir); + + const scripts = await fs.readdir(srcDir); + for (const script of scripts) { + await fs.move( + path.join(srcDir, script), + path.join(dstDir, script), + {overwrite: true} + ); + } + + await fs.rmdir(srcDir); + } + + this.emit('versionsArchived', oldVersions.length); } else - this.emit('noVersionsDeleted'); + this.emit('versionsArchived'); + + if (opts.purge) { + const versionDb = new VersionDb(myt, opts.versionsDir); + versionDb.load(); + + const archiveDb = new VersionDb(myt, archiveDir); + archiveDb.load(); + + const [res] = await conn.query( + `SELECT number, file FROM versionLog + WHERE code = ? + ORDER BY number, file`, + [opts.code] + ); + + let nPurged = 0; + for (const script of res) { + const hasVersion = await versionDb.hasScript(script); + const hasArchive = await archiveDb.hasScript(script); + + if (!hasVersion && !hasArchive) { + await conn.query( + `DELETE FROM versionLog + WHERE code = ? AND number = ? AND file = ?`, + [opts.code, script.number, script.file] + ); + nPurged++; + } + } + + this.emit('versionLogPurged', nPurged); + } + } +} + +class VersionDb { + constructor(myt, baseDir) { + Object.assign(this, {myt, baseDir}); + } + + async load() { + const map = this.map = new Map(); + if (await fs.pathExists(this.baseDir)) { + const dirs = await fs.readdir(this.baseDir); + for (const dir of dirs) { + const version = this.myt.parseVersionDir(dir); + if (!version) continue; + let subdirs = map.get(version.number); + if (!subdirs) map.set(version.number, subdirs = []); + subdirs.push(dir); + } + } + return map; + } + + async hasScript(script) { + const dirs = this.map.get(script.number); + if (dirs) + for (const dir of dirs) { + const scriptPath = path.join(this.baseDir, dir, script.file); + if (await fs.pathExists(scriptPath)) return true; + } + return false; } } diff --git a/myt-push.js b/myt-push.js index c91482d..3862312 100644 --- a/myt-push.js +++ b/myt-push.js @@ -47,43 +47,51 @@ class Push extends Command { + `\n -> Commit: ${version.gitCommit}` ); }, - version(data, action) { - let {version} = data; - let name = data.dir; - let num, color; - switch(action) { - case 'apply': - num = version.number; - name = version.name; - color = 'cyan'; - break; - case 'badVersion': - num = '?????'; - color = 'yellow'; - break; - case 'wrongDirectory': - num = '*****'; - color = 'gray'; - break; - } - console.log('', `[${num[color].bold}]`, name); - }, - logScript(script, action, error) { + version(version, error) { let actionMsg; - switch(action) { - case 'apply': - actionMsg = '[+]'.green; - break; - case 'ignore': - actionMsg = '[I]'.blue; - break; - default: - actionMsg = '[W]'.yellow; - break; + let number, color; + + if (!error) { + actionMsg = version.apply + ? '[A]'.green + : '[I]'.blue; + number = version.number; + color = 'cyan'; + } else { + actionMsg = '[W]'.yellow; + switch(action) { + case 'badVersion': + number = '?????'; + color = 'yellow'; + break; + case 'wrongDirectory': + number = '*****'; + color = 'gray'; + break; + } } - console.log(' ', actionMsg.bold, script); + + const numberMsg = `[${number}]`[color]; + console.log('', `${actionMsg}${numberMsg}`.bold, version.name); + }, + logScript(script) { + let actionMsg; + if (script.apply) + actionMsg = '[+]'.green; + else if (!script.matchRegex) + actionMsg = '[W]'.yellow; + else + actionMsg = '[I]'.blue; + + console.log(' ', actionMsg.bold, script.file); }, change(status, ignore, change) { + let actionMsg; + if (ignore) + actionMsg = '[I]'.blue; + else + actionMsg = '[A]'.green; + let statusMsg; switch(status) { case 'added': @@ -97,25 +105,24 @@ class Push extends Command { break; } - let actionMsg; - if (ignore) - actionMsg = '[I]'.blue; - else - actionMsg = '[A]'.green; - const typeMsg = `[${change.type.abbr}]`[change.type.color]; console.log('', - (statusMsg + actionMsg).bold, + (actionMsg + statusMsg).bold, typeMsg.bold, change.fullName ); }, + versionsApplied: function(nVersions, nChanges) { + if (nVersions) { + console.log(` -> ${nVersions} versions with ${nChanges} changes applied.`); + } else + console.log(` -> No versions applied.`); + }, routinesApplied: function(nRoutines) { - if (nRoutines > 0) { - console.log(` -> ${nRoutines} routines have changed.`); - } else { + if (nRoutines) { + console.log(` -> ${nRoutines} routines changed.`); + } else console.log(` -> No routines changed.`); - } } }; @@ -207,26 +214,23 @@ class Push extends Command { // Get database version - const version = await myt.fetchDbVersion() || {}; - this.emit('dbInfo', version); + const dbVersion = await myt.fetchDbVersion() || {}; + this.emit('dbInfo', dbVersion); - if (!version.number) - version.number = String('0').padStart(opts.versionDigits, '0'); - if (!/^[0-9]*$/.test(version.number)) + if (!dbVersion.number) + dbVersion.number = String('0').padStart(opts.versionDigits, '0'); + if (!/^[0-9]*$/.test(dbVersion.number)) throw new Error('Wrong database version'); // Apply versions this.emit('applyingVersions'); + let nVersions = 0; let nChanges = 0; - let silent = true; + let showLog = false; const versionsDir = opts.versionsDir; - function isUndoScript(script) { - return /\.undo\.sql$/.test(script); - } - const skipFiles = new Set([ 'README.md', '.archive' @@ -234,83 +238,31 @@ class Push extends Command { if (await fs.pathExists(versionsDir)) { const versionDirs = await fs.readdir(versionsDir); - const [[row]] = await this.conn.query( - `SELECT realm FROM versionConfig` - ); - const realm = row?.realm; - for (const versionDir of versionDirs) { if (skipFiles.has(versionDir)) continue; + const version = await myt.loadVersion(versionDir); - const dirVersion = myt.parseVersionDir(versionDir); - const versionData = { - version: dirVersion, - current: version - }; + let apply = false; - if (!dirVersion) { - this.emit('version', versionData, 'wrongDirectory'); - continue; - } + if (!version) + this.emit('version', version, 'wrongDirectory'); + else if (version.number.length != dbVersion.number.length) + this.emit('version', version, 'badVersion'); + else + apply = version.apply; - const versionNumber = dirVersion.number; - if (versionNumber.length != version.number.length) { - this.emit('version', versionData, 'badVersion'); - continue; - } + if (apply) showLog = true; + if (showLog) this.emit('version', version); + if (!apply) continue; - const scriptsDir = `${versionsDir}/${versionDir}`; - const scripts = await fs.readdir(scriptsDir); - - const [versionLog] = await conn.query( - `SELECT file FROM versionLog - WHERE code = ? - AND number = ? - AND errorNumber IS NULL`, - [opts.code, versionNumber] - ); - - for (const script of scripts) - if (!isUndoScript(script) - && versionLog.findIndex(x => x.file == script) === -1) { - silent = false; - break; - } - - if (silent) continue; - this.emit('version', versionData, 'apply'); - - for (const script of scripts) { - const match = script.match(/^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(?:\.undo)?\.sql$/); - - if (!match) { - this.emit('logScript', script, 'warn', 'wrongFile'); - continue; - } - - const skipRealm = match[1] && match[1] !== realm; - if (isUndoScript(script) || skipRealm) - continue; - - const [[row]] = await conn.query( - `SELECT errorNumber FROM versionLog - WHERE code = ? - AND number = ? - AND file = ?`, - [ - opts.code, - versionNumber, - script - ] - ); - const apply = !row || row.errorNumber; - this.emit('logScript', script, apply ? 'apply' : 'ignore'); - if (!apply) continue; + for (const script of version.scripts) { + this.emit('logScript', script); + if (!script.apply) continue; let err; try { await connExt.queryFromFile(pushConn, - `${scriptsDir}/${script}`); + `${versionsDir}/${versionDir}/${script.file}`); } catch (e) { err = e; } @@ -331,8 +283,8 @@ class Push extends Command { errorMessage = VALUES(errorMessage)`, [ opts.code, - versionNumber, - script, + version.number, + script.file, err && err.errno, err && err.message ] @@ -342,16 +294,19 @@ class Push extends Command { nChanges++; } - await this.updateVersion('number', versionNumber); + await this.updateVersion('number', version.number); + nVersions++; } } + this.emit('versionsApplied', nVersions, nChanges); + // Apply routines this.emit('applyingRoutines'); let nRoutines = 0; - const changes = await this.changedRoutines(version.gitCommit); + const changes = await this.changedRoutines(dbVersion.gitCommit); const routines = []; for (const change of changes) @@ -471,12 +426,11 @@ class Push extends Command { this.emit('routinesApplied', nRoutines); const gitExists = await fs.pathExists(`${opts.workspace}/.git`); - if (gitExists && opts.commit) { const repo = await nodegit.Repository.open(this.opts.workspace); const head = await repo.getHeadCommit(); - if (head && version.gitCommit !== head.sha()) + if (head && dbVersion.gitCommit !== head.sha()) await this.updateVersion('gitCommit', head.sha()); } diff --git a/myt-run.js b/myt-run.js index 3b6591e..6d9ccb4 100644 --- a/myt-run.js +++ b/myt-run.js @@ -9,9 +9,9 @@ const connExt = require('./lib/conn'); const SqlString = require('sqlstring'); /** - * Builds the database image and runs a container. It only rebuilds the - * image when dump have been modified. Some workarounds have been used to avoid - * a bug with OverlayFS driver on MacOS. + * Builds the database image and runs a container. It only rebuilds the image + * when dump have been modified. Some workarounds have been used to avoid a bug + * with OverlayFS driver on MacOS. */ class Run extends Command { static usage = { @@ -153,7 +153,7 @@ class Run extends Command { const hasTriggers = await fs.exists(`${dumpDataDir}/triggers.sql`); Object.assign(opts, { - triggers: hasTriggers, + triggers: !hasTriggers, commit: true, dbConfig }); diff --git a/myt-version.js b/myt-version.js index c64e331..219bcaf 100644 --- a/myt-version.js +++ b/myt-version.js @@ -82,9 +82,9 @@ class Version extends Command { const versionNames = new Set(); const versionDirs = await fs.readdir(opts.versionsDir); for (const versionDir of versionDirs) { - const dirVersion = myt.parseVersionDir(versionDir); - if (!dirVersion) continue; - versionNames.add(dirVersion.name); + const version = myt.parseVersionDir(versionDir); + if (!version) continue; + versionNames.add(version.name); } if (!versionName) { diff --git a/myt.js b/myt.js index 8bf24f5..e84c941 100755 --- a/myt.js +++ b/myt.js @@ -11,6 +11,8 @@ const mysql = require('mysql2/promise'); const nodegit = require('nodegit'); const camelToSnake = require('./lib/util').camelToSnake; +const scriptRegex = /^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(\.undo)?\.sql$/; + class Myt { static usage = { description: 'Utility for database versioning', @@ -311,6 +313,11 @@ class Myt { `${__dirname}/assets/structure.sql`, 'utf8'); await conn.query(structure); } + + const [[realm]] = await conn.query( + `SELECT realm FROM versionConfig` + ); + this.realm = realm; } return this.conn; @@ -338,6 +345,58 @@ class Myt { }; } + async loadVersion(versionDir) { + const {opts} = this; + + const info = this.parseVersionDir(versionDir); + if (!info) return null; + + const versionsDir = opts.versionsDir; + const scriptsDir = `${versionsDir}/${versionDir}`; + const scriptList = await fs.readdir(scriptsDir); + + const [res] = await this.conn.query( + `SELECT file, errorNumber IS NOT NULL hasError + FROM versionLog + WHERE code = ? + AND number = ?`, + [opts.code, info.number] + ); + const versionLog = new Map(); + res.map(x => versionLog.set(x.file, x)); + + let applyVersion = false; + const scripts = []; + + for (const file of scriptList) { + const match = file.match(scriptRegex); + if (match) { + const scriptRealm = match[1]; + const isUndo = !!match[2]; + + if ((scriptRealm && scriptRealm !== this.realm) || isUndo) + continue; + } + + const logInfo = versionLog.get(file); + const apply = !logInfo || logInfo.hasError; + if (apply) applyVersion = true; + + scripts.push({ + file, + matchRegex: !!match, + apply + }); + } + + return { + number: info.number, + name: info.name, + scripts, + apply: applyVersion + }; + } + async openRepo() { const {opts} = this; diff --git a/package-lock.json b/package-lock.json index 22c8791..638072a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@verdnatura/myt", - "version": "1.5.23", + "version": "1.5.27", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@verdnatura/myt", - "version": "1.5.23", + "version": "1.5.27", "license": "GPL-3.0", "dependencies": { "@sqltools/formatter": "^1.2.5", diff --git a/package.json b/package.json index 48d2b72..bd1d0b1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@verdnatura/myt", - "version": "1.5.24", + "version": "1.5.28", "author": "Verdnatura Levante SL", "description": "MySQL version control", "license": "GPL-3.0",