Merge branch 'master' into 5483-unifyDb

README.md

@@ -250,7 +250,7 @@ $ myt create [-t <type>] <schema>.<name>
 Cleans all already applied versions older than  *maxOldVersions*.
 
 ```text
-$ myt clean
+$ myt clean [-p|--purge]
 ```
 
 ## Local server commands

myt-clean.js

@@ -8,34 +8,55 @@ const path = require('path');
  */
 class Clean extends Command {
     static usage = {
-        description: 'Cleans old applied versions'
+        description: 'Cleans old applied versions',
+        params: {
+            purge: 'Wether to remove non-existent scripts from DB log'
+        }
     };
 
     static opts = {
+        alias: {
+            purge: 'p'
+        },
+        boolean: [
+            'purge'
+        ],
         default: {
             remote: 'production'
         }
     };
 
     static reporter = {
-        versionsDeleted: function(nVersions) {
+        versionsArchived: function(nVersions) {
-            console.log(`Old versions deleted: ${nVersions}`);
+            if (nVersions)
+                console.log(` -> ${oldVersions.length} versions archived.`);
+            else
+                console.log(` -> No versions archived.`);
         },
-        noVersionsDeleted: 'No versions to delete.'
+        versionLogPurged: function(nPurged) {
+            if (nPurged)
+                console.log(` -> ${nPurged} changes purged from log.`);
+            else
+                console.log(` -> No logs purged.`);
+        }
     };
 
     async run(myt, opts) {
-        await myt.dbConnect();
+        const conn = await myt.dbConnect();
-        const version = await myt.fetchDbVersion() || {};
+        const archiveDir = path.join(opts.versionsDir, '.archive');
-        const number = version.number;
+
+        const dbVersion = await myt.fetchDbVersion() || {};
+        const number = parseInt(dbVersion.number);
 
         const oldVersions = [];
         const versionDirs = await fs.readdir(opts.versionsDir);
         for (const versionDir of versionDirs) {
-            const dirVersion = myt.parseVersionDir(versionDir);
+            const version = await myt.loadVersion(versionDir);
-            if (!dirVersion) continue;
+            const shouldArchive = version
+                && !version.apply
+                && parseInt(version.number) < number;
 
-            if (parseInt(dirVersion.number) < parseInt(number))
+            if (shouldArchive)
                 oldVersions.push(versionDir);
         }
 
@@ -43,19 +64,94 @@ class Clean extends Command {
         && oldVersions.length > opts.maxOldVersions) {
             oldVersions.splice(-opts.maxOldVersions);
 
-            const archiveDir = path.join(opts.versionsDir, '.archive');
             if (!await fs.pathExists(archiveDir))
                 await fs.mkdir(archiveDir);
 
-            for (const oldVersion of oldVersions)
+            for (const oldVersion of oldVersions) {
-                await fs.move(
+                const srcDir = path.join(opts.versionsDir, oldVersion);
-                    path.join(opts.versionsDir, oldVersion),
+                const dstDir = path.join(archiveDir, oldVersion);
-                    path.join(archiveDir, oldVersion)
-                );
 
-            this.emit('versionsDeleted', oldVersions.length);
+                if (!await fs.pathExists(dstDir))
+                    await fs.mkdir(dstDir);
+
+                const scripts = await fs.readdir(srcDir);
+                for (const script of scripts) {
+                    await fs.move(
+                        path.join(srcDir, script),
+                        path.join(dstDir, script),
+                        {overwrite: true}
+                    );
+                }
+
+                await fs.rmdir(srcDir);
+            }
+
+            this.emit('versionsArchived', oldVersions.length);
         } else
-            this.emit('noVersionsDeleted');
+            this.emit('versionsArchived');
+
+        if (opts.purge) {
+            const versionDb = new VersionDb(myt, opts.versionsDir);
+            versionDb.load();
+
+            const archiveDb = new VersionDb(myt, archiveDir);
+            archiveDb.load();
+
+            const [res] = await conn.query(
+                `SELECT number, file FROM versionLog
+                    WHERE code = ?
+                    ORDER BY number, file`,
+                [opts.code]
+            );
+
+            let nPurged = 0;
+            for (const script of res) {
+                const hasVersion = await versionDb.hasScript(script);
+                const hasArchive = await archiveDb.hasScript(script);
+    
+                if (!hasVersion && !hasArchive) {
+                    await conn.query(
+                        `DELETE FROM versionLog
+                            WHERE code = ? AND number = ? AND file = ?`,
+                        [opts.code, script.number, script.file]
+                    );
+                    nPurged++;
+                }
+            }
+
+            this.emit('versionLogPurged', nPurged);
+        }
+    }
+}
+
+class VersionDb {
+    constructor(myt, baseDir) {
+        Object.assign(this, {myt, baseDir});
+    }
+
+    async load() {
+        const map = this.map = new Map();
+        if (await fs.pathExists(this.baseDir)) {
+            const dirs = await fs.readdir(this.baseDir);
+            for (const dir of dirs) {
+                const version = this.myt.parseVersionDir(dir);
+                if (!version) continue;
+                let subdirs = map.get(version.number);
+                if (!subdirs) map.set(version.number, subdirs = []);
+                subdirs.push(dir);
+            }
+        }
+        return map;
+    }
+
+    async hasScript(script) {
+        const dirs = this.map.get(script.number);
+        if (dirs)
+            for (const dir of dirs) {
+                const scriptPath = path.join(this.baseDir, dir, script.file);
+                if (await fs.pathExists(scriptPath)) return true;
+            }
+        return false;
     }
 }
 

myt-push.js

@@ -47,43 +47,51 @@ class Push extends Command {
                 + `\n -> Commit: ${version.gitCommit}`
             );
         },
-        version(data, action) {
+        version(version, error) {
-            let {version} = data;
-            let name = data.dir;
-            let num, color;
-            switch(action) {
-                case 'apply':
-                    num = version.number;
-                    name = version.name;
-                    color = 'cyan';
-                    break;
-                case 'badVersion':
-                    num = '?????';
-                    color = 'yellow';
-                    break;
-                case 'wrongDirectory':
-                    num = '*****';
-                    color = 'gray';
-                    break;
-            }
-            console.log('', `[${num[color].bold}]`, name);
-        },
-        logScript(script, action, error) {
             let actionMsg;
-            switch(action) {
+            let number, color;
-                case 'apply':
+
-                    actionMsg = '[+]'.green;
+            if (!error) {
-                    break;
+                actionMsg = version.apply
-                case 'ignore':
+                    ? '[A]'.green
-                    actionMsg = '[I]'.blue;
+                    : '[I]'.blue;
-                    break;
+                number = version.number;
-                default:
+                color = 'cyan';
-                    actionMsg = '[W]'.yellow;
+            } else {
-                    break;
+                actionMsg = '[W]'.yellow;
+                switch(action) {
+                    case 'badVersion':
+                        number = '?????';
+                        color = 'yellow';
+                        break;
+                    case 'wrongDirectory':
+                        number = '*****';
+                        color = 'gray';
+                        break;
+                }
             }
-            console.log(' ', actionMsg.bold, script);
+
+            const numberMsg = `[${number}]`[color];
+            console.log('', `${actionMsg}${numberMsg}`.bold, version.name);
+        },
+        logScript(script) {
+            let actionMsg;
+            if (script.apply)
+                actionMsg = '[+]'.green;
+            else if (!script.matchRegex)
+                actionMsg = '[W]'.yellow;
+            else
+                actionMsg = '[I]'.blue;
+
+            console.log(' ', actionMsg.bold, script.file);
         },
         change(status, ignore, change) {
+            let actionMsg;
+            if (ignore)
+                actionMsg = '[I]'.blue;
+            else
+                actionMsg = '[A]'.green;
+
             let statusMsg;
             switch(status) {
                 case 'added':
@@ -97,25 +105,24 @@ class Push extends Command {
                     break;
             }
 
-            let actionMsg;
-            if (ignore)
-                actionMsg = '[I]'.blue;
-            else
-                actionMsg = '[A]'.green;
-
             const typeMsg = `[${change.type.abbr}]`[change.type.color];
             console.log('',
-                (statusMsg + actionMsg).bold,
+                (actionMsg + statusMsg).bold,
                 typeMsg.bold,
                 change.fullName
             );
         },
+        versionsApplied: function(nVersions, nChanges) {
+            if (nVersions) {
+                console.log(` -> ${nVersions} versions with ${nChanges} changes applied.`);
+            } else
+                console.log(` -> No versions applied.`);
+        },
         routinesApplied: function(nRoutines) {
-            if (nRoutines > 0) {
+            if (nRoutines) {
-                console.log(` -> ${nRoutines} routines have changed.`);
+                console.log(` -> ${nRoutines} routines changed.`);
-            } else {
+            } else
                 console.log(` -> No routines changed.`);
-            }
         }
     };
 
@@ -207,26 +214,23 @@ class Push extends Command {
 
         // Get database version
 
-        const version = await myt.fetchDbVersion() || {};
+        const dbVersion = await myt.fetchDbVersion() || {};
-        this.emit('dbInfo', version);
+        this.emit('dbInfo', dbVersion);
 
-        if (!version.number)
+        if (!dbVersion.number)
-            version.number = String('0').padStart(opts.versionDigits, '0');
+            dbVersion.number = String('0').padStart(opts.versionDigits, '0');
-        if (!/^[0-9]*$/.test(version.number))
+        if (!/^[0-9]*$/.test(dbVersion.number))
             throw new Error('Wrong database version');
 
         // Apply versions
 
         this.emit('applyingVersions');
 
+        let nVersions = 0;
         let nChanges = 0;
-        let silent = true;
+        let showLog = false;
         const versionsDir = opts.versionsDir;
 
-        function isUndoScript(script) {
-            return /\.undo\.sql$/.test(script);
-        }
-       
         const skipFiles = new Set([
             'README.md',
             '.archive'
@@ -234,83 +238,31 @@ class Push extends Command {
 
         if (await fs.pathExists(versionsDir)) {
             const versionDirs = await fs.readdir(versionsDir);
-            const [[row]] = await this.conn.query(
-                `SELECT realm FROM versionConfig`
-            );
-            const realm = row?.realm;
-
             for (const versionDir of versionDirs) {
                 if (skipFiles.has(versionDir)) continue;
+                const version = await myt.loadVersion(versionDir);
 
-                const dirVersion = myt.parseVersionDir(versionDir);
+                let apply = false;
-                const versionData = {
-                    version: dirVersion,
-                    current: version
-                };
 
-                if (!dirVersion) {
+                if (!version)
-                    this.emit('version', versionData, 'wrongDirectory');
+                    this.emit('version', version, 'wrongDirectory');
-                    continue;
+                else if (version.number.length != dbVersion.number.length)
-                }
+                    this.emit('version', version, 'badVersion');
+                else
+                    apply = version.apply;
 
-                const versionNumber = dirVersion.number;
+                if (apply) showLog = true;
-                if (versionNumber.length != version.number.length) {
+                if (showLog) this.emit('version', version);
-                    this.emit('version', versionData, 'badVersion');
+                if (!apply) continue;
-                    continue;
-                }
 
-                const scriptsDir = `${versionsDir}/${versionDir}`;
+                for (const script of version.scripts) {
-                const scripts = await fs.readdir(scriptsDir);
+                    this.emit('logScript', script);
-
+                    if (!script.apply) continue;
-                const [versionLog] = await conn.query(
-                    `SELECT file FROM versionLog
-                        WHERE code = ?
-                            AND number = ?
-                            AND errorNumber IS NULL`,
-                    [opts.code, versionNumber]
-                );
-
-                for (const script of scripts)
-                if (!isUndoScript(script)
-                && versionLog.findIndex(x => x.file == script) === -1) {
-                    silent = false;
-                    break;
-                }
-
-                if (silent) continue;
-                this.emit('version', versionData, 'apply');
-
-                for (const script of scripts) {
-                    const match = script.match(/^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(?:\.undo)?\.sql$/);
-                    
-                    if (!match) {
-                        this.emit('logScript', script, 'warn', 'wrongFile');
-                        continue;
-                    }
-
-                    const skipRealm = match[1] && match[1] !== realm;
-                    if (isUndoScript(script) || skipRealm)
-                        continue;
-
-                    const [[row]] = await conn.query(
-                        `SELECT errorNumber FROM versionLog
-                            WHERE code = ?
-                                AND number = ?
-                                AND file = ?`,
-                        [
-                            opts.code,
-                            versionNumber,
-                            script
-                        ]
-                    );
-                    const apply = !row || row.errorNumber;
-                    this.emit('logScript', script, apply ? 'apply' : 'ignore');
-                    if (!apply) continue;
 
                     let err;
                     try {
                         await connExt.queryFromFile(pushConn,
-                            `${scriptsDir}/${script}`);
+                            `${versionsDir}/${versionDir}/${script.file}`);
                     } catch (e) {
                         err = e;
                     }
@@ -331,8 +283,8 @@ class Push extends Command {
                                 errorMessage = VALUES(errorMessage)`,
                         [
                             opts.code,
-                            versionNumber,
+                            version.number,
-                            script,
+                            script.file,
                             err && err.errno,
                             err && err.message
                         ]
@@ -342,16 +294,19 @@ class Push extends Command {
                     nChanges++;
                 }
 
-                await this.updateVersion('number', versionNumber);
+                await this.updateVersion('number', version.number);
+                nVersions++;
             }
         }
 
+        this.emit('versionsApplied', nVersions, nChanges);
+
         // Apply routines
 
         this.emit('applyingRoutines');
 
         let nRoutines = 0;
-        const changes = await this.changedRoutines(version.gitCommit);
+        const changes = await this.changedRoutines(dbVersion.gitCommit);
 
         const routines = [];
         for (const change of changes)
@@ -471,12 +426,11 @@ class Push extends Command {
         this.emit('routinesApplied', nRoutines);
 
         const gitExists = await fs.pathExists(`${opts.workspace}/.git`);
-
         if (gitExists && opts.commit) {
             const repo = await nodegit.Repository.open(this.opts.workspace);
             const head = await repo.getHeadCommit();
 
-            if (head && version.gitCommit !== head.sha())
+            if (head && dbVersion.gitCommit !== head.sha())
                 await this.updateVersion('gitCommit', head.sha());
         }
 

myt-run.js

@@ -9,9 +9,9 @@ const connExt = require('./lib/conn');
 const SqlString = require('sqlstring');
 
 /**
- * Builds the database image and runs a container. It only rebuilds the
+ * Builds the database image and runs a container. It only rebuilds the image
- * image when dump have been modified. Some workarounds have been used to avoid
+ * when dump have been modified. Some workarounds have been used to avoid a bug
- * a bug with OverlayFS driver on MacOS.
+ * with OverlayFS driver on MacOS.
  */
 class Run extends Command {
     static usage = {
@@ -153,7 +153,7 @@ class Run extends Command {
         const hasTriggers = await fs.exists(`${dumpDataDir}/triggers.sql`);
 
         Object.assign(opts, {
-            triggers: hasTriggers,
+            triggers: !hasTriggers,
             commit: true,
             dbConfig
         });

myt-version.js

@@ -82,9 +82,9 @@ class Version extends Command {
             const versionNames = new Set();
             const versionDirs = await fs.readdir(opts.versionsDir);
             for (const versionDir of versionDirs) {
-                const dirVersion = myt.parseVersionDir(versionDir);
+                const version = myt.parseVersionDir(versionDir);
-                if (!dirVersion) continue;
+                if (!version) continue;
-                versionNames.add(dirVersion.name);
+                versionNames.add(version.name);
             }
 
             if (!versionName) {

myt.js

@@ -11,6 +11,8 @@ const mysql = require('mysql2/promise');
 const nodegit = require('nodegit');
 const camelToSnake = require('./lib/util').camelToSnake;
 
+const scriptRegex = /^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(\.undo)?\.sql$/;
+
 class Myt {
     static usage = {
         description: 'Utility for database versioning',
@@ -311,6 +313,11 @@ class Myt {
                     `${__dirname}/assets/structure.sql`, 'utf8');
                 await conn.query(structure);
             }
+
+            const [[realm]] = await conn.query(
+                `SELECT realm FROM versionConfig`
+            );
+            this.realm = realm;
         }
 
         return this.conn;
@@ -338,6 +345,58 @@ class Myt {
         };
     }
 
+    async loadVersion(versionDir) {
+        const {opts} = this;
+
+        const info = this.parseVersionDir(versionDir);
+        if (!info) return null;
+
+        const versionsDir = opts.versionsDir;
+        const scriptsDir = `${versionsDir}/${versionDir}`;
+        const scriptList = await fs.readdir(scriptsDir);
+
+        const [res] = await this.conn.query(
+            `SELECT file, errorNumber IS NOT NULL hasError
+                FROM versionLog
+                WHERE code = ?
+                    AND number = ?`,
+            [opts.code, info.number]
+        );
+        const versionLog = new Map();
+        res.map(x => versionLog.set(x.file, x));
+
+        let applyVersion = false;
+        const scripts = [];
+
+        for (const file of scriptList) {
+            const match = file.match(scriptRegex);
+            if (match) {
+                const scriptRealm = match[1];
+                const isUndo = !!match[2];
+
+                if ((scriptRealm && scriptRealm !== this.realm) || isUndo)
+                    continue;
+            }
+
+            const logInfo = versionLog.get(file);
+            const apply = !logInfo || logInfo.hasError;
+            if (apply) applyVersion = true;
+
+            scripts.push({
+                file,
+                matchRegex: !!match,
+                apply
+            });
+        }
+
+        return {
+            number: info.number,
+            name: info.name,
+            scripts,
+            apply: applyVersion
+        };
+    }
+
     async openRepo() {
         const {opts} = this;
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@verdnatura/myt",
-  "version": "1.5.23",
+  "version": "1.5.27",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@verdnatura/myt",
-      "version": "1.5.23",
+      "version": "1.5.27",
       "license": "GPL-3.0",
       "dependencies": {
         "@sqltools/formatter": "^1.2.5",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@verdnatura/myt",
-  "version": "1.5.24",
+  "version": "1.5.28",
   "author": "Verdnatura Levante SL",
   "description": "MySQL version control",
   "license": "GPL-3.0",