feat: refs #5483 Split logging from logic #3

Merged
juan merged 4 commits from 5483-unifyDb into master 2024-01-19 10:54:27 +00:00
8 changed files with 265 additions and 156 deletions
Showing only changes of commit efc391a11a - Show all commits

View File

@ -250,7 +250,7 @@ $ myt create [-t <type>] <schema>.<name>
Cleans all already applied versions older than *maxOldVersions*. Cleans all already applied versions older than *maxOldVersions*.
```text ```text
$ myt clean $ myt clean [-p|--purge]
``` ```
## Local server commands ## Local server commands

View File

@ -8,34 +8,55 @@ const path = require('path');
*/ */
class Clean extends Command { class Clean extends Command {
static usage = { static usage = {
description: 'Cleans old applied versions' description: 'Cleans old applied versions',
params: {
purge: 'Wether to remove non-existent scripts from DB log'
}
}; };
static opts = { static opts = {
alias: {
purge: 'p'
},
boolean: [
'purge'
],
default: { default: {
remote: 'production' remote: 'production'
} }
}; };
static reporter = { static reporter = {
versionsDeleted: function(nVersions) { versionsArchived: function(nVersions) {
console.log(`Old versions deleted: ${nVersions}`); if (nVersions)
console.log(` -> ${oldVersions.length} versions archived.`);
else
console.log(` -> No versions archived.`);
}, },
noVersionsDeleted: 'No versions to delete.' versionLogPurged: function(nPurged) {
if (nPurged)
console.log(` -> ${nPurged} changes purged from log.`);
else
console.log(` -> No logs purged.`);
}
}; };
async run(myt, opts) { async run(myt, opts) {
await myt.dbConnect(); const conn = await myt.dbConnect();
const version = await myt.fetchDbVersion() || {}; const archiveDir = path.join(opts.versionsDir, '.archive');
const number = version.number;
const dbVersion = await myt.fetchDbVersion() || {};
const number = parseInt(dbVersion.number);
const oldVersions = []; const oldVersions = [];
const versionDirs = await fs.readdir(opts.versionsDir); const versionDirs = await fs.readdir(opts.versionsDir);
for (const versionDir of versionDirs) { for (const versionDir of versionDirs) {
const dirVersion = myt.parseVersionDir(versionDir); const version = await myt.loadVersion(versionDir);
if (!dirVersion) continue; const shouldArchive = version
&& !version.apply
&& parseInt(version.number) < number;
if (parseInt(dirVersion.number) < parseInt(number)) if (shouldArchive)
oldVersions.push(versionDir); oldVersions.push(versionDir);
} }
@ -43,19 +64,94 @@ class Clean extends Command {
&& oldVersions.length > opts.maxOldVersions) { && oldVersions.length > opts.maxOldVersions) {
oldVersions.splice(-opts.maxOldVersions); oldVersions.splice(-opts.maxOldVersions);
const archiveDir = path.join(opts.versionsDir, '.archive');
if (!await fs.pathExists(archiveDir)) if (!await fs.pathExists(archiveDir))
await fs.mkdir(archiveDir); await fs.mkdir(archiveDir);
for (const oldVersion of oldVersions) for (const oldVersion of oldVersions) {
const srcDir = path.join(opts.versionsDir, oldVersion);
const dstDir = path.join(archiveDir, oldVersion);
if (!await fs.pathExists(dstDir))
await fs.mkdir(dstDir);
const scripts = await fs.readdir(srcDir);
for (const script of scripts) {
await fs.move( await fs.move(
path.join(opts.versionsDir, oldVersion), path.join(srcDir, script),
path.join(archiveDir, oldVersion) path.join(dstDir, script),
{overwrite: true}
);
}
await fs.rmdir(srcDir);
}
this.emit('versionsArchived', oldVersions.length);
} else
this.emit('versionsArchived');
if (opts.purge) {
const versionDb = new VersionDb(myt, opts.versionsDir);
versionDb.load();
const archiveDb = new VersionDb(myt, archiveDir);
archiveDb.load();
const [res] = await conn.query(
`SELECT number, file FROM versionLog
WHERE code = ?
ORDER BY number, file`,
[opts.code]
); );
this.emit('versionsDeleted', oldVersions.length); let nPurged = 0;
} else for (const script of res) {
this.emit('noVersionsDeleted'); const hasVersion = await versionDb.hasScript(script);
const hasArchive = await archiveDb.hasScript(script);
if (!hasVersion && !hasArchive) {
await conn.query(
`DELETE FROM versionLog
WHERE code = ? AND number = ? AND file = ?`,
[opts.code, script.number, script.file]
);
nPurged++;
}
}
this.emit('versionLogPurged', nPurged);
}
}
}
class VersionDb {
constructor(myt, baseDir) {
Object.assign(this, {myt, baseDir});
}
async load() {
const map = this.map = new Map();
if (await fs.pathExists(this.baseDir)) {
const dirs = await fs.readdir(this.baseDir);
for (const dir of dirs) {
const version = this.myt.parseVersionDir(dir);
if (!version) continue;
let subdirs = map.get(version.number);
if (!subdirs) map.set(version.number, subdirs = []);
subdirs.push(dir);
}
}
return map;
}
async hasScript(script) {
const dirs = this.map.get(script.number);
if (dirs)
for (const dir of dirs) {
const scriptPath = path.join(this.baseDir, dir, script.file);
if (await fs.pathExists(scriptPath)) return true;
}
return false;
} }
} }

View File

@ -47,43 +47,51 @@ class Push extends Command {
+ `\n -> Commit: ${version.gitCommit}` + `\n -> Commit: ${version.gitCommit}`
); );
}, },
version(data, action) { version(version, error) {
let {version} = data; let actionMsg;
let name = data.dir; let number, color;
let num, color;
switch(action) { if (!error) {
case 'apply': actionMsg = version.apply
num = version.number; ? '[A]'.green
name = version.name; : '[I]'.blue;
number = version.number;
color = 'cyan'; color = 'cyan';
break; } else {
actionMsg = '[W]'.yellow;
switch(action) {
case 'badVersion': case 'badVersion':
num = '?????'; number = '?????';
color = 'yellow'; color = 'yellow';
break; break;
case 'wrongDirectory': case 'wrongDirectory':
num = '*****'; number = '*****';
color = 'gray'; color = 'gray';
break; break;
} }
console.log('', `[${num[color].bold}]`, name);
},
logScript(script, action, error) {
let actionMsg;
switch(action) {
case 'apply':
actionMsg = '[+]'.green;
break;
case 'ignore':
actionMsg = '[I]'.blue;
break;
default:
actionMsg = '[W]'.yellow;
break;
} }
console.log(' ', actionMsg.bold, script);
const numberMsg = `[${number}]`[color];
console.log('', `${actionMsg}${numberMsg}`.bold, version.name);
},
logScript(script) {
let actionMsg;
if (script.apply)
actionMsg = '[+]'.green;
else if (!script.matchRegex)
actionMsg = '[W]'.yellow;
else
actionMsg = '[I]'.blue;
console.log(' ', actionMsg.bold, script.file);
}, },
change(status, ignore, change) { change(status, ignore, change) {
let actionMsg;
if (ignore)
actionMsg = '[I]'.blue;
else
actionMsg = '[A]'.green;
let statusMsg; let statusMsg;
switch(status) { switch(status) {
case 'added': case 'added':
@ -97,26 +105,25 @@ class Push extends Command {
break; break;
} }
let actionMsg;
if (ignore)
actionMsg = '[I]'.blue;
else
actionMsg = '[A]'.green;
const typeMsg = `[${change.type.abbr}]`[change.type.color]; const typeMsg = `[${change.type.abbr}]`[change.type.color];
console.log('', console.log('',
(statusMsg + actionMsg).bold, (actionMsg + statusMsg).bold,
typeMsg.bold, typeMsg.bold,
change.fullName change.fullName
); );
}, },
versionsApplied: function(nVersions, nChanges) {
if (nVersions) {
console.log(` -> ${nVersions} versions with ${nChanges} changes applied.`);
} else
console.log(` -> No versions applied.`);
},
routinesApplied: function(nRoutines) { routinesApplied: function(nRoutines) {
if (nRoutines > 0) { if (nRoutines) {
console.log(` -> ${nRoutines} routines have changed.`); console.log(` -> ${nRoutines} routines changed.`);
} else { } else
console.log(` -> No routines changed.`); console.log(` -> No routines changed.`);
} }
}
}; };
async run(myt, opts) { async run(myt, opts) {
@ -207,26 +214,23 @@ class Push extends Command {
// Get database version // Get database version
const version = await myt.fetchDbVersion() || {}; const dbVersion = await myt.fetchDbVersion() || {};
this.emit('dbInfo', version); this.emit('dbInfo', dbVersion);
if (!version.number) if (!dbVersion.number)
version.number = String('0').padStart(opts.versionDigits, '0'); dbVersion.number = String('0').padStart(opts.versionDigits, '0');
if (!/^[0-9]*$/.test(version.number)) if (!/^[0-9]*$/.test(dbVersion.number))
throw new Error('Wrong database version'); throw new Error('Wrong database version');
// Apply versions // Apply versions
this.emit('applyingVersions'); this.emit('applyingVersions');
let nVersions = 0;
let nChanges = 0; let nChanges = 0;
let silent = true; let showLog = false;
const versionsDir = opts.versionsDir; const versionsDir = opts.versionsDir;
function isUndoScript(script) {
return /\.undo\.sql$/.test(script);
}
const skipFiles = new Set([ const skipFiles = new Set([
'README.md', 'README.md',
'.archive' '.archive'
@ -234,83 +238,31 @@ class Push extends Command {
if (await fs.pathExists(versionsDir)) { if (await fs.pathExists(versionsDir)) {
const versionDirs = await fs.readdir(versionsDir); const versionDirs = await fs.readdir(versionsDir);
const [[row]] = await this.conn.query(
`SELECT realm FROM versionConfig`
);
const realm = row?.realm;
for (const versionDir of versionDirs) { for (const versionDir of versionDirs) {
if (skipFiles.has(versionDir)) continue; if (skipFiles.has(versionDir)) continue;
const version = await myt.loadVersion(versionDir);
const dirVersion = myt.parseVersionDir(versionDir); let apply = false;
const versionData = {
version: dirVersion,
current: version
};
if (!dirVersion) { if (!version)
this.emit('version', versionData, 'wrongDirectory'); this.emit('version', version, 'wrongDirectory');
continue; else if (version.number.length != dbVersion.number.length)
} this.emit('version', version, 'badVersion');
else
apply = version.apply;
const versionNumber = dirVersion.number; if (apply) showLog = true;
if (versionNumber.length != version.number.length) { if (showLog) this.emit('version', version);
this.emit('version', versionData, 'badVersion');
continue;
}
const scriptsDir = `${versionsDir}/${versionDir}`;
const scripts = await fs.readdir(scriptsDir);
const [versionLog] = await conn.query(
`SELECT file FROM versionLog
WHERE code = ?
AND number = ?
AND errorNumber IS NULL`,
[opts.code, versionNumber]
);
for (const script of scripts)
if (!isUndoScript(script)
&& versionLog.findIndex(x => x.file == script) === -1) {
silent = false;
break;
}
if (silent) continue;
this.emit('version', versionData, 'apply');
for (const script of scripts) {
const match = script.match(/^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(?:\.undo)?\.sql$/);
if (!match) {
this.emit('logScript', script, 'warn', 'wrongFile');
continue;
}
const skipRealm = match[1] && match[1] !== realm;
if (isUndoScript(script) || skipRealm)
continue;
const [[row]] = await conn.query(
`SELECT errorNumber FROM versionLog
WHERE code = ?
AND number = ?
AND file = ?`,
[
opts.code,
versionNumber,
script
]
);
const apply = !row || row.errorNumber;
this.emit('logScript', script, apply ? 'apply' : 'ignore');
if (!apply) continue; if (!apply) continue;
for (const script of version.scripts) {
this.emit('logScript', script);
if (!script.apply) continue;
let err; let err;
try { try {
await connExt.queryFromFile(pushConn, await connExt.queryFromFile(pushConn,
`${scriptsDir}/${script}`); `${versionsDir}/${versionDir}/${script.file}`);
} catch (e) { } catch (e) {
err = e; err = e;
} }
@ -331,8 +283,8 @@ class Push extends Command {
errorMessage = VALUES(errorMessage)`, errorMessage = VALUES(errorMessage)`,
[ [
opts.code, opts.code,
versionNumber, version.number,
script, script.file,
err && err.errno, err && err.errno,
err && err.message err && err.message
] ]
@ -342,16 +294,19 @@ class Push extends Command {
nChanges++; nChanges++;
} }
await this.updateVersion('number', versionNumber); await this.updateVersion('number', version.number);
nVersions++;
} }
} }
this.emit('versionsApplied', nVersions, nChanges);
// Apply routines // Apply routines
this.emit('applyingRoutines'); this.emit('applyingRoutines');
let nRoutines = 0; let nRoutines = 0;
const changes = await this.changedRoutines(version.gitCommit); const changes = await this.changedRoutines(dbVersion.gitCommit);
const routines = []; const routines = [];
for (const change of changes) for (const change of changes)
@ -471,12 +426,11 @@ class Push extends Command {
this.emit('routinesApplied', nRoutines); this.emit('routinesApplied', nRoutines);
const gitExists = await fs.pathExists(`${opts.workspace}/.git`); const gitExists = await fs.pathExists(`${opts.workspace}/.git`);
if (gitExists && opts.commit) { if (gitExists && opts.commit) {
const repo = await nodegit.Repository.open(this.opts.workspace); const repo = await nodegit.Repository.open(this.opts.workspace);
const head = await repo.getHeadCommit(); const head = await repo.getHeadCommit();
if (head && version.gitCommit !== head.sha()) if (head && dbVersion.gitCommit !== head.sha())
await this.updateVersion('gitCommit', head.sha()); await this.updateVersion('gitCommit', head.sha());
} }

View File

@ -9,9 +9,9 @@ const connExt = require('./lib/conn');
const SqlString = require('sqlstring'); const SqlString = require('sqlstring');
/** /**
* Builds the database image and runs a container. It only rebuilds the * Builds the database image and runs a container. It only rebuilds the image
* image when dump have been modified. Some workarounds have been used to avoid * when dump have been modified. Some workarounds have been used to avoid a bug
* a bug with OverlayFS driver on MacOS. * with OverlayFS driver on MacOS.
*/ */
class Run extends Command { class Run extends Command {
static usage = { static usage = {
@ -153,7 +153,7 @@ class Run extends Command {
const hasTriggers = await fs.exists(`${dumpDataDir}/triggers.sql`); const hasTriggers = await fs.exists(`${dumpDataDir}/triggers.sql`);
Object.assign(opts, { Object.assign(opts, {
triggers: hasTriggers, triggers: !hasTriggers,
commit: true, commit: true,
dbConfig dbConfig
}); });

View File

@ -82,9 +82,9 @@ class Version extends Command {
const versionNames = new Set(); const versionNames = new Set();
const versionDirs = await fs.readdir(opts.versionsDir); const versionDirs = await fs.readdir(opts.versionsDir);
for (const versionDir of versionDirs) { for (const versionDir of versionDirs) {
const dirVersion = myt.parseVersionDir(versionDir); const version = myt.parseVersionDir(versionDir);
if (!dirVersion) continue; if (!version) continue;
versionNames.add(dirVersion.name); versionNames.add(version.name);
} }
if (!versionName) { if (!versionName) {

59
myt.js
View File

@ -11,6 +11,8 @@ const mysql = require('mysql2/promise');
const nodegit = require('nodegit'); const nodegit = require('nodegit');
const camelToSnake = require('./lib/util').camelToSnake; const camelToSnake = require('./lib/util').camelToSnake;
const scriptRegex = /^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(\.undo)?\.sql$/;
class Myt { class Myt {
static usage = { static usage = {
description: 'Utility for database versioning', description: 'Utility for database versioning',
@ -311,6 +313,11 @@ class Myt {
`${__dirname}/assets/structure.sql`, 'utf8'); `${__dirname}/assets/structure.sql`, 'utf8');
await conn.query(structure); await conn.query(structure);
} }
const [[realm]] = await conn.query(
`SELECT realm FROM versionConfig`
);
this.realm = realm;
} }
return this.conn; return this.conn;
@ -338,6 +345,58 @@ class Myt {
}; };
} }
async loadVersion(versionDir) {
const {opts} = this;
const info = this.parseVersionDir(versionDir);
if (!info) return null;
const versionsDir = opts.versionsDir;
const scriptsDir = `${versionsDir}/${versionDir}`;
const scriptList = await fs.readdir(scriptsDir);
const [res] = await this.conn.query(
`SELECT file, errorNumber IS NOT NULL hasError
FROM versionLog
WHERE code = ?
AND number = ?`,
[opts.code, info.number]
);
const versionLog = new Map();
res.map(x => versionLog.set(x.file, x));
let applyVersion = false;
const scripts = [];
for (const file of scriptList) {
const match = file.match(scriptRegex);
if (match) {
const scriptRealm = match[1];
const isUndo = !!match[2];
if ((scriptRealm && scriptRealm !== this.realm) || isUndo)
continue;
}
const logInfo = versionLog.get(file);
const apply = !logInfo || logInfo.hasError;
if (apply) applyVersion = true;
scripts.push({
file,
matchRegex: !!match,
apply
});
}
return {
number: info.number,
name: info.name,
scripts,
apply: applyVersion
};
}
async openRepo() { async openRepo() {
const {opts} = this; const {opts} = this;

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{ {
"name": "@verdnatura/myt", "name": "@verdnatura/myt",
"version": "1.5.23", "version": "1.5.27",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@verdnatura/myt", "name": "@verdnatura/myt",
"version": "1.5.23", "version": "1.5.27",
"license": "GPL-3.0", "license": "GPL-3.0",
"dependencies": { "dependencies": {
"@sqltools/formatter": "^1.2.5", "@sqltools/formatter": "^1.2.5",

View File

@ -1,6 +1,6 @@
{ {
"name": "@verdnatura/myt", "name": "@verdnatura/myt",
"version": "1.5.24", "version": "1.5.28",
"author": "Verdnatura Levante SL", "author": "Verdnatura Levante SL",
"description": "MySQL version control", "description": "MySQL version control",
"license": "GPL-3.0", "license": "GPL-3.0",