Merge branch 'master' into 5483-unifyDb
This commit is contained in:
commit
efc391a11a
|
|
@ -250,7 +250,7 @@ $ myt create [-t <type>] <schema>.<name>
|
|||
Cleans all already applied versions older than *maxOldVersions*.
|
||||
|
||||
```text
|
||||
$ myt clean
|
||||
$ myt clean [-p|--purge]
|
||||
```
|
||||
|
||||
## Local server commands
|
||||
|
|
|
|||
132
myt-clean.js
132
myt-clean.js
|
|
@ -8,34 +8,55 @@ const path = require('path');
|
|||
*/
|
||||
class Clean extends Command {
|
||||
static usage = {
|
||||
description: 'Cleans old applied versions'
|
||||
description: 'Cleans old applied versions',
|
||||
params: {
|
||||
purge: 'Wether to remove non-existent scripts from DB log'
|
||||
}
|
||||
};
|
||||
|
||||
static opts = {
|
||||
alias: {
|
||||
purge: 'p'
|
||||
},
|
||||
boolean: [
|
||||
'purge'
|
||||
],
|
||||
default: {
|
||||
remote: 'production'
|
||||
}
|
||||
};
|
||||
|
||||
static reporter = {
|
||||
versionsDeleted: function(nVersions) {
|
||||
console.log(`Old versions deleted: ${nVersions}`);
|
||||
versionsArchived: function(nVersions) {
|
||||
if (nVersions)
|
||||
console.log(` -> ${oldVersions.length} versions archived.`);
|
||||
else
|
||||
console.log(` -> No versions archived.`);
|
||||
},
|
||||
noVersionsDeleted: 'No versions to delete.'
|
||||
versionLogPurged: function(nPurged) {
|
||||
if (nPurged)
|
||||
console.log(` -> ${nPurged} changes purged from log.`);
|
||||
else
|
||||
console.log(` -> No logs purged.`);
|
||||
}
|
||||
};
|
||||
|
||||
async run(myt, opts) {
|
||||
await myt.dbConnect();
|
||||
const version = await myt.fetchDbVersion() || {};
|
||||
const number = version.number;
|
||||
const conn = await myt.dbConnect();
|
||||
const archiveDir = path.join(opts.versionsDir, '.archive');
|
||||
|
||||
const dbVersion = await myt.fetchDbVersion() || {};
|
||||
const number = parseInt(dbVersion.number);
|
||||
|
||||
const oldVersions = [];
|
||||
const versionDirs = await fs.readdir(opts.versionsDir);
|
||||
for (const versionDir of versionDirs) {
|
||||
const dirVersion = myt.parseVersionDir(versionDir);
|
||||
if (!dirVersion) continue;
|
||||
const version = await myt.loadVersion(versionDir);
|
||||
const shouldArchive = version
|
||||
&& !version.apply
|
||||
&& parseInt(version.number) < number;
|
||||
|
||||
if (parseInt(dirVersion.number) < parseInt(number))
|
||||
if (shouldArchive)
|
||||
oldVersions.push(versionDir);
|
||||
}
|
||||
|
||||
|
|
@ -43,19 +64,94 @@ class Clean extends Command {
|
|||
&& oldVersions.length > opts.maxOldVersions) {
|
||||
oldVersions.splice(-opts.maxOldVersions);
|
||||
|
||||
const archiveDir = path.join(opts.versionsDir, '.archive');
|
||||
if (!await fs.pathExists(archiveDir))
|
||||
await fs.mkdir(archiveDir);
|
||||
|
||||
for (const oldVersion of oldVersions)
|
||||
await fs.move(
|
||||
path.join(opts.versionsDir, oldVersion),
|
||||
path.join(archiveDir, oldVersion)
|
||||
);
|
||||
for (const oldVersion of oldVersions) {
|
||||
const srcDir = path.join(opts.versionsDir, oldVersion);
|
||||
const dstDir = path.join(archiveDir, oldVersion);
|
||||
|
||||
this.emit('versionsDeleted', oldVersions.length);
|
||||
if (!await fs.pathExists(dstDir))
|
||||
await fs.mkdir(dstDir);
|
||||
|
||||
const scripts = await fs.readdir(srcDir);
|
||||
for (const script of scripts) {
|
||||
await fs.move(
|
||||
path.join(srcDir, script),
|
||||
path.join(dstDir, script),
|
||||
{overwrite: true}
|
||||
);
|
||||
}
|
||||
|
||||
await fs.rmdir(srcDir);
|
||||
}
|
||||
|
||||
this.emit('versionsArchived', oldVersions.length);
|
||||
} else
|
||||
this.emit('noVersionsDeleted');
|
||||
this.emit('versionsArchived');
|
||||
|
||||
if (opts.purge) {
|
||||
const versionDb = new VersionDb(myt, opts.versionsDir);
|
||||
versionDb.load();
|
||||
|
||||
const archiveDb = new VersionDb(myt, archiveDir);
|
||||
archiveDb.load();
|
||||
|
||||
const [res] = await conn.query(
|
||||
`SELECT number, file FROM versionLog
|
||||
WHERE code = ?
|
||||
ORDER BY number, file`,
|
||||
[opts.code]
|
||||
);
|
||||
|
||||
let nPurged = 0;
|
||||
for (const script of res) {
|
||||
const hasVersion = await versionDb.hasScript(script);
|
||||
const hasArchive = await archiveDb.hasScript(script);
|
||||
|
||||
if (!hasVersion && !hasArchive) {
|
||||
await conn.query(
|
||||
`DELETE FROM versionLog
|
||||
WHERE code = ? AND number = ? AND file = ?`,
|
||||
[opts.code, script.number, script.file]
|
||||
);
|
||||
nPurged++;
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('versionLogPurged', nPurged);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class VersionDb {
|
||||
constructor(myt, baseDir) {
|
||||
Object.assign(this, {myt, baseDir});
|
||||
}
|
||||
|
||||
async load() {
|
||||
const map = this.map = new Map();
|
||||
if (await fs.pathExists(this.baseDir)) {
|
||||
const dirs = await fs.readdir(this.baseDir);
|
||||
for (const dir of dirs) {
|
||||
const version = this.myt.parseVersionDir(dir);
|
||||
if (!version) continue;
|
||||
let subdirs = map.get(version.number);
|
||||
if (!subdirs) map.set(version.number, subdirs = []);
|
||||
subdirs.push(dir);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
async hasScript(script) {
|
||||
const dirs = this.map.get(script.number);
|
||||
if (dirs)
|
||||
for (const dir of dirs) {
|
||||
const scriptPath = path.join(this.baseDir, dir, script.file);
|
||||
if (await fs.pathExists(scriptPath)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
208
myt-push.js
208
myt-push.js
|
|
@ -47,43 +47,51 @@ class Push extends Command {
|
|||
+ `\n -> Commit: ${version.gitCommit}`
|
||||
);
|
||||
},
|
||||
version(data, action) {
|
||||
let {version} = data;
|
||||
let name = data.dir;
|
||||
let num, color;
|
||||
switch(action) {
|
||||
case 'apply':
|
||||
num = version.number;
|
||||
name = version.name;
|
||||
color = 'cyan';
|
||||
break;
|
||||
case 'badVersion':
|
||||
num = '?????';
|
||||
color = 'yellow';
|
||||
break;
|
||||
case 'wrongDirectory':
|
||||
num = '*****';
|
||||
color = 'gray';
|
||||
break;
|
||||
}
|
||||
console.log('', `[${num[color].bold}]`, name);
|
||||
},
|
||||
logScript(script, action, error) {
|
||||
version(version, error) {
|
||||
let actionMsg;
|
||||
switch(action) {
|
||||
case 'apply':
|
||||
actionMsg = '[+]'.green;
|
||||
break;
|
||||
case 'ignore':
|
||||
actionMsg = '[I]'.blue;
|
||||
break;
|
||||
default:
|
||||
actionMsg = '[W]'.yellow;
|
||||
break;
|
||||
let number, color;
|
||||
|
||||
if (!error) {
|
||||
actionMsg = version.apply
|
||||
? '[A]'.green
|
||||
: '[I]'.blue;
|
||||
number = version.number;
|
||||
color = 'cyan';
|
||||
} else {
|
||||
actionMsg = '[W]'.yellow;
|
||||
switch(action) {
|
||||
case 'badVersion':
|
||||
number = '?????';
|
||||
color = 'yellow';
|
||||
break;
|
||||
case 'wrongDirectory':
|
||||
number = '*****';
|
||||
color = 'gray';
|
||||
break;
|
||||
}
|
||||
}
|
||||
console.log(' ', actionMsg.bold, script);
|
||||
|
||||
const numberMsg = `[${number}]`[color];
|
||||
console.log('', `${actionMsg}${numberMsg}`.bold, version.name);
|
||||
},
|
||||
logScript(script) {
|
||||
let actionMsg;
|
||||
if (script.apply)
|
||||
actionMsg = '[+]'.green;
|
||||
else if (!script.matchRegex)
|
||||
actionMsg = '[W]'.yellow;
|
||||
else
|
||||
actionMsg = '[I]'.blue;
|
||||
|
||||
console.log(' ', actionMsg.bold, script.file);
|
||||
},
|
||||
change(status, ignore, change) {
|
||||
let actionMsg;
|
||||
if (ignore)
|
||||
actionMsg = '[I]'.blue;
|
||||
else
|
||||
actionMsg = '[A]'.green;
|
||||
|
||||
let statusMsg;
|
||||
switch(status) {
|
||||
case 'added':
|
||||
|
|
@ -97,25 +105,24 @@ class Push extends Command {
|
|||
break;
|
||||
}
|
||||
|
||||
let actionMsg;
|
||||
if (ignore)
|
||||
actionMsg = '[I]'.blue;
|
||||
else
|
||||
actionMsg = '[A]'.green;
|
||||
|
||||
const typeMsg = `[${change.type.abbr}]`[change.type.color];
|
||||
console.log('',
|
||||
(statusMsg + actionMsg).bold,
|
||||
(actionMsg + statusMsg).bold,
|
||||
typeMsg.bold,
|
||||
change.fullName
|
||||
);
|
||||
},
|
||||
versionsApplied: function(nVersions, nChanges) {
|
||||
if (nVersions) {
|
||||
console.log(` -> ${nVersions} versions with ${nChanges} changes applied.`);
|
||||
} else
|
||||
console.log(` -> No versions applied.`);
|
||||
},
|
||||
routinesApplied: function(nRoutines) {
|
||||
if (nRoutines > 0) {
|
||||
console.log(` -> ${nRoutines} routines have changed.`);
|
||||
} else {
|
||||
if (nRoutines) {
|
||||
console.log(` -> ${nRoutines} routines changed.`);
|
||||
} else
|
||||
console.log(` -> No routines changed.`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -207,26 +214,23 @@ class Push extends Command {
|
|||
|
||||
// Get database version
|
||||
|
||||
const version = await myt.fetchDbVersion() || {};
|
||||
this.emit('dbInfo', version);
|
||||
const dbVersion = await myt.fetchDbVersion() || {};
|
||||
this.emit('dbInfo', dbVersion);
|
||||
|
||||
if (!version.number)
|
||||
version.number = String('0').padStart(opts.versionDigits, '0');
|
||||
if (!/^[0-9]*$/.test(version.number))
|
||||
if (!dbVersion.number)
|
||||
dbVersion.number = String('0').padStart(opts.versionDigits, '0');
|
||||
if (!/^[0-9]*$/.test(dbVersion.number))
|
||||
throw new Error('Wrong database version');
|
||||
|
||||
// Apply versions
|
||||
|
||||
this.emit('applyingVersions');
|
||||
|
||||
let nVersions = 0;
|
||||
let nChanges = 0;
|
||||
let silent = true;
|
||||
let showLog = false;
|
||||
const versionsDir = opts.versionsDir;
|
||||
|
||||
function isUndoScript(script) {
|
||||
return /\.undo\.sql$/.test(script);
|
||||
}
|
||||
|
||||
const skipFiles = new Set([
|
||||
'README.md',
|
||||
'.archive'
|
||||
|
|
@ -234,83 +238,31 @@ class Push extends Command {
|
|||
|
||||
if (await fs.pathExists(versionsDir)) {
|
||||
const versionDirs = await fs.readdir(versionsDir);
|
||||
const [[row]] = await this.conn.query(
|
||||
`SELECT realm FROM versionConfig`
|
||||
);
|
||||
const realm = row?.realm;
|
||||
|
||||
for (const versionDir of versionDirs) {
|
||||
if (skipFiles.has(versionDir)) continue;
|
||||
const version = await myt.loadVersion(versionDir);
|
||||
|
||||
const dirVersion = myt.parseVersionDir(versionDir);
|
||||
const versionData = {
|
||||
version: dirVersion,
|
||||
current: version
|
||||
};
|
||||
let apply = false;
|
||||
|
||||
if (!dirVersion) {
|
||||
this.emit('version', versionData, 'wrongDirectory');
|
||||
continue;
|
||||
}
|
||||
if (!version)
|
||||
this.emit('version', version, 'wrongDirectory');
|
||||
else if (version.number.length != dbVersion.number.length)
|
||||
this.emit('version', version, 'badVersion');
|
||||
else
|
||||
apply = version.apply;
|
||||
|
||||
const versionNumber = dirVersion.number;
|
||||
if (versionNumber.length != version.number.length) {
|
||||
this.emit('version', versionData, 'badVersion');
|
||||
continue;
|
||||
}
|
||||
if (apply) showLog = true;
|
||||
if (showLog) this.emit('version', version);
|
||||
if (!apply) continue;
|
||||
|
||||
const scriptsDir = `${versionsDir}/${versionDir}`;
|
||||
const scripts = await fs.readdir(scriptsDir);
|
||||
|
||||
const [versionLog] = await conn.query(
|
||||
`SELECT file FROM versionLog
|
||||
WHERE code = ?
|
||||
AND number = ?
|
||||
AND errorNumber IS NULL`,
|
||||
[opts.code, versionNumber]
|
||||
);
|
||||
|
||||
for (const script of scripts)
|
||||
if (!isUndoScript(script)
|
||||
&& versionLog.findIndex(x => x.file == script) === -1) {
|
||||
silent = false;
|
||||
break;
|
||||
}
|
||||
|
||||
if (silent) continue;
|
||||
this.emit('version', versionData, 'apply');
|
||||
|
||||
for (const script of scripts) {
|
||||
const match = script.match(/^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(?:\.undo)?\.sql$/);
|
||||
|
||||
if (!match) {
|
||||
this.emit('logScript', script, 'warn', 'wrongFile');
|
||||
continue;
|
||||
}
|
||||
|
||||
const skipRealm = match[1] && match[1] !== realm;
|
||||
if (isUndoScript(script) || skipRealm)
|
||||
continue;
|
||||
|
||||
const [[row]] = await conn.query(
|
||||
`SELECT errorNumber FROM versionLog
|
||||
WHERE code = ?
|
||||
AND number = ?
|
||||
AND file = ?`,
|
||||
[
|
||||
opts.code,
|
||||
versionNumber,
|
||||
script
|
||||
]
|
||||
);
|
||||
const apply = !row || row.errorNumber;
|
||||
this.emit('logScript', script, apply ? 'apply' : 'ignore');
|
||||
if (!apply) continue;
|
||||
for (const script of version.scripts) {
|
||||
this.emit('logScript', script);
|
||||
if (!script.apply) continue;
|
||||
|
||||
let err;
|
||||
try {
|
||||
await connExt.queryFromFile(pushConn,
|
||||
`${scriptsDir}/${script}`);
|
||||
`${versionsDir}/${versionDir}/${script.file}`);
|
||||
} catch (e) {
|
||||
err = e;
|
||||
}
|
||||
|
|
@ -331,8 +283,8 @@ class Push extends Command {
|
|||
errorMessage = VALUES(errorMessage)`,
|
||||
[
|
||||
opts.code,
|
||||
versionNumber,
|
||||
script,
|
||||
version.number,
|
||||
script.file,
|
||||
err && err.errno,
|
||||
err && err.message
|
||||
]
|
||||
|
|
@ -342,16 +294,19 @@ class Push extends Command {
|
|||
nChanges++;
|
||||
}
|
||||
|
||||
await this.updateVersion('number', versionNumber);
|
||||
await this.updateVersion('number', version.number);
|
||||
nVersions++;
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('versionsApplied', nVersions, nChanges);
|
||||
|
||||
// Apply routines
|
||||
|
||||
this.emit('applyingRoutines');
|
||||
|
||||
let nRoutines = 0;
|
||||
const changes = await this.changedRoutines(version.gitCommit);
|
||||
const changes = await this.changedRoutines(dbVersion.gitCommit);
|
||||
|
||||
const routines = [];
|
||||
for (const change of changes)
|
||||
|
|
@ -471,12 +426,11 @@ class Push extends Command {
|
|||
this.emit('routinesApplied', nRoutines);
|
||||
|
||||
const gitExists = await fs.pathExists(`${opts.workspace}/.git`);
|
||||
|
||||
if (gitExists && opts.commit) {
|
||||
const repo = await nodegit.Repository.open(this.opts.workspace);
|
||||
const head = await repo.getHeadCommit();
|
||||
|
||||
if (head && version.gitCommit !== head.sha())
|
||||
if (head && dbVersion.gitCommit !== head.sha())
|
||||
await this.updateVersion('gitCommit', head.sha());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,9 +9,9 @@ const connExt = require('./lib/conn');
|
|||
const SqlString = require('sqlstring');
|
||||
|
||||
/**
|
||||
* Builds the database image and runs a container. It only rebuilds the
|
||||
* image when dump have been modified. Some workarounds have been used to avoid
|
||||
* a bug with OverlayFS driver on MacOS.
|
||||
* Builds the database image and runs a container. It only rebuilds the image
|
||||
* when dump have been modified. Some workarounds have been used to avoid a bug
|
||||
* with OverlayFS driver on MacOS.
|
||||
*/
|
||||
class Run extends Command {
|
||||
static usage = {
|
||||
|
|
@ -153,7 +153,7 @@ class Run extends Command {
|
|||
const hasTriggers = await fs.exists(`${dumpDataDir}/triggers.sql`);
|
||||
|
||||
Object.assign(opts, {
|
||||
triggers: hasTriggers,
|
||||
triggers: !hasTriggers,
|
||||
commit: true,
|
||||
dbConfig
|
||||
});
|
||||
|
|
|
|||
|
|
@ -82,9 +82,9 @@ class Version extends Command {
|
|||
const versionNames = new Set();
|
||||
const versionDirs = await fs.readdir(opts.versionsDir);
|
||||
for (const versionDir of versionDirs) {
|
||||
const dirVersion = myt.parseVersionDir(versionDir);
|
||||
if (!dirVersion) continue;
|
||||
versionNames.add(dirVersion.name);
|
||||
const version = myt.parseVersionDir(versionDir);
|
||||
if (!version) continue;
|
||||
versionNames.add(version.name);
|
||||
}
|
||||
|
||||
if (!versionName) {
|
||||
|
|
|
|||
59
myt.js
59
myt.js
|
|
@ -11,6 +11,8 @@ const mysql = require('mysql2/promise');
|
|||
const nodegit = require('nodegit');
|
||||
const camelToSnake = require('./lib/util').camelToSnake;
|
||||
|
||||
const scriptRegex = /^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(\.undo)?\.sql$/;
|
||||
|
||||
class Myt {
|
||||
static usage = {
|
||||
description: 'Utility for database versioning',
|
||||
|
|
@ -311,6 +313,11 @@ class Myt {
|
|||
`${__dirname}/assets/structure.sql`, 'utf8');
|
||||
await conn.query(structure);
|
||||
}
|
||||
|
||||
const [[realm]] = await conn.query(
|
||||
`SELECT realm FROM versionConfig`
|
||||
);
|
||||
this.realm = realm;
|
||||
}
|
||||
|
||||
return this.conn;
|
||||
|
|
@ -338,6 +345,58 @@ class Myt {
|
|||
};
|
||||
}
|
||||
|
||||
async loadVersion(versionDir) {
|
||||
const {opts} = this;
|
||||
|
||||
const info = this.parseVersionDir(versionDir);
|
||||
if (!info) return null;
|
||||
|
||||
const versionsDir = opts.versionsDir;
|
||||
const scriptsDir = `${versionsDir}/${versionDir}`;
|
||||
const scriptList = await fs.readdir(scriptsDir);
|
||||
|
||||
const [res] = await this.conn.query(
|
||||
`SELECT file, errorNumber IS NOT NULL hasError
|
||||
FROM versionLog
|
||||
WHERE code = ?
|
||||
AND number = ?`,
|
||||
[opts.code, info.number]
|
||||
);
|
||||
const versionLog = new Map();
|
||||
res.map(x => versionLog.set(x.file, x));
|
||||
|
||||
let applyVersion = false;
|
||||
const scripts = [];
|
||||
|
||||
for (const file of scriptList) {
|
||||
const match = file.match(scriptRegex);
|
||||
if (match) {
|
||||
const scriptRealm = match[1];
|
||||
const isUndo = !!match[2];
|
||||
|
||||
if ((scriptRealm && scriptRealm !== this.realm) || isUndo)
|
||||
continue;
|
||||
}
|
||||
|
||||
const logInfo = versionLog.get(file);
|
||||
const apply = !logInfo || logInfo.hasError;
|
||||
if (apply) applyVersion = true;
|
||||
|
||||
scripts.push({
|
||||
file,
|
||||
matchRegex: !!match,
|
||||
apply
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
number: info.number,
|
||||
name: info.name,
|
||||
scripts,
|
||||
apply: applyVersion
|
||||
};
|
||||
}
|
||||
|
||||
async openRepo() {
|
||||
const {opts} = this;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "@verdnatura/myt",
|
||||
"version": "1.5.23",
|
||||
"version": "1.5.27",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@verdnatura/myt",
|
||||
"version": "1.5.23",
|
||||
"version": "1.5.27",
|
||||
"license": "GPL-3.0",
|
||||
"dependencies": {
|
||||
"@sqltools/formatter": "^1.2.5",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@verdnatura/myt",
|
||||
"version": "1.5.24",
|
||||
"version": "1.5.28",
|
||||
"author": "Verdnatura Levante SL",
|
||||
"description": "MySQL version control",
|
||||
"license": "GPL-3.0",
|
||||
|
|
|
|||
Loading…
Reference in New Issue