verdnatura/myt
verdnatura
/
myt
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into 5483-unifyDb

This commit is contained in:
Juan Ferrer 2024-01-19 11:50:24 +01:00
parent d23a5bee05 9ab0c0b09c
commit efc391a11a
8 changed files with 265 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -250,7 +250,7 @@ $ myt create [-t <type>] <schema>.<name>
Cleans all already applied versions older than *maxOldVersions*.
```text
$ myt clean
$ myt clean [-p|--purge]
```
## Local server commands

130
myt-clean.js
View File

@ -8,34 +8,55 @@ const path = require('path');
*/
class Clean extends Command {
static usage = {
description: 'Cleans old applied versions'
description: 'Cleans old applied versions',
params: {
purge: 'Wether to remove non-existent scripts from DB log'
}
};
static opts = {
alias: {
purge: 'p'
},
boolean: [
'purge'
],
default: {
remote: 'production'
}
};
static reporter = {
versionsDeleted: function(nVersions) {
console.log(`Old versions deleted: ${nVersions}`);
versionsArchived: function(nVersions) {
if (nVersions)
console.log(` -> ${oldVersions.length} versions archived.`);
else
console.log(` -> No versions archived.`);
},
noVersionsDeleted: 'No versions to delete.'
versionLogPurged: function(nPurged) {
if (nPurged)
console.log(` -> ${nPurged} changes purged from log.`);
else
console.log(` -> No logs purged.`);
}
};
async run(myt, opts) {
await myt.dbConnect();
const version = await myt.fetchDbVersion() || {};
const number = version.number;
const conn = await myt.dbConnect();
const archiveDir = path.join(opts.versionsDir, '.archive');
const dbVersion = await myt.fetchDbVersion() || {};
const number = parseInt(dbVersion.number);
const oldVersions = [];
const versionDirs = await fs.readdir(opts.versionsDir);
for (const versionDir of versionDirs) {
const dirVersion = myt.parseVersionDir(versionDir);
if (!dirVersion) continue;
const version = await myt.loadVersion(versionDir);
const shouldArchive = version
&& !version.apply
&& parseInt(version.number) < number;
if (parseInt(dirVersion.number) < parseInt(number))
if (shouldArchive)
oldVersions.push(versionDir);
}
@ -43,19 +64,94 @@ class Clean extends Command {
&& oldVersions.length > opts.maxOldVersions) {
oldVersions.splice(-opts.maxOldVersions);
const archiveDir = path.join(opts.versionsDir, '.archive');
if (!await fs.pathExists(archiveDir))
await fs.mkdir(archiveDir);
for (const oldVersion of oldVersions)
for (const oldVersion of oldVersions) {
const srcDir = path.join(opts.versionsDir, oldVersion);
const dstDir = path.join(archiveDir, oldVersion);
if (!await fs.pathExists(dstDir))
await fs.mkdir(dstDir);
const scripts = await fs.readdir(srcDir);
for (const script of scripts) {
await fs.move(
path.join(opts.versionsDir, oldVersion),
path.join(archiveDir, oldVersion)
path.join(srcDir, script),
path.join(dstDir, script),
{overwrite: true}
);
}
await fs.rmdir(srcDir);
}
this.emit('versionsArchived', oldVersions.length);
} else
this.emit('versionsArchived');
if (opts.purge) {
const versionDb = new VersionDb(myt, opts.versionsDir);
versionDb.load();
const archiveDb = new VersionDb(myt, archiveDir);
archiveDb.load();
const [res] = await conn.query(
`SELECT number, file FROM versionLog
WHERE code = ?
ORDER BY number, file`,
[opts.code]
);
this.emit('versionsDeleted', oldVersions.length);
} else
this.emit('noVersionsDeleted');
let nPurged = 0;
for (const script of res) {
const hasVersion = await versionDb.hasScript(script);
const hasArchive = await archiveDb.hasScript(script);
if (!hasVersion && !hasArchive) {
await conn.query(
`DELETE FROM versionLog
WHERE code = ? AND number = ? AND file = ?`,
[opts.code, script.number, script.file]
);
nPurged++;
}
}
this.emit('versionLogPurged', nPurged);
}
}
}
class VersionDb {
constructor(myt, baseDir) {
Object.assign(this, {myt, baseDir});
}
async load() {
const map = this.map = new Map();
if (await fs.pathExists(this.baseDir)) {
const dirs = await fs.readdir(this.baseDir);
for (const dir of dirs) {
const version = this.myt.parseVersionDir(dir);
if (!version) continue;
let subdirs = map.get(version.number);
if (!subdirs) map.set(version.number, subdirs = []);
subdirs.push(dir);
}
}
return map;
}
async hasScript(script) {
const dirs = this.map.get(script.number);
if (dirs)
for (const dir of dirs) {
const scriptPath = path.join(this.baseDir, dir, script.file);
if (await fs.pathExists(scriptPath)) return true;
}
return false;
}
}

194
myt-push.js
View File

@ -47,43 +47,51 @@ class Push extends Command {
+ `\n -> Commit: ${version.gitCommit}`
);
},
version(data, action) {
let {version} = data;
let name = data.dir;
let num, color;
switch(action) {
case 'apply':
num = version.number;
name = version.name;
version(version, error) {
let actionMsg;
let number, color;
if (!error) {
actionMsg = version.apply
? '[A]'.green
: '[I]'.blue;
number = version.number;
color = 'cyan';
break;
} else {
actionMsg = '[W]'.yellow;
switch(action) {
case 'badVersion':
num = '?????';
number = '?????';
color = 'yellow';
break;
case 'wrongDirectory':
num = '*****';
number = '*****';
color = 'gray';
break;
}
console.log('', `[${num[color].bold}]`, name);
},
logScript(script, action, error) {
let actionMsg;
switch(action) {
case 'apply':
actionMsg = '[+]'.green;
break;
case 'ignore':
actionMsg = '[I]'.blue;
break;
default:
actionMsg = '[W]'.yellow;
break;
}
console.log(' ', actionMsg.bold, script);
const numberMsg = `[${number}]`[color];
console.log('', `${actionMsg}${numberMsg}`.bold, version.name);
},
logScript(script) {
let actionMsg;
if (script.apply)
actionMsg = '[+]'.green;
else if (!script.matchRegex)
actionMsg = '[W]'.yellow;
else
actionMsg = '[I]'.blue;
console.log(' ', actionMsg.bold, script.file);
},
change(status, ignore, change) {
let actionMsg;
if (ignore)
actionMsg = '[I]'.blue;
else
actionMsg = '[A]'.green;
let statusMsg;
switch(status) {
case 'added':
@ -97,26 +105,25 @@ class Push extends Command {
break;
}
let actionMsg;
if (ignore)
actionMsg = '[I]'.blue;
else
actionMsg = '[A]'.green;
const typeMsg = `[${change.type.abbr}]`[change.type.color];
console.log('',
(statusMsg + actionMsg).bold,
(actionMsg + statusMsg).bold,
typeMsg.bold,
change.fullName
);
},
versionsApplied: function(nVersions, nChanges) {
if (nVersions) {
console.log(` -> ${nVersions} versions with ${nChanges} changes applied.`);
} else
console.log(` -> No versions applied.`);
},
routinesApplied: function(nRoutines) {
if (nRoutines > 0) {
console.log(` -> ${nRoutines} routines have changed.`);
} else {
if (nRoutines) {
console.log(` -> ${nRoutines} routines changed.`);
} else
console.log(` -> No routines changed.`);
}
}
};
async run(myt, opts) {
@ -207,26 +214,23 @@ class Push extends Command {
// Get database version
const version = await myt.fetchDbVersion() || {};
this.emit('dbInfo', version);
const dbVersion = await myt.fetchDbVersion() || {};
this.emit('dbInfo', dbVersion);
if (!version.number)
version.number = String('0').padStart(opts.versionDigits, '0');
if (!/^[0-9]*$/.test(version.number))
if (!dbVersion.number)
dbVersion.number = String('0').padStart(opts.versionDigits, '0');
if (!/^[0-9]*$/.test(dbVersion.number))
throw new Error('Wrong database version');
// Apply versions
this.emit('applyingVersions');
let nVersions = 0;
let nChanges = 0;
let silent = true;
let showLog = false;
const versionsDir = opts.versionsDir;
function isUndoScript(script) {
return /\.undo\.sql$/.test(script);
}
const skipFiles = new Set([
'README.md',
'.archive'
@ -234,83 +238,31 @@ class Push extends Command {
if (await fs.pathExists(versionsDir)) {
const versionDirs = await fs.readdir(versionsDir);
const [[row]] = await this.conn.query(
`SELECT realm FROM versionConfig`
);
const realm = row?.realm;
for (const versionDir of versionDirs) {
if (skipFiles.has(versionDir)) continue;
const version = await myt.loadVersion(versionDir);
const dirVersion = myt.parseVersionDir(versionDir);
const versionData = {
version: dirVersion,
current: version
};
let apply = false;
if (!dirVersion) {
this.emit('version', versionData, 'wrongDirectory');
continue;
}
if (!version)
this.emit('version', version, 'wrongDirectory');
else if (version.number.length != dbVersion.number.length)
this.emit('version', version, 'badVersion');
else
apply = version.apply;
const versionNumber = dirVersion.number;
if (versionNumber.length != version.number.length) {
this.emit('version', versionData, 'badVersion');
continue;
}
const scriptsDir = `${versionsDir}/${versionDir}`;
const scripts = await fs.readdir(scriptsDir);
const [versionLog] = await conn.query(
`SELECT file FROM versionLog
WHERE code = ?
AND number = ?
AND errorNumber IS NULL`,
[opts.code, versionNumber]
);
for (const script of scripts)
if (!isUndoScript(script)
&& versionLog.findIndex(x => x.file == script) === -1) {
silent = false;
break;
}
if (silent) continue;
this.emit('version', versionData, 'apply');
for (const script of scripts) {
const match = script.match(/^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(?:\.undo)?\.sql$/);
if (!match) {
this.emit('logScript', script, 'warn', 'wrongFile');
continue;
}
const skipRealm = match[1] && match[1] !== realm;
if (isUndoScript(script) || skipRealm)
continue;
const [[row]] = await conn.query(
`SELECT errorNumber FROM versionLog
WHERE code = ?
AND number = ?
AND file = ?`,
[
opts.code,
versionNumber,
script
]
);
const apply = !row || row.errorNumber;
this.emit('logScript', script, apply ? 'apply' : 'ignore');
if (apply) showLog = true;
if (showLog) this.emit('version', version);
if (!apply) continue;
for (const script of version.scripts) {
this.emit('logScript', script);
if (!script.apply) continue;
let err;
try {
await connExt.queryFromFile(pushConn,
`${scriptsDir}/${script}`);
`${versionsDir}/${versionDir}/${script.file}`);
} catch (e) {
err = e;
}
@ -331,8 +283,8 @@ class Push extends Command {
errorMessage = VALUES(errorMessage)`,
[
opts.code,
versionNumber,
script,
version.number,
script.file,
err && err.errno,
err && err.message
]
@ -342,16 +294,19 @@ class Push extends Command {
nChanges++;
}
await this.updateVersion('number', versionNumber);
await this.updateVersion('number', version.number);
nVersions++;
}
}
this.emit('versionsApplied', nVersions, nChanges);
// Apply routines
this.emit('applyingRoutines');
let nRoutines = 0;
const changes = await this.changedRoutines(version.gitCommit);
const changes = await this.changedRoutines(dbVersion.gitCommit);
const routines = [];
for (const change of changes)
@ -471,12 +426,11 @@ class Push extends Command {
this.emit('routinesApplied', nRoutines);
const gitExists = await fs.pathExists(`${opts.workspace}/.git`);
if (gitExists && opts.commit) {
const repo = await nodegit.Repository.open(this.opts.workspace);
const head = await repo.getHeadCommit();
if (head && version.gitCommit !== head.sha())
if (head && dbVersion.gitCommit !== head.sha())
await this.updateVersion('gitCommit', head.sha());
}

8
myt-run.js
View File

@ -9,9 +9,9 @@ const connExt = require('./lib/conn');
const SqlString = require('sqlstring');
/**
* Builds the database image and runs a container. It only rebuilds the
* image when dump have been modified. Some workarounds have been used to avoid
* a bug with OverlayFS driver on MacOS.
* Builds the database image and runs a container. It only rebuilds the image
* when dump have been modified. Some workarounds have been used to avoid a bug
* with OverlayFS driver on MacOS.
*/
class Run extends Command {
static usage = {
@ -153,7 +153,7 @@ class Run extends Command {
const hasTriggers = await fs.exists(`${dumpDataDir}/triggers.sql`);
Object.assign(opts, {
triggers: hasTriggers,
triggers: !hasTriggers,
commit: true,
dbConfig
});

6
myt-version.js
View File

@ -82,9 +82,9 @@ class Version extends Command {
const versionNames = new Set();
const versionDirs = await fs.readdir(opts.versionsDir);
for (const versionDir of versionDirs) {
const dirVersion = myt.parseVersionDir(versionDir);
if (!dirVersion) continue;
versionNames.add(dirVersion.name);
const version = myt.parseVersionDir(versionDir);
if (!version) continue;
versionNames.add(version.name);
}
if (!versionName) {

59
myt.js
View File

@ -11,6 +11,8 @@ const mysql = require('mysql2/promise');
const nodegit = require('nodegit');
const camelToSnake = require('./lib/util').camelToSnake;
const scriptRegex = /^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(\.undo)?\.sql$/;
class Myt {
static usage = {
description: 'Utility for database versioning',
@ -311,6 +313,11 @@ class Myt {
`${__dirname}/assets/structure.sql`, 'utf8');
await conn.query(structure);
}
const [[realm]] = await conn.query(
`SELECT realm FROM versionConfig`
);
this.realm = realm;
}
return this.conn;
@ -338,6 +345,58 @@ class Myt {
};
}
async loadVersion(versionDir) {
const {opts} = this;
const info = this.parseVersionDir(versionDir);
if (!info) return null;
const versionsDir = opts.versionsDir;
const scriptsDir = `${versionsDir}/${versionDir}`;
const scriptList = await fs.readdir(scriptsDir);
const [res] = await this.conn.query(
`SELECT file, errorNumber IS NOT NULL hasError
FROM versionLog
WHERE code = ?
AND number = ?`,
[opts.code, info.number]
);
const versionLog = new Map();
res.map(x => versionLog.set(x.file, x));
let applyVersion = false;
const scripts = [];
for (const file of scriptList) {
const match = file.match(scriptRegex);
if (match) {
const scriptRealm = match[1];
const isUndo = !!match[2];
if ((scriptRealm && scriptRealm !== this.realm) || isUndo)
continue;
}
const logInfo = versionLog.get(file);
const apply = !logInfo || logInfo.hasError;
if (apply) applyVersion = true;
scripts.push({
file,
matchRegex: !!match,
apply
});
}
return {
number: info.number,
name: info.name,
scripts,
apply: applyVersion
};
}
async openRepo() {
const {opts} = this;

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "@verdnatura/myt",
"version": "1.5.23",
"version": "1.5.27",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@verdnatura/myt",
"version": "1.5.23",
"version": "1.5.27",
"license": "GPL-3.0",
"dependencies": {
"@sqltools/formatter": "^1.2.5",

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "@verdnatura/myt",
"version": "1.5.24",
"version": "1.5.28",
"author": "Verdnatura Levante SL",
"description": "MySQL version control",
"license": "GPL-3.0",