verdnatura/myt
verdnatura
/
myt
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

#4036 Fixes, refactor, generated fixtures, code clean

This commit is contained in:
Juan Ferrer 2022-12-21 13:34:17 +01:00
parent 578190458f
commit 313655ea12
31 changed files with 561 additions and 617 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
db.ini → assets/db.ini
View File

0
myvc.default.yml → assets/myvc.default.yml
View File

0
structure.sql → assets/structure.sql
View File

16
lib/command.js Normal file
View File

@ -0,0 +1,16 @@
/**
* Base class for MyVC commands.
*/
module.exports = class MyVCCommand {
get usage() {
return {};
}
get localOpts() {
return {};
}
async run(myvc, opts) {
throw new Error('run command not defined');
}
}

22
docker.js → lib/docker.js
View File

@ -1,6 +1,6 @@
const spawn = require('child_process').spawn;
const execFile = require('child_process').execFile;
const camelToSnake = require('./lib').camelToSnake;
const camelToSnake = require('./util').camelToSnake;
const docker = {
async run(image, commandArgs, options, execOptions) {
@ -9,7 +9,7 @@ const docker = {
: image;
const execMode = options.detach ? 'exec' : 'spawn';
const child = await this.exec('run',
const child = await this.command('run',
args,
options,
execMode,
@ -21,7 +21,7 @@ const docker = {
},
async build(url, options, execOptions) {
return await this.exec('build',
return await this.command('build',
url,
options,
'spawn',
@ -50,7 +50,7 @@ const docker = {
return await ct.inspect(options);
},
async exec(command, args, options, execMode, execOptions) {
async command(command, args, options, execMode, execOptions) {
const execArgs = [command];
if (options)
@ -106,21 +106,27 @@ class Container {
}
async start(options) {
await docker.exec('start', this.id, options);
await docker.command('start', this.id, options);
}
async stop(options) {
await docker.exec('stop', this.id, options);
await docker.command('stop', this.id, options);
}
async rm(options) {
await docker.exec('rm', this.id, options);
await docker.command('rm', this.id, options);
}
async inspect(options) {
const child = await docker.exec('inspect', this.id, options);
const child = await docker.command('inspect', this.id, options);
return JSON.parse(child.stdout);
}
async exec(options, command, commandArgs, execMode, execOptions) {
let args = [this.id, command];
if (commandArgs) args = args.concat(commandArgs);
await docker.command('exec', args, options, execMode, execOptions);
}
}
module.exports = docker;

112
lib/exporter-engine.js Normal file
View File

@ -0,0 +1,112 @@
const shajs = require('sha.js');
const fs = require('fs-extra');
const Exporter = require('./exporter');
module.exports = class ExporterEngine {
constructor(conn, myvcDir) {
this.conn = conn;
this.pullFile = `${myvcDir}/.pullinfo.json`;
this.exporters = [];
this.exporterMap = {};
}
async init () {
if (await fs.pathExists(this.pullFile)) {
this.pullInfo = JSON.parse(await fs.readFile(this.pullFile, 'utf8'));
const lastPull = this.pullInfo.lastPull;
if (lastPull)
this.pullInfo.lastPull = new Date(lastPull);
} else
this.pullInfo = {
lastPull: null,
shaSums: {}
};
this.shaSums = this.pullInfo.shaSums;
this.lastPull = this.pullInfo.lastPull;
this.infoChanged = false;
const types = [
'function',
'procedure',
'view',
'trigger',
'event'
];
for (const type of types) {
const exporter = new Exporter(this, type, this.conn);
await exporter.init();
this.exporters.push(exporter);
this.exporterMap[type] = exporter;
}
}
async fetchRoutine(type, schema, name) {
const exporter = this.exporterMap[type];
const [row] = await exporter.query(schema, name);
return row && exporter.format(row);
}
async fetchShaSum(type, schema, name) {
const sql = await this.fetchRoutine(type, schema, name);
this.setShaSum(type, schema, name, this.shaSum(sql));
}
shaSum(sql) {
if (!sql) return null;
return shajs('sha256')
.update(JSON.stringify(sql))
.digest('hex');
}
getShaSum(type, schema, name) {
try {
return this.shaSums[schema][type][name];
} catch (e) {};
return null;
}
setShaSum(type, schema, name, shaSum) {
if (!shaSum) {
this.deleteShaSum(type, schema, name);
return;
}
const shaSums = this.shaSums;
if (!shaSums[schema])
shaSums[schema] = {};
if (!shaSums[schema][type])
shaSums[schema][type] = {};
shaSums[schema][type][name] = shaSum;
this.infoChanged = true;
}
deleteShaSum(type, schema, name) {
try {
delete this.shaSums[schema][type][name];
this.infoChanged = true;
} catch (e) {};
}
deleteSchemaSums(schema) {
delete this.shaSums[schema];
this.infoChanged = true;
}
async refreshPullDate() {
const [[row]] = await this.conn.query(`SELECT NOW() now`);
this.pullInfo.lastPull = row.now;
this.infoChanged = true;
}
async saveInfo() {
if (!this.infoChanged) return;
await fs.writeFile(this.pullFile,
JSON.stringify(this.pullInfo, null, ' '));
this.infoChanged = false;
}
}

120
lib.js → lib/exporter.js
View File

@ -1,13 +1,8 @@
const ejs = require('ejs');
const shajs = require('sha.js');
const fs = require('fs-extra');
function camelToSnake(str) {
return str.replace(/[A-Z]/g, match => `-${match.toLowerCase()}`);
}
class Exporter {
module.exports = class Exporter {
constructor(engine, objectType, conn) {
this.engine = engine;
this.objectType = objectType;
@ -16,7 +11,7 @@ class Exporter {
}
async init() {
const templateDir = `${__dirname}/exporters/${this.objectType}`;
const templateDir = `${__dirname}/../exporters/${this.objectType}`;
this.sql = await fs.readFile(`${templateDir}.sql`, 'utf8');
const templateFile = await fs.readFile(`${templateDir}.ejs`, 'utf8');
@ -118,114 +113,3 @@ class Exporter {
return this.template(params);
}
}
class ExporterEngine {
constructor(conn, myvcDir) {
this.conn = conn;
this.pullFile = `${myvcDir}/.pullinfo.json`;
this.exporters = [];
this.exporterMap = {};
}
async init () {
if (await fs.pathExists(this.pullFile)) {
this.pullInfo = JSON.parse(await fs.readFile(this.pullFile, 'utf8'));
const lastPull = this.pullInfo.lastPull;
if (lastPull)
this.pullInfo.lastPull = new Date(lastPull);
} else
this.pullInfo = {
lastPull: null,
shaSums: {}
};
this.shaSums = this.pullInfo.shaSums;
this.lastPull = this.pullInfo.lastPull;
this.infoChanged = false;
const types = [
'function',
'procedure',
'view',
'trigger',
'event'
];
for (const type of types) {
const exporter = new Exporter(this, type, this.conn);
await exporter.init();
this.exporters.push(exporter);
this.exporterMap[type] = exporter;
}
}
async fetchRoutine(type, schema, name) {
const exporter = this.exporterMap[type];
const [row] = await exporter.query(schema, name);
return row && exporter.format(row);
}
async fetchShaSum(type, schema, name) {
const sql = await this.fetchRoutine(type, schema, name);
this.setShaSum(type, schema, name, this.shaSum(sql));
}
shaSum(sql) {
if (!sql) return null;
return shajs('sha256')
.update(JSON.stringify(sql))
.digest('hex');
}
getShaSum(type, schema, name) {
try {
return this.shaSums[schema][type][name];
} catch (e) {};
return null;
}
setShaSum(type, schema, name, shaSum) {
if (!shaSum) {
this.deleteShaSum(type, schema, name);
return;
}
const shaSums = this.shaSums;
if (!shaSums[schema])
shaSums[schema] = {};
if (!shaSums[schema][type])
shaSums[schema][type] = {};
shaSums[schema][type][name] = shaSum;
this.infoChanged = true;
}
deleteShaSum(type, schema, name) {
try {
delete this.shaSums[schema][type][name];
this.infoChanged = true;
} catch (e) {};
}
deleteSchemaSums(schema) {
delete this.shaSums[schema];
this.infoChanged = true;
}
async refreshPullDate() {
const [[row]] = await this.conn.query(`SELECT NOW() now`);
this.pullInfo.lastPull = row.now;
this.infoChanged = true;
}
async saveInfo() {
if (!this.infoChanged) return;
await fs.writeFile(this.pullFile,
JSON.stringify(this.pullInfo, null, ' '));
this.infoChanged = false;
}
}
module.exports.camelToSnake = camelToSnake;
module.exports.Exporter = Exporter;
module.exports.ExporterEngine = ExporterEngine;

0
server/server.js → lib/server.js
View File

6
lib/util.js Normal file
View File

@ -0,0 +1,6 @@
function camelToSnake(str) {
return str.replace(/[A-Z]/g, match => `-${match.toLowerCase()}`);
}
module.exports.camelToSnake = camelToSnake;

28
myvc-clean.js
View File

@ -1,33 +1,29 @@
const MyVC = require('./myvc');
const Command = require('./lib/command');
const fs = require('fs-extra');
/**
* Cleans old applied versions.
*/
class Clean {
get usage() {
return {
description: 'Cleans old applied versions'
};
}
class Clean extends Command {
static usage = {
description: 'Cleans old applied versions'
};
get localOpts() {
return {
default: {
remote: 'production'
}
};
}
static localOpts = {
default: {
remote: 'production'
}
};
async run(myvc, opts) {
await myvc.dbConnect();
const version = await myvc.fetchDbVersion() || {};
const number = version.number;
const verionsDir =`${opts.myvcDir}/versions`;
const oldVersions = [];
const versionDirs = await fs.readdir(verionsDir);
const versionDirs = await fs.readdir(opts.versionsDir);
for (const versionDir of versionDirs) {
const dirVersion = myvc.parseVersionDir(versionDir);
if (!dirVersion) continue;
@ -41,7 +37,7 @@ class Clean {
oldVersions.splice(-opts.maxOldVersions);
for (const oldVersion of oldVersions)
await fs.remove(`${verionsDir}/${oldVersion}`,
await fs.remove(`${opts.versionsDir}/${oldVersion}`,
{recursive: true});
console.log(`Old versions deleted: ${oldVersions.length}`);

30
myvc-dump.js
View File

@ -1,23 +1,20 @@
const MyVC = require('./myvc');
const Command = require('./lib/command');
const fs = require('fs-extra');
const path = require('path');
class Dump {
get usage() {
return {
description: 'Dumps structure and fixtures from remote',
operand: 'remote'
};
}
class Dump extends Command {
static usage = {
description: 'Dumps structure and fixtures from remote',
operand: 'remote'
};
get localOpts() {
return {
default: {
remote: 'production'
}
};
}
static localOpts = {
default: {
remote: 'production'
}
};
async run(myvc, opts) {
const dumpStream = await myvc.initDump('.dump.sql');
@ -33,7 +30,7 @@ class Dump {
'--databases'
];
dumpArgs = dumpArgs.concat(opts.schemas);
await myvc.runDump('myvc-dump.sh', dumpArgs, dumpStream);
await myvc.runDump('docker-dump.sh', dumpArgs, dumpStream);
console.log('Dumping fixtures.');
await myvc.dumpFixtures(dumpStream, opts.fixtures);
@ -60,9 +57,8 @@ class Dump {
await myvc.dbConnect();
const version = await myvc.fetchDbVersion();
if (version) {
const dumpDir = path.join(opts.myvcDir, 'dump');
await fs.writeFile(
`${dumpDir}/.dump.json`,
`${opts.dumpDir}/.dump.json`,
JSON.stringify(version)
);
}

3
myvc-dump.sh
View File

@ -1,3 +0,0 @@
#!/bin/bash
mysqldump $@ | sed 's/ AUTO_INCREMENT=[0-9]* //g'

27
myvc-fixtures.js
View File

@ -1,25 +1,22 @@
const MyVC = require('./myvc');
const Command = require('./lib/command');
class Fixtures {
get usage() {
return {
description: 'Dumps local fixtures from database',
operand: 'remote'
};
}
class Fixtures extends Command {
static usage = {
description: 'Dumps local fixtures from database',
operand: 'remote'
};
get localOpts() {
return {
default: {
remote: 'docker'
}
};
}
static localOpts = {
default: {
remote: 'docker'
}
};
async run(myvc, opts) {
const dumpStream = await myvc.initDump('fixtures.sql');
await myvc.dumpFixtures(dumpStream, opts.localFixtures);
await myvc.dumpFixtures(dumpStream, opts.localFixtures, true);
await dumpStream.end();
}
}

11
myvc-init.js
View File

@ -1,13 +1,12 @@
const MyVC = require('./myvc');
const Command = require('./lib/command');
const fs = require('fs-extra');
class Init {
get usage() {
return {
description: 'Initialize an empty workspace'
};
}
class Init extends Command {
static usage = {
description: 'Initialize an empty workspace'
};
async run(myvc, opts) {
const templateDir = `${__dirname}/template`;

73
myvc-pull.js
View File

@ -1,38 +1,35 @@
const MyVC = require('./myvc');
const Command = require('./lib/command');
const fs = require('fs-extra');
const nodegit = require('nodegit');
const ExporterEngine = require('./lib').ExporterEngine;
class Pull {
get usage() {
return {
description: 'Incorporate database routine changes into workspace',
params: {
force: 'Do it even if there are local changes',
checkout: 'Move to same database commit before pull',
update: 'Update all routines',
sums: 'Save SHA sums of all objects'
},
operand: 'remote'
};
}
const ExporterEngine = require('./lib/exporter-engine');
class Pull extends Command {
static usage = {
description: 'Incorporate database routine changes into workspace',
params: {
force: 'Do it even if there are local changes',
checkout: 'Move to same database commit before pull',
update: 'Update all routines',
sums: 'Save SHA sums of all objects'
},
operand: 'remote'
};
get localOpts() {
return {
alias: {
force: 'f',
checkout: 'c',
update: 'u',
sums: 's'
},
boolean: [
'force',
'checkout',
'update',
'sums'
]
};
}
static localOpts = {
alias: {
force: 'f',
checkout: 'c',
update: 'u',
sums: 's'
},
boolean: [
'force',
'checkout',
'update',
'sums'
]
};
async run(myvc, opts) {
const conn = await myvc.dbConnect();
@ -91,32 +88,32 @@ class Pull {
await engine.init();
const shaSums = engine.shaSums;
const exportDir = `${opts.myvcDir}/routines`;
if (!await fs.pathExists(exportDir))
await fs.mkdir(exportDir);
const routinesDir = opts.routinesDir;
if (!await fs.pathExists(routinesDir))
await fs.mkdir(routinesDir);
// Delete old schemas
const schemas = await fs.readdir(exportDir);
const schemas = await fs.readdir(routinesDir);
for (const schema of schemas) {
if (opts.schemas.indexOf(schema) == -1)
await fs.remove(`${exportDir}/${schema}`, {recursive: true});
await fs.remove(`${routinesDir}/${schema}`, {recursive: true});
}
for (const schema in shaSums) {
if (!await fs.pathExists(`${exportDir}/${schema}`))
if (!await fs.pathExists(`${routinesDir}/${schema}`))
engine.deleteSchemaSums(schema);
}
// Export objects to SQL files
for (const schema of opts.schemas) {
let schemaDir = `${exportDir}/${schema}`;
let schemaDir = `${routinesDir}/${schema}`;
if (!await fs.pathExists(schemaDir))
await fs.mkdir(schemaDir);
for (const exporter of engine.exporters)
await exporter.export(exportDir,
await exporter.export(routinesDir,
schema, opts.update, opts.sums);
}

195
myvc-push.js
View File

@ -1,39 +1,36 @@
const MyVC = require('./myvc');
const Command = require('./lib/command');
const fs = require('fs-extra');
const nodegit = require('nodegit');
const ExporterEngine = require('./lib').ExporterEngine;
const ExporterEngine = require('./lib/exporter-engine');
/**
* Pushes changes to remote.
*/
class Push {
get usage() {
return {
description: 'Apply changes into database',
params: {
force: 'Answer yes to all questions',
commit: 'Wether to save the commit SHA into database',
sums: 'Save SHA sums of pushed objects'
},
operand: 'remote'
};
}
class Push extends Command {
static usage = {
description: 'Apply changes into database',
params: {
force: 'Answer yes to all questions',
commit: 'Wether to save the commit SHA into database',
sums: 'Save SHA sums of pushed objects'
},
operand: 'remote'
};
get localOpts() {
return {
alias: {
force: 'f',
commit: 'c',
sums: 's'
},
boolean: [
'force',
'commit',
'sums'
]
};
}
static localOpts = {
alias: {
force: 'f',
commit: 'c',
sums: 's'
},
boolean: [
'force',
'commit',
'sums'
]
};
async run(myvc, opts) {
const conn = await myvc.dbConnect();
@ -132,7 +129,7 @@ class Push {
let nChanges = 0;
let silent = true;
const versionsDir = `${opts.myvcDir}/versions`;
const versionsDir = opts.versionsDir;
function logVersion(version, name, error) {
console.log('', version.bold, name);
@ -217,7 +214,7 @@ class Push {
let err;
try {
await this.queryFromFile(pushConn,
await myvc.queryFromFile(pushConn,
`${scriptsDir}/${script}`);
} catch (e) {
err = e;
@ -261,9 +258,7 @@ class Push {
const gitExists = await fs.pathExists(`${opts.workspace}/.git`);
let nRoutines = 0;
let changes = gitExists
? await myvc.changedRoutines(version.gitCommit)
: await myvc.cachedChanges();
let changes = await myvc.changedRoutines(version.gitCommit);
changes = this.parseChanges(changes);
const routines = [];
@ -305,7 +300,7 @@ class Push {
const schema = change.schema;
const name = change.name;
const type = change.type.name.toLowerCase();
const fullPath = `${opts.myvcDir}/routines/${change.path}.sql`;
const fullPath = `${opts.routinesDir}/${change.path}.sql`;
const exists = await fs.pathExists(fullPath);
let newSql;
@ -333,7 +328,7 @@ class Push {
if (change.type.name === 'VIEW')
await pushConn.query(`USE ${scapedSchema}`);
await this.multiQuery(pushConn, newSql);
await myvc.multiQuery(pushConn, newSql);
if (change.isRoutine) {
await conn.query(
@ -415,104 +410,6 @@ class Push {
);
}
/**
* Executes a multi-query string.
*
* @param {Connection} conn MySQL connection object
* @param {String} sql SQL multi-query string
* @returns {Array<Result>} The resultset
*/
async multiQuery(conn, sql) {
let results = [];
const stmts = this.querySplit(sql);
for (const stmt of stmts)
results = results.concat(await conn.query(stmt));
return results;
}
/**
* Executes an SQL script.
*
* @param {Connection} conn MySQL connection object
* @returns {Array<Result>} The resultset
*/
async queryFromFile(conn, file) {
const sql = await fs.readFile(file, 'utf8');
return await this.multiQuery(conn, sql);
}
/**
* Splits an SQL muti-query into a single-query array, it does an small
* parse to correctly handle the DELIMITER statement.
*
* @param {Array<String>} stmts The splitted SQL statements
*/
querySplit(sql) {
const stmts = [];
let i,
char,
token,
escaped,
stmtStart;
let delimiter = ';';
const delimiterRe = /\s*delimiter\s+(\S+)[^\S\r\n]*(?:\r?\n|\r|$)/yi;
function begins(str) {
let j;
for (j = 0; j < str.length; j++)
if (sql[i + j] != str[j])
return false;
i += j;
return true;
}
for (i = 0; i < sql.length;) {
stmtStart = i;
delimiterRe.lastIndex = i;
const match = sql.match(delimiterRe);
if (match) {
delimiter = match[1];
i += match[0].length;
continue;
}
let delimiterFound = false;
while (i < sql.length) {
char = sql[i];
if (token) {
if (!escaped && begins(token.end))
token = null;
else {
escaped = !escaped && token.escape(char);
i++;
}
} else {
delimiterFound = begins(delimiter);
if (delimiterFound) break;
const tok = tokenIndex.get(char);
if (tok && begins(tok.start))
token = tok;
else
i++;
}
}
let len = i - stmtStart;
if (delimiterFound) len -= delimiter.length;
const stmt = sql.substr(stmtStart, len);
if (!/^\s*$/.test(stmt))
stmts.push(stmt);
}
return stmts;
}
}
const typeMap = {
@ -572,40 +469,6 @@ class Routine {
}
}
const tokens = {
string: {
start: '\'',
end: '\'',
escape: char => char == '\'' || char == '\\'
},
quotedString: {
start: '"',
end: '"',
escape: char => char == '"' || char == '\\'
},
id: {
start: '`',
end: '`',
escape: char => char == '`'
},
multiComment: {
start: '/*',
end: '*/',
escape: () => false
},
singleComment: {
start: '-- ',
end: '\n',
escape: () => false
}
};
const tokenIndex = new Map();
for (const tokenId in tokens) {
const token = tokens[tokenId];
tokenIndex.set(token.start[0], token);
}
module.exports = Push;
if (require.main === module)

123
myvc-run.js
View File

@ -1,10 +1,11 @@
const MyVC = require('./myvc');
const docker = require('./docker');
const Container = require('./docker').Container;
const Command = require('./lib/command');
const Push = require('./myvc-push');
const docker = require('./lib/docker');
const fs = require('fs-extra');
const path = require('path');
const Server = require('./server/server');
const Server = require('./lib/server');
/**
* Builds the database image and runs a container. It only rebuilds the
@ -12,65 +13,33 @@ const Server = require('./server/server');
* image was built is different to today. Some workarounds have been used
* to avoid a bug with OverlayFS driver on MacOS.
*/
class Run {
get usage() {
return {
description: 'Build and start local database server container',
params: {
ci: 'Workaround for continuous integration system',
random: 'Whether to use a random container name or port'
}
};
}
class Run extends Command {
static usage = {
description: 'Build and start local database server container',
params: {
ci: 'Workaround for continuous integration system',
random: 'Whether to use a random container name or port'
}
};
get localOpts() {
return {
alias: {
ci: 'c',
random: 'r'
},
boolean: [
'ci',
'random'
]
};
}
static localOpts = {
alias: {
ci: 'c',
random: 'r'
},
boolean: [
'ci',
'random'
]
};
async run(myvc, opts) {
const dumpDir = `${opts.myvcDir}/dump`;
const dumpDir = opts.dumpDir;
const serverDir = path.join(__dirname, 'server');
// Fetch dump information
if (!await fs.pathExists(`${dumpDir}/.dump.sql`))
throw new Error('To run local database you have to create a dump first');
const dumpInfo = `${dumpDir}/.dump.json`;
if (await fs.pathExists(dumpInfo)) {
const cache = await myvc.cachedChanges();
const version = JSON.parse(
await fs.readFileSync(dumpInfo, 'utf8')
);
const changes = await myvc.changedRoutines(version.gitCommit);
let isEqual = false;
if (cache && changes && cache.length == changes.length)
for (let i = 0; i < changes.length; i++) {
isEqual = cache[i].path == changes[i].path
&& cache[i].mark == changes[i].mark;
if (!isEqual) break;
}
if (!isEqual) {
const fd = await fs.open(`${dumpDir}/.changes`, 'w+');
for (const change of changes)
await fs.write(fd, change.mark + change.path + '\n');
await fs.close(fd);
}
}
// Build base server image
let serverDockerfile = path.join(dumpDir, 'Dockerfile');
@ -119,7 +88,7 @@ class Run {
publish: `3306:${dbConfig.port}`
};
try {
const server = new Server(new Container(opts.code));
const server = new Server(new docker.Container(opts.code));
await server.rm();
} catch (e) {}
}
@ -128,13 +97,14 @@ class Run {
Object.assign(runOptions, null, {
env: `RUN_CHOWN=${runChown}`,
detach: true
detach: true,
volume: `${path.join(dumpDir, 'fixtures.sql')}:/fixtures.sql:ro`
});
const ct = await docker.run(opts.code, null, runOptions);
const server = new Server(ct, dbConfig);
try {
if (isRandom) {
if (isRandom) {
try {
const netSettings = await ct.inspect({
format: '{{json .NetworkSettings}}'
});
@ -143,14 +113,43 @@ class Run {
dbConfig.host = netSettings.Gateway;
dbConfig.port = netSettings.Ports['3306/tcp'][0].HostPort;
}
} catch (err) {
if (isRandom)
} catch (err) {
await server.rm();
throw err;
throw err;
}
}
await server.wait();
// Apply changes
Object.assign(opts, {
commit: true,
dbConfig
});
await myvc.runCommand(Push, opts);
// Apply fixtures
console.log('Applying fixtures.');
await ct.exec(null,
'docker-import.sh', ['/fixtures'], 'spawn', opts.debug);
// Create triggers
console.log('Creating triggers.');
const conn = await myvc.createConnection();
for (const schema of opts.schemas) {
const triggersPath = `${opts.routinesDir}/${schema}/triggers`;
if (!await fs.pathExists(triggersPath))
continue;
const triggersDir = await fs.readdir(triggersPath);
for (const triggerFile of triggersDir)
await myvc.queryFromFile(conn, `${triggersPath}/${triggerFile}`);
}
return server;
}
}

15
myvc-start.js
View File

@ -1,7 +1,8 @@
const MyVC = require('./myvc');
const Container = require('./docker').Container;
const Server = require('./server/server');
const Command = require('./lib/command');
const Container = require('./lib/docker').Container;
const Server = require('./lib/server');
const Run = require('./myvc-run');
/**
@ -10,12 +11,10 @@ const Run = require('./myvc-run');
* mind that when you do not rebuild the docker you may be using an outdated
* version of it.
*/
class Start {
get usage() {
return {
description: 'Start local database server container'
};
}
class Start extends Command {
static usage = {
description: 'Start local database server container'
};
async run(myvc, opts) {
const ct = new Container(opts.code);

48
myvc-version.js
View File

@ -1,38 +1,34 @@
const MyVC = require('./myvc');
const Command = require('./lib/command');
const fs = require('fs-extra');
/**
* Creates a new version.
*/
class Version {
get usage() {
return {
description: 'Creates a new version',
params: {
name: 'Name for the new version'
},
operand: 'name'
};
}
class Version extends Command {
static usage = {
description: 'Creates a new version',
params: {
name: 'Name for the new version'
},
operand: 'name'
};
get localOpts() {
return {
alias: {
name: 'n'
},
string: [
'name'
],
default: {
remote: 'production'
}
};
}
static localOpts = {
alias: {
name: 'n'
},
string: [
'name'
],
default: {
remote: 'production'
}
};
async run(myvc, opts) {
let newVersionDir;
const verionsDir =`${opts.myvcDir}/versions`;
// Fetch last version number
@ -77,7 +73,7 @@ class Version {
let versionName = opts.name;
const versionNames = new Set();
const versionDirs = await fs.readdir(verionsDir);
const versionDirs = await fs.readdir(opts.versionsDir);
for (const versionDir of versionDirs) {
const dirVersion = myvc.parseVersionDir(versionDir);
if (!dirVersion) continue;
@ -107,7 +103,7 @@ class Version {
// Create version
const versionFolder = `${newVersion}-${versionName}`;
newVersionDir = `${verionsDir}/${versionFolder}`;
newVersionDir = `${opts.versionsDir}/${versionFolder}`;
await conn.query(
`INSERT INTO version

256
myvc.js
View File

@ -9,17 +9,13 @@ const ini = require('ini');
const path = require('path');
const mysql = require('mysql2/promise');
const nodegit = require('nodegit');
const camelToSnake = require('./lib').camelToSnake;
const docker = require('./docker');
const camelToSnake = require('./lib/util').camelToSnake;
const docker = require('./lib/docker');
const Command = require('./lib/command');
class MyVC {
async run(command) {
console.log(
'MyVC (MySQL Version Control)'.green,
`v${packageJson.version}`.magenta
);
const usage = {
get usage() {
return {
description: 'Utility for database versioning',
params: {
remote: 'Name of remote to use',
@ -30,7 +26,10 @@ class MyVC {
help: 'Display this help message'
}
};
const baseOpts = {
}
get localOpts() {
return {
alias: {
remote: 'r',
workspace: 'w',
@ -48,6 +47,15 @@ class MyVC {
workspace: process.cwd()
}
};
}
async run(CommandClass) {
console.log(
'MyVC (MySQL Version Control)'.green,
`v${packageJson.version}`.magenta
);
const baseOpts = this.localOpts;
const opts = this.getopts(baseOpts);
if (opts.debug) {
@ -60,37 +68,28 @@ class MyVC {
try {
const commandName = opts._[0];
if (!command && commandName) {
const commands = [
'init',
'pull',
'push',
'version',
'clean',
'dump',
'fixtures',
'start',
'run'
];
if (!CommandClass && commandName) {
if (!/^[a-z]+$/.test(commandName))
throw new Error (`Invalid command name '${commandName}'`);
if (commands.indexOf(commandName) == -1)
const commandFile = path.join(__dirname, `myvc-${commandName}.js`);
if (!await fs.pathExists(commandFile))
throw new Error (`Unknown command '${commandName}'`);
const Klass = require(`./myvc-${commandName}`);
command = new Klass();
CommandClass = require(commandFile);
}
if (!command) {
this.showHelp(baseOpts, usage);
if (!CommandClass) {
this.showHelp(baseOpts, this.usage);
process.exit(0);
}
const allOpts = Object.assign({}, baseOpts);
if (command.localOpts)
for (const key in command.localOpts) {
if (CommandClass.localOpts)
for (const key in CommandClass.localOpts) {
const baseValue = baseOpts[key];
const cmdValue = command.localOpts[key];
const cmdValue = CommandClass.localOpts[key];
if (Array.isArray(baseValue))
allOpts[key] = baseValue.concat(cmdValue);
else if (typeof baseValue == 'object')
@ -104,7 +103,7 @@ class MyVC {
console.log('Command options:'.magenta, commandOpts);
Object.assign(opts, commandOpts);
const operandToOpt = command.usage.operand;
const operandToOpt = CommandClass.usage.operand;
if (opts._.length >= 2 && operandToOpt)
opts[operandToOpt] = opts._[1];
@ -112,7 +111,7 @@ class MyVC {
console.log('Final options:'.magenta, opts);
if (opts.help) {
this.showHelp(command.localOpts, command.usage, commandName);
this.showHelp(CommandClass.localOpts, CommandClass.usage, commandName);
process.exit(0);
}
@ -151,8 +150,7 @@ class MyVC {
parameter('Remote:', opts.remote || 'local');
await this.load(opts);
command.opts = opts;
await command.run(this, opts);
await this.runCommand(CommandClass, opts);
await this.unload();
} catch (err) {
if (err.name == 'Error' && !opts.debug) {
@ -171,10 +169,16 @@ class MyVC {
process.exit();
}
async runCommand(CommandClass, opts) {
const command = new CommandClass();
command.opts = opts;
await command.run(this, opts);
}
async load(opts) {
// Configuration file
const config = require(`${__dirname}/myvc.default.yml`);
const config = require(`${__dirname}/assets/myvc.default.yml`);
const configFile = 'myvc.config.yml';
const configPath = path.join(opts.workspace, configFile);
@ -187,9 +191,13 @@ class MyVC {
if (!opts.myvcDir)
opts.myvcDir = path.join(opts.workspace, opts.subdir || '');
opts.routinesDir = path.join(opts.myvcDir, 'routines');
opts.versionsDir = path.join(opts.myvcDir, 'versions');
opts.dumpDir = path.join(opts.myvcDir, 'dump');
// Database configuration
let iniDir = __dirname;
let iniDir = path.join(__dirname, 'assets');
let iniFile = 'db.ini';
if (opts.remote) {
@ -277,7 +285,7 @@ class MyVC {
if (!res.tableExists) {
const structure = await fs.readFile(
`${__dirname}/structure.sql`, 'utf8');
`${__dirname}/assets/structure.sql`, 'utf8');
await conn.query(structure);
}
}
@ -402,32 +410,8 @@ class MyVC {
});
}
async cachedChanges() {
const dumpDir = path.join(this.opts.myvcDir, 'dump');
const dumpChanges = path.join(dumpDir, '.changes');
if (!await fs.pathExists(dumpChanges))
return null;
const readline = require('readline');
const rl = readline.createInterface({
input: fs.createReadStream(dumpChanges),
//output: process.stdout,
console: false
});
const changes = [];
for await (const line of rl) {
changes.push({
mark: line.charAt(0),
path: line.substring(1)
});
}
return changes;
}
async initDump(dumpFile) {
const dumpDir = path.join(this.opts.myvcDir, 'dump');
const dumpDir = this.opts.dumpDir;
if (!await fs.pathExists(dumpDir))
await fs.mkdir(dumpDir);
@ -445,12 +429,21 @@ class MyVC {
return dumpStream;
}
async dumpFixtures(dumpStream, tables) {
async dumpFixtures(dumpStream, tables, replace) {
const fixturesArgs = [
'--no-create-info',
'--skip-triggers',
'--insert-ignore'
'--skip-extended-insert',
'--skip-disable-keys',
'--skip-add-locks',
'--skip-set-charset',
'--skip-comments',
'--skip-tz-utc'
];
if (replace)
fixturesArgs.push('--replace');
for (const schema in tables) {
const escapedSchema = '`'+ schema.replace('`', '``') +'`';
await dumpStream.write(
@ -512,6 +505,139 @@ class MyVC {
}
}
}
/**
* Executes an SQL script.
*
* @param {Connection} conn MySQL connection object
* @returns {Array<Result>} The resultset
*/
async queryFromFile(conn, file) {
const sql = await fs.readFile(file, 'utf8');
return await this.multiQuery(conn, sql);
}
/**
* Executes a multi-query string.
*
* @param {Connection} conn MySQL connection object
* @param {String} sql SQL multi-query string
* @returns {Array<Result>} The resultset
*/
async multiQuery(conn, sql) {
let results = [];
const stmts = this.querySplit(sql);
for (const stmt of stmts)
results = results.concat(await conn.query(stmt));
return results;
}
/**
* Splits an SQL muti-query into a single-query array, it does an small
* parse to correctly handle the DELIMITER statement.
*
* @param {Array<String>} stmts The splitted SQL statements
*/
querySplit(sql) {
const stmts = [];
let i,
char,
token,
escaped,
stmtStart;
let delimiter = ';';
const delimiterRe = /\s*delimiter\s+(\S+)[^\S\r\n]*(?:\r?\n|\r|$)/yi;
function begins(str) {
let j;
for (j = 0; j < str.length; j++)
if (sql[i + j] != str[j])
return false;
i += j;
return true;
}
for (i = 0; i < sql.length;) {
stmtStart = i;
delimiterRe.lastIndex = i;
const match = sql.match(delimiterRe);
if (match) {
delimiter = match[1];
i += match[0].length;
continue;
}
let delimiterFound = false;
while (i < sql.length) {
char = sql[i];
if (token) {
if (!escaped && begins(token.end))
token = null;
else {
escaped = !escaped && token.escape(char);
i++;
}
} else {
delimiterFound = begins(delimiter);
if (delimiterFound) break;
const tok = tokenIndex.get(char);
if (tok && begins(tok.start))
token = tok;
else
i++;
}
}
let len = i - stmtStart;
if (delimiterFound) len -= delimiter.length;
const stmt = sql.substr(stmtStart, len);
if (!/^\s*$/.test(stmt))
stmts.push(stmt);
}
return stmts;
}
}
const tokens = {
string: {
start: '\'',
end: '\'',
escape: char => char == '\'' || char == '\\'
},
quotedString: {
start: '"',
end: '"',
escape: char => char == '"' || char == '\\'
},
id: {
start: '`',
end: '`',
escape: char => char == '`'
},
multiComment: {
start: '/*',
end: '*/',
escape: () => false
},
singleComment: {
start: '-- ',
end: '\n',
escape: () => false
}
};
const tokenIndex = new Map();
for (const tokenId in tokens) {
const token = tokens[tokenId];
tokenIndex.set(token.start[0], token);
}
module.exports = MyVC;

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "myvc",
"version": "1.4.19",
"version": "1.5.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "myvc",
"version": "1.4.19",
"version": "1.5.0",
"license": "GPL-3.0",
"dependencies": {
"@sqltools/formatter": "^1.2.3",

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "myvc",
"version": "1.4.19",
"version": "1.5.0",
"author": "Verdnatura Levante SL",
"description": "MySQL Version Control",
"license": "GPL-3.0",

5
server/Dockerfile.client
View File

@ -7,5 +7,8 @@ RUN apt-get update \
libmariadb3 \
&& rm -rf /var/lib/apt/lists/*
COPY myvc-dump.sh /usr/local/bin/
COPY \
server/docker-dump.sh \
server/docker-fixtures.sh \
/usr/local/bin/
WORKDIR /workspace

12
server/Dockerfile.dump
View File

@ -7,21 +7,9 @@ COPY \
dump/beforeDump.sql \
dump/afterDump.sql \
dump/
COPY myvc.config.yml \
./
RUN gosu mysql docker-init.sh
COPY routines routines
COPY versions versions
COPY \
dump/fixtures.sql \
dump/.changes \
dump/
ARG STAMP=unknown
RUN gosu mysql docker-push.sh
RUN echo "[LOG] Import finished." \
&& rm -rf /workspace

31
server/Dockerfile.server
View File

@ -3,44 +3,15 @@ FROM myvc/base
USER root
ENV MYSQL_ROOT_PASSWORD root
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
curl \
&& curl -sL https://deb.nodesource.com/setup_14.x | bash - \
&& apt-get install -y --no-install-recommends \
nodejs \
&& rm -rf /var/lib/apt/lists/*
RUN mkdir /mysql-data \
&& chown -R mysql:mysql /mysql-data
WORKDIR /myvc
COPY \
package.json \
./
RUN npm install --only=prod
COPY \
structure.sql \
myvc.js \
myvc-push.js \
lib.js \
docker.js \
myvc.default.yml \
db.ini \
./
COPY exporters exporters
RUN ln -s /myvc/myvc.js /usr/local/bin/myvc
WORKDIR /workspace
COPY server/docker.cnf /etc/mysql/conf.d/
COPY \
server/docker-init.sh \
server/docker-push.sh \
server/docker-dump.sh \
server/docker-import.sh \
server/docker-start.sh \
/usr/local/bin/

9
server/docker-dump.sh
View File

@ -1,9 +1,4 @@
#!/bin/bash
FILE="$1.sql"
#if [ -f "$FILE" ]; then
echo "[LOG] -> Importing $FILE"
export MYSQL_PWD=root
mysql -u root --default-character-set=utf8 --comments -f < "$FILE"
#fi
# FIXME: It can corrupt data
mysqldump $@ | sed 's/ AUTO_INCREMENT=[0-9]* //g'

4
server/docker-fixtures.sh Executable file
View File

@ -0,0 +1,4 @@
#!/bin/bash
# FIXME: It can corrupt data
mysqldump $@ | sed -E 's/(VALUES |\),)\(/\1\n\t\(/g'

6
server/docker-import.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
FILE="$1.sql"
echo "[LOG] -> Importing $FILE"
export MYSQL_PWD=root
mysql -u root --default-character-set=utf8 --comments -f < "$FILE"

6
server/docker-init.sh
View File

@ -13,8 +13,8 @@ docker_temp_server_start "$CMD"
docker_setup_db
docker_process_init_files /docker-entrypoint-initdb.d/*
docker-dump.sh dump/beforeDump
docker-dump.sh dump/.dump
docker-dump.sh dump/afterDump
docker-import.sh dump/beforeDump
docker-import.sh dump/.dump
docker-import.sh dump/afterDump
docker_temp_server_stop

12
server/docker-push.sh
View File

@ -1,12 +0,0 @@
#!/bin/bash
. /usr/local/bin/docker-entrypoint.sh
CMD=mysqld
docker_setup_env "$CMD"
docker_temp_server_start "$CMD"
myvc push --socket --commit
docker-dump.sh dump/fixtures
docker_temp_server_stop

2
template/package.json
View File

@ -8,6 +8,6 @@
"type": "git"
},
"dependencies": {
"myvc": "^1.4.19"
"myvc": "^1.5.0"
}
}