verdnatura/myt
verdnatura
/
myt
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

SHA sum, export definer, don't stop on errors

This commit is contained in:
Juan Ferrer 2021-10-22 16:11:03 +02:00
parent 1566847ecd
commit 99461688cd
16 changed files with 3507 additions and 1568 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -44,7 +44,7 @@ otherwise indicated, the default remote is *local*.
Database versioning commands:
* **init**: Initialize an empty workspace.
* **pull**: Export database routines into workspace.
* **pull**: Incorporates database routines changes into workspace.
* **push**: Apply changes into database.
Local server management commands:
@ -163,4 +163,5 @@ Create a lock during push to avoid collisions.
* [Git](https://git-scm.com/)
* [nodejs](https://nodejs.org/)
* [NodeGit](https://www.nodegit.org/)
* [docker](https://www.docker.com/)

4
exporters/event.ejs
View File

@ -1,6 +1,6 @@
DROP EVENT IF EXISTS `<%- schema %>`.`<%- name %>`;
DROP EVENT IF EXISTS <%- schema %>.<%- name %>;
DELIMITER $$
CREATE DEFINER=`root`@`%` EVENT `<%- schema %>`.`<%- name %>`
CREATE DEFINER=<%- definer %> EVENT <%- schema %>.<%- name %>
ON SCHEDULE EVERY <%- intervalValue %> <%- intervalField %>
ON COMPLETION <%- onCompletion %>
<% if (status == 'ENABLED') { %>ENABLE<% } else { %>DISABLE<% } %>

31
exporters/event.sql
View File

@ -1,17 +1,18 @@
SELECT
EVENT_NAME `name`,
DEFINER `definer`,
EVENT_DEFINITION `body`,
EVENT_TYPE `type`,
EXECUTE_AT `execute_at`,
INTERVAL_VALUE `intervalValue`,
INTERVAL_FIELD `intervalField`,
STARTS `starts`,
ENDS `ends`,
STATUS `status`,
ON_COMPLETION `onCompletion`,
EVENT_COMMENT `comment`,
LAST_ALTERED `modified`
FROM information_schema.EVENTS
WHERE EVENT_SCHEMA = ?
`EVENT_NAME` AS `name`,
`DEFINER` AS `definer`,
`EVENT_DEFINITION` AS `body`,
`EVENT_TYPE` AS `type`,
`EXECUTE_AT` AS `execute_at`,
`INTERVAL_VALUE` AS `intervalValue`,
`INTERVAL_FIELD` AS `intervalField`,
`STARTS` AS `starts`,
`ENDS` AS `ends`,
`STATUS` AS `status`,
`ON_COMPLETION` AS `onCompletion`,
`EVENT_COMMENT` AS `comment`,
`LAST_ALTERED` AS `modified`
FROM `information_schema`.`EVENTS`
WHERE `EVENT_SCHEMA` = ?
ORDER BY `name`

4
exporters/function.ejs
View File

@ -1,6 +1,6 @@
DROP FUNCTION IF EXISTS `<%- schema %>`.`<%- name %>`;
DROP FUNCTION IF EXISTS <%- schema %>.<%- name %>;
DELIMITER $$
CREATE DEFINER='root'@'%' FUNCTION `<%- schema %>`.`<%- name %>`(<%- paramList %>)
CREATE DEFINER=<%- definer %> FUNCTION <%- schema %>.<%- name %>(<%- paramList %>)
RETURNS <%- returns %>
<% if (isDeterministic != 'NO') { %>DETERMINISTIC<% } else { %>NOT DETERMINISTIC<% } %>
<%- body %>$$

7
exporters/function.sql
View File

@ -2,10 +2,11 @@
SELECT
`name`,
`definer`,
`param_list` paramList,
`param_list` AS `paramList`,
`returns`,
`is_deterministic` isDeterministic,
`is_deterministic` AS `isDeterministic`,
`body`,
`modified`
FROM mysql.proc
FROM `mysql`.`proc`
WHERE `db` = ? AND `type` = 'FUNCTION'
ORDER BY `name`

4
exporters/procedure.ejs
View File

@ -1,5 +1,5 @@
DROP PROCEDURE IF EXISTS `<%- schema %>`.`<%- name %>`;
DROP PROCEDURE IF EXISTS <%- schema %>.<%- name %>;
DELIMITER $$
CREATE DEFINER='root'@'%' PROCEDURE `<%- schema %>`.`<%- name %>`(<%- paramList %>)
CREATE DEFINER=<%- definer %> PROCEDURE <%- schema %>.<%- name %>(<%- paramList %>)
<%- body %>$$
DELIMITER ;

7
exporters/procedure.sql
View File

@ -2,8 +2,9 @@
SELECT
`name`,
`definer`,
`param_list` paramList,
`param_list` AS `paramList`,
`body`,
`modified`
FROM mysql.proc
WHERE db = ? AND type = 'PROCEDURE'
FROM `mysql`.`proc`
WHERE `db` = ? AND `type` = 'PROCEDURE'
ORDER BY `name`

4
exporters/trigger.ejs
View File

@ -1,6 +1,6 @@
DROP TRIGGER IF EXISTS `<%- schema %>`.`<%- name %>`;
DROP TRIGGER IF EXISTS <%- schema %>.<%- name %>;
DELIMITER $$
CREATE DEFINER=`root`@`%` TRIGGER `<%- schema %>`.`<%- name %>`
CREATE DEFINER=<%- definer %> TRIGGER <%- schema %>.<%- name %>
<%- actionTiming %> <%- actionType %> ON `<%- table %>`
FOR EACH ROW
<%- body %>$$

19
exporters/trigger.sql
View File

@ -1,11 +1,12 @@
SELECT
TRIGGER_NAME `name`,
DEFINER `definer`,
ACTION_TIMING `actionTiming`,
EVENT_MANIPULATION `actionType`,
EVENT_OBJECT_TABLE `table`,
ACTION_STATEMENT `body`,
CREATED `modified`
FROM information_schema.TRIGGERS
WHERE TRIGGER_SCHEMA = ?
`TRIGGER_NAME` AS `name`,
`DEFINER` AS `definer`,
`ACTION_TIMING` AS `actionTiming`,
`EVENT_MANIPULATION` AS `actionType`,
`EVENT_OBJECT_TABLE` AS `table`,
`ACTION_STATEMENT` AS `body`,
`CREATED` AS `modified`
FROM `information_schema`.`TRIGGERS`
WHERE `TRIGGER_SCHEMA` = ?
ORDER BY `name`

4
exporters/view.ejs
View File

@ -1,5 +1,5 @@
CREATE OR REPLACE DEFINER = `root`@`%`
CREATE OR REPLACE DEFINER=<%- definer %>
SQL SECURITY <%- securityType %>
VIEW `<%- schema %>`.`<%- name %>`
VIEW <%- schema %>.<%- name %>
AS <%- definition %><% if (checkOption != 'NONE') { %>
WITH CASCADED CHECK OPTION<% } %>

17
exporters/view.sql
View File

@ -1,10 +1,11 @@
SELECT
TABLE_NAME `name`,
VIEW_DEFINITION `definition`,
CHECK_OPTION `checkOption`,
IS_UPDATABLE `isUpdatable`,
DEFINER `definer`,
SECURITY_TYPE `securityType`
FROM information_schema.VIEWS
WHERE TABLE_SCHEMA = ?
`TABLE_NAME` AS `name`,
`VIEW_DEFINITION` AS `definition`,
`CHECK_OPTION` AS `checkOption`,
`IS_UPDATABLE` AS `isUpdatable`,
`DEFINER` AS `definer`,
`SECURITY_TYPE` AS `securityType`
FROM `information_schema`.`VIEWS`
WHERE `TABLE_SCHEMA` = ?
ORDER BY `name`

132
myvc-pull.js
View File

@ -2,27 +2,72 @@
const MyVC = require('./myvc');
const fs = require('fs-extra');
const ejs = require('ejs');
const shajs = require('sha.js');
const nodegit = require('nodegit');
class Pull {
async run(myvc, opts) {
const conn = await myvc.dbConnect();
/*
const version = await myvc.fetchDbVersion();
let repo;
if (version && version.gitCommit) {
console.log(version);
repo = await nodegit.Repository.open(opts.workspace);
const commit = await repo.getCommit(version.gitCommit);
const now = parseInt(new Date().getTime() / 1000);
const branch = await nodegit.Branch.create(repo,
`myvc_${now}`, commit, () => {});
await repo.checkoutBranch(branch);
get myOpts() {
return {
alias: {
force: 'f',
checkout: 'c'
}
};
}
return;
*/
async run(myvc, opts) {
const conn = await myvc.dbConnect();
const repo = await myvc.openRepo();
if (!opts.force) {
async function hasChanges(diff) {
if (diff)
for (const patch of await diff.patches()) {
const match = patch
.newFile()
.path()
.match(/^routines\/(.+)\.sql$/);
if (match) return true;
}
return false;
}
// Check for unstaged changes
const unstagedDiff = await myvc.getUnstaged(repo);
if (await hasChanges(unstagedDiff))
throw new Error('You have unstaged changes, save them before pull');
// Check for staged changes
const stagedDiff = await myvc.getStaged(repo);
if (await hasChanges(stagedDiff))
throw new Error('You have staged changes, save them before pull');
}
// Checkout to remote commit
if (opts.checkout) {
const version = await myvc.fetchDbVersion();
if (version && version.gitCommit) {
const now = parseInt(new Date().toJSON());
const branchName = `myvc-pull_${now}`;
console.log(`Creating branch '${branchName}' from database commit.`);
const commit = await repo.getCommit(version.gitCommit);
const branch = await nodegit.Branch.create(repo,
`myvc-pull_${now}`, commit, () => {});
await repo.checkoutBranch(branch);
}
}
// Export routines to SQL files
console.log(`Incorporating routine changes.`);
for (const exporter of exporters)
await exporter.init();
@ -36,26 +81,45 @@ class Pull {
await fs.remove(`${exportDir}/${schema}`, {recursive: true});
}
let shaSums;
const shaFile = `${opts.workspace}/.shasums.json`;
if (await fs.pathExists(shaFile))
shaSums = JSON.parse(await fs.readFile(shaFile, 'utf8'));
else
shaSums = {};
for (const schema of opts.schemas) {
let schemaDir = `${exportDir}/${schema}`;
if (!await fs.pathExists(schemaDir))
await fs.mkdir(schemaDir);
for (const exporter of exporters)
await exporter.export(conn, exportDir, schema);
let schemaSums = shaSums[schema];
if (!schemaSums) schemaSums = shaSums[schema] = {};
for (const exporter of exporters) {
const objectType = exporter.objectType;
let objectSums = schemaSums[objectType];
if (!objectSums) objectSums = schemaSums[objectType] = {};
await exporter.export(conn, exportDir, schema, objectSums);
}
}
await fs.writeFile(shaFile, JSON.stringify(shaSums, null, ' '));
}
}
class Exporter {
constructor(objectName) {
this.objectName = objectName;
this.dstDir = `${objectName}s`;
constructor(objectType) {
this.objectType = objectType;
this.dstDir = `${objectType}s`;
}
async init() {
const templateDir = `${__dirname}/exporters/${this.objectName}`;
const templateDir = `${__dirname}/exporters/${this.objectType}`;
this.query = await fs.readFile(`${templateDir}.sql`, 'utf8');
const templateFile = await fs.readFile(`${templateDir}.ejs`, 'utf8');
@ -65,7 +129,7 @@ class Exporter {
this.formatter = require(`${templateDir}.js`);
}
async export(conn, exportDir, schema) {
async export(conn, exportDir, schema, shaSums) {
const [res] = await conn.query(this.query, [schema]);
if (!res.length) return;
@ -90,17 +154,31 @@ class Exporter {
if (this.formatter)
this.formatter(params, schema)
params.schema = schema;
const routineName = params.name;
const split = params.definer.split('@');
params.schema = conn.escapeId(schema);
params.name = conn.escapeId(routineName, true);
params.definer =
`${conn.escapeId(split[0], true)}@${conn.escapeId(split[1], true)}`;
const sql = this.template(params);
const routineFile = `${routineDir}/${params.name}.sql`;
const routineFile = `${routineDir}/${routineName}.sql`;
const shaSum = shajs('sha256')
.update(JSON.stringify(sql))
.digest('hex');
shaSums[routineName] = shaSum;
let changed = true;
if (await fs.pathExists(routineFile)) {
const currentSql = await fs.readFile(routineFile, 'utf8');
changed = currentSql !== sql;
changed = shaSums[routineName] !== shaSum;;
}
if (changed)
if (changed) {
await fs.writeFile(routineFile, sql);
shaSums[routineName] = shaSum;
}
}
}
}

20
myvc-push.js
View File

@ -177,15 +177,25 @@ class Push {
console.log('', actionMsg.bold, typeMsg.bold, change.fullName);
if (exists)
try {
const scapedSchema = pushConn.escapeId(change.schema, true);
if (exists) {
await pushConn.query(`USE ${scapedSchema}`);
await this.queryFromFile(pushConn, `routines/${change.path}.sql`);
else {
} else {
const escapedName =
conn.escapeId(change.schema, true) + '.' +
conn.escapeId(change.name, true);
scapedSchema + '.' +
pushConn.escapeId(change.name, true);
const query = `DROP ${change.type.name} IF EXISTS ${escapedName}`;
await conn.query(query);
await pushConn.query(query);
}
} catch (err) {
if (err.sqlState)
console.warn('Warning:'.yellow, err.message);
else
throw err;
}
nRoutines++;

49
myvc.js
View File

@ -196,15 +196,9 @@ class MyVC {
}
async changedRoutines(commitSha) {
const {opts} = this;
if (!await fs.pathExists(`${opts.workspace}/.git`))
throw new Error ('Git not initialized');
const repo = await this.openRepo();
const changes = [];
const changesMap = new Map();
const repo = await nodegit.Repository.open(opts.workspace);
const head = await repo.getHeadCommit();
async function pushChanges(diff) {
const patches = await diff.patches();
@ -224,7 +218,7 @@ class MyVC {
}
}
// Committed
const head = await repo.getHeadCommit();
if (head && commitSha) {
const commit = await repo.getCommit(commitSha);
@ -235,15 +229,30 @@ class MyVC {
await pushChanges(diff);
}
// Unstaged
await pushChanges(await this.getUnstaged(repo));
const stagedDiff = await this.getStaged(repo);
if (stagedDiff) await pushChanges(stagedDiff);
return changes.sort((a, b) => {
if (b.mark != a.mark)
return b.mark == '-' ? 1 : -1;
return a.path.localeCompare(b.path);
const diff = await nodegit.Diff.indexToWorkdir(repo, null, {
flags: nodegit.Diff.OPTION.SHOW_UNTRACKED_CONTENT
| nodegit.Diff.OPTION.RECURSE_UNTRACKED_DIRS
});
await pushChanges(diff);
}
// Staged
async openRepo() {
const {opts} = this;
if (!await fs.pathExists(`${opts.workspace}/.git`))
throw new Error ('Git not initialized');
return await nodegit.Repository.open(opts.workspace);
}
async getStaged(repo) {
const head = await repo.getHeadCommit();
try {
const emptyTree = '4b825dc642cb6eb9a060e54bf8d69288fbee4904';
@ -251,15 +260,17 @@ class MyVC {
? head.getTree()
: nodegit.Tree.lookup(repo, emptyTree)
);
const stagedDiff = await nodegit.Diff.treeToIndex(repo, headTree, null);
await pushChanges(stagedDiff);
return await nodegit.Diff.treeToIndex(repo, headTree, null);
} catch (err) {
console.warn('Cannot fetch staged changes:', err.message);
}
}
return changes.sort(
(a, b) => b.mark == '-' && b.mark != a.mark ? 1 : -1
);
async getUnstaged(repo) {
return await nodegit.Diff.indexToWorkdir(repo, null, {
flags: nodegit.Diff.OPTION.SHOW_UNTRACKED_CONTENT
| nodegit.Diff.OPTION.RECURSE_UNTRACKED_DIRS
});
}
async cachedChanges() {

1856
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

6
package.json
View File

@ -1,6 +1,6 @@
{
"name": "myvc",
"version": "1.1.10",
"version": "1.1.11",
"author": "Verdnatura Levante SL",
"description": "MySQL Version Control",
"license": "GPL-3.0",
@ -18,10 +18,10 @@
"ini": "^1.3.8",
"mysql2": "^2.2.5",
"nodegit": "^0.27.0",
"require-yaml": "0.0.1"
"require-yaml": "0.0.1",
"sha.js": "^2.4.11"
},
"main": "index.js",
"devDependencies": {},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},