#3936 #3935 #3934

Commit not found fix, SHAsums not used anymore, clean old versions

README.md

@@ -95,7 +95,7 @@ From now on, you can use the project as if it were a standard git repository
 desired remote.
 
 ```text
-$ myvc push [<remote>]
+$ myvc push [<remote>] [--save-commit]
 ```
 
 ### Routines
@@ -164,10 +164,10 @@ $ myvc init
 Incorporates database routine changes into workspace.
 
 ```text
-$ myvc pull [remote] [-f|--force] [-c|--checkout] [-u|--update]
+$ myvc pull [remote] [-f|--force] [-c|--checkout] [-u|--update] [-s|save-sums]
 ```
 
-When *checkout* option is provided, it does the following before export:
+When *--checkout* option is provided, it does the following before export:
 
 1. Get the last database push commit (saved in versioning tables).
 2. Creates and checkout to a new branch based in database commit.
@@ -177,16 +177,21 @@ When *checkout* option is provided, it does the following before export:
 Applies versions and routine changes into database.
 
 ```text
-$ myvc push [<remote>] [-f|--force]
+$ myvc push [<remote>] [-f|--force] [-c|--save-commit] [-s|save-sums]
 ```
 
+Commit is saved into database only if *--save-commit* option is provided, it
+prevents from accidentally saving local commits into shared servers, causing 
+subsequent pushes from other clients to fail because they can't get that 
+commit from the git tree in order to get differences.
+
 ### version
 
 Creates a new version folder, when name is not specified it generates a random 
 name mixing a color with a plant name.
 
 ```text
-$ myvc version [<name>]
+$ myvc version [<name>] [-c|--no-clean]
 ```
 
 ## Local server commands

lib.js

@@ -24,7 +24,7 @@ class Exporter {
         this.attrs = require(`${templateDir}.js`);
     }
 
-    async export(exportDir, schema, newSums, oldSums, update) {
+    async export(exportDir, schema, update, saveSum) {
         const res = await this.query(schema);
         if (!res.length) return; 
 
@@ -45,15 +45,32 @@ class Exporter {
                 await fs.remove(`${routineDir}/${routine}.sql`);
         }
 
+        const engine = this.engine;
+
         for (const params of res) {
             const routineName = params.name;
             const sql = this.format(params);
             const routineFile = `${routineDir}/${routineName}.sql`;
 
-            const shaSum = this.engine.shaSum(sql);
+            const oldSum = engine.getShaSum(routineName);
-            newSums[routineName] = shaSum;
+            if (oldSum || saveSum) {
+                const shaSum = engine.shaSum(sql);
+                if (oldSum !== shaSum) {
+                    engine.setShaSum(
+                        this.objectType, schema, routineName, shaSum);
+                    update = true;
+                }
+            } else if (params.modified && engine.lastPull) {
+                if (params.modified > engine.lastPull)
+                    update = true;
+            } else if (await fs.pathExists(routineFile)) {
+                const currentSql = await fs.readFile(routineFile, 'utf8');
+                if (sql != currentSql)
+                    update = true;
+            } else
+                update = true;
 
-            if (oldSums[routineName] !== shaSum || update)
+            if (update)
                 await fs.writeFile(routineFile, sql);
         }
     }
@@ -104,16 +121,26 @@ class Exporter {
 class ExporterEngine {
     constructor(conn, myvcDir) {
         this.conn = conn;
-        this.shaFile = `${myvcDir}/.shasums.json`;
+        this.pullFile = `${myvcDir}/.pullinfo.json`;
         this.exporters = [];
         this.exporterMap = {};
     }
 
     async init () {
-        if (await fs.pathExists(this.shaFile))
+        if (await fs.pathExists(this.pullFile)) {
-            this.shaSums = JSON.parse(await fs.readFile(this.shaFile, 'utf8'));
+            this.pullInfo = JSON.parse(await fs.readFile(this.pullFile, 'utf8'));
-        else
+            const lastPull = this.pullInfo.lastPull;
-            this.shaSums = {};
+            if (lastPull)
+                this.pullInfo.lastPull = new Date(lastPull);
+        } else
+            this.pullInfo = {
+                lastPull: null,
+                shaSums: {}
+            };
+
+        this.shaSums = this.pullInfo.shaSums;
+        this.lastPull = this.pullInfo.lastPull;
+        this.infoChanged = false;
 
         const types = [
             'function',
@@ -150,6 +177,14 @@ class ExporterEngine {
             .digest('hex');
     }
 
+    getShaSum(type, schema, name) {
+        try {
+            return this.shaSums[schema][type][name];
+        } catch (e) {};
+
+        return null;
+    }
+
     setShaSum(type, schema, name, shaSum) {
         if (!shaSum) {
             this.deleteShaSum(type, schema, name);
@@ -162,17 +197,32 @@ class ExporterEngine {
         if (!shaSums[schema][type])
             shaSums[schema][type] = {};
         shaSums[schema][type][name] = shaSum;
+        this.infoChanged = true;
     }
 
     deleteShaSum(type, schema, name) {
         try {
             delete this.shaSums[schema][type][name];
+            this.infoChanged = true;
         } catch (e) {};
     }
 
-    async saveShaSums() {
+    deleteSchemaSums(schema) {
-        await fs.writeFile(this.shaFile,
+        delete this.shaSums[schema];
-            JSON.stringify(this.shaSums, null, '  '));
+        this.infoChanged = true;
+    }
+
+    async refreshPullDate() {
+        const [[row]] = await this.conn.query(`SELECT NOW() now`);
+        this.pullInfo.lastPull = row.now;
+        this.infoChanged = true;
+    }
+
+    async saveInfo() {
+        if (!this.infoChanged) return;
+        await fs.writeFile(this.pullFile,
+            JSON.stringify(this.pullInfo, null, '  '));
+        this.infoChanged = false;
     }
 }
 

myvc-pull.js

@@ -10,7 +10,8 @@ class Pull {
             params: {
                 force: 'Do it even if there are local changes',
                 checkout: 'Move to same database commit before pull',
-                update: 'Update routine file even is shasum is the same'
+                updateAll: 'Update all routines',
+                saveSums: 'Save SHA sums of all objects'
             },
             operand: 'remote'
         };
@@ -21,12 +22,14 @@ class Pull {
             alias: {
                 force: 'f',
                 checkout: 'c',
-                update: 'u'
+                updateAll: 'u',
+                saveSums: 's'
             },
             boolean: [
                 'force',
                 'checkout',
-                'update'
+                'updateAll',
+                'saveSums'
             ]
         };
     }
@@ -102,7 +105,7 @@ class Pull {
 
         for (const schema in shaSums) {
             if (!await fs.pathExists(`${exportDir}/${schema}`))
-                delete shaSums[schema];
+                engine.deleteSchemaSums(schema);
         }
 
         // Export objects to SQL files
@@ -111,19 +114,14 @@ class Pull {
             let schemaDir = `${exportDir}/${schema}`;
             if (!await fs.pathExists(schemaDir))
                 await fs.mkdir(schemaDir);
-            if (!shaSums[schema])
-                shaSums[schema] = {};
-            const sums = shaSums[schema];
 
-            for (const exporter of engine.exporters) {
+            for (const exporter of engine.exporters)
-                const type = exporter.objectType;
+                await exporter.export(exportDir,
-                const oldSums = sums[type] || {};
+                    schema, opts.update, opts.saveSums);
-                sums[type] = {};
-                await exporter.export(exportDir, schema, sums[type], oldSums, opts.update);
-            }
         }
 
-        await engine.saveShaSums();
+        await engine.refreshPullDate();
+        await engine.saveInfo();
     }
 }
 

myvc-push.js

@@ -12,7 +12,9 @@ class Push {
         return {
             description: 'Apply changes into database',
             params: {
-                force: 'Answer yes to all questions'
+                force: 'Answer yes to all questions',
+                saveCommit: 'Wether to save the commit SHA into database',
+                saveSums: 'Save SHA sums of pushed objects'
             },
             operand: 'remote'
         };
@@ -21,10 +23,14 @@ class Push {
     get localOpts() {
         return {
             alias: {
-                force: 'f'
+                force: 'f',
+                saveCommit: 'c',
+                saveSums: 's'
             },
             boolean: [
-                'force'
+                'force',
+                'saveCommit',
+                'saveSums'
             ]
         };
     }
@@ -33,6 +39,9 @@ class Push {
         const conn = await myvc.dbConnect();
         this.conn = conn;
 
+        if (opts.saveCommit == null && opts.remote == 'local')
+            opts.saveCommit = true;
+
         // Obtain exclusive lock
 
         const [[row]] = await conn.query(
@@ -142,16 +151,16 @@ class Push {
                 if (versionDir == 'README.md')
                     continue;
 
-                const match = versionDir.match(/^([0-9]+)-([a-zA-Z0-9]+)?$/);
+                const dirVersion = myvc.parseVersionDir(versionDir);
-                if (!match) {
+                if (!dirVersion) {
                     logVersion('[?????]'.yellow, versionDir,
                         `Wrong directory name.`
                     );
                     continue;
                 }
 
-                const versionNumber = match[1];
+                const versionNumber = dirVersion.number;
-                const versionName = match[2];
+                const versionName = dirVersion.name;
 
                 if (versionNumber.length != version.number.length) {
                     logVersion('[*****]'.gray, versionDir,
@@ -283,7 +292,7 @@ class Push {
         await engine.init();
 
         async function finalize() {
-            await engine.saveShaSums();
+            await engine.saveInfo();
 
             if (routines.length) {
                 await conn.query('FLUSH PRIVILEGES');
@@ -303,6 +312,7 @@ class Push {
             if (exists)
                 newSql = await fs.readFile(fullPath, 'utf8');
             const oldSql = await engine.fetchRoutine(type, schema, name);
+            const oldSum = engine.getShaSum(type, schema, name);
             const isEqual = newSql == oldSql;
 
             let actionMsg;
@@ -336,6 +346,7 @@ class Push {
                         );
                     }
 
+                    if (opts.saveSums || oldSum)
                         await engine.fetchShaSum(type, schema, name);
                 } else {
                     const escapedName =
@@ -366,7 +377,7 @@ class Push {
         } else
             console.log(` -> No routines changed.`);
 
-        if (gitExists) {
+        if (gitExists && opts.saveCommit) {
             const repo = await nodegit.Repository.open(this.opts.workspace);
             const head = await repo.getHeadCommit();
 

myvc-version.js

@@ -10,7 +10,8 @@ class Version {
         return {
             description: 'Creates a new version',
             params: {
-                name: 'Name for the new version'
+                name: 'Name for the new version',
+                noClean: 'Do not clean old versions'
             },
             operand: 'name'
         };
@@ -19,11 +20,15 @@ class Version {
     get localOpts() {
         return {
             alias: {
-                name: 'n'
+                name: 'n',
+                noClean: 'c'
             },
             string: [
                 'name'
             ],
+            boolean: [
+                'noClean'
+            ],
             default: {
                 remote: 'production'
             }
@@ -31,7 +36,9 @@ class Version {
     }
 
     async run(myvc, opts) {
-        let versionDir;
+        let newVersionDir;
+        const verionsDir =`${opts.myvcDir}/versions`;
+        const oldVersions = [];
 
         // Fetch last version number
 
@@ -74,14 +81,16 @@ class Version {
             // Get version name
 
             let versionName = opts.name;
-            const verionsDir =`${opts.myvcDir}/versions`;
 
             const versionNames = new Set();
             const versionDirs = await fs.readdir(verionsDir);
-            for (const versionNameDir of versionDirs) {
+            for (const versionDir of versionDirs) {
-                const split = versionNameDir.split('-');
+                const dirVersion = myvc.parseVersionDir(versionDir);
-                const versionName = split[1];
+                if (!dirVersion) continue;
-                if (versionName) versionNames.add(versionName);
+                versionNames.add(dirVersion.name);
+
+                if (parseInt(dirVersion.number) < parseInt(number))
+                    oldVersions.push(versionDir);
             }
 
             if (!versionName) {
@@ -107,7 +116,7 @@ class Version {
             // Create version
 
             const versionFolder = `${newVersion}-${versionName}`;
-            versionDir = `${verionsDir}/${versionFolder}`;
+            newVersionDir = `${verionsDir}/${versionFolder}`;
 
             await conn.query(
                 `INSERT INTO version
@@ -117,9 +126,9 @@ class Version {
                         lastNumber = VALUES(lastNumber)`,
                 [opts.code, newVersion]
             );
-            await fs.mkdir(versionDir);
+            await fs.mkdir(newVersionDir);
             await fs.writeFile(
-                `${versionDir}/00-firstScript.sql`,
+                `${newVersionDir}/00-firstScript.sql`,
                 '-- Place your SQL code here\n'
             );
             console.log(`New version created: ${versionFolder}`);
@@ -127,10 +136,23 @@ class Version {
             await conn.query('COMMIT');
         } catch (err) {
             await conn.query('ROLLBACK');
-            if (versionDir && await fs.pathExists(versionDir))
+            if (newVersionDir && await fs.pathExists(newVersionDir))
-                await fs.remove(versionDir, {recursive: true});
+                await fs.remove(newVersionDir, {recursive: true});
             throw err;
         }
+
+        // Remove old versions
+
+        if (opts.maxOldVersions && !opts.noClean
+        && oldVersions.length > opts.maxOldVersions) {
+            oldVersions.splice(-opts.maxOldVersions);
+
+            for (const oldVersion of oldVersions)
+                await fs.remove(`${verionsDir}/${oldVersion}`,
+                    {recursive: true});
+
+            console.log(`Old versions deleted: ${oldVersions.length}`);
+        }
     }
 }
 

myvc.default.yml

@@ -1,5 +1,6 @@
 versionSchema: myvc
 versionDigits: 5
+maxOldVersions: 30
 schemas:
   - myvc
 fixtures:

myvc.js

@@ -33,7 +33,7 @@ class MyVC {
             alias: {
                 remote: 'r',
                 workspace: 'w',
-                socket: 's',
+                socket: 'k',
                 debug: 'd',
                 version: 'v',
                 help: 'h'
@@ -275,6 +275,15 @@ class MyVC {
         return version;
     }
 
+    parseVersionDir(versionDir) {
+        const match = versionDir.match(/^([0-9]+)-([a-zA-Z0-9]+)?$/);
+        if (!match) return null;
+        return {
+            number: match[1],
+            name: match[2]
+        };
+    }
+
     async changedRoutines(commitSha) {
         const repo = await this.openRepo();
         const changes = [];
@@ -302,7 +311,20 @@ class MyVC {
         const head = await repo.getHeadCommit();
 
         if (head && commitSha) {
-            const commit = await repo.getCommit(commitSha);
+            let commit;
+            try {
+                await repo.fetchAll();
+            } catch(err) {
+                console.warn(err.message.yellow);
+            }
+            try {
+                commit = await repo.getCommit(commitSha);
+            } catch (err) {
+                if (err.errorFunction == 'Commit.lookup')
+                    throw new Error(`Commit id (${commitSha}) not found, you may have to run 'git fetch' first`);
+                else
+                    throw err;
+            }
             const commitTree = await commit.getTree();
 
             const headTree = await head.getTree();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "myvc",
-  "version": "1.3.13",
+  "version": "1.4.0",
   "author": "Verdnatura Levante SL",
   "description": "MySQL Version Control",
   "license": "GPL-3.0",

server/Dockerfile

@@ -26,7 +26,6 @@ WORKDIR /myvc
 
 COPY \
     package.json \
-    package-lock.json \
     ./
 RUN npm install --only=prod