Gulp updated, packages updated, db docker bugs fixed

gulpfile.js

@@ -1,6 +1,5 @@
 require('require-yaml');
 const gulp = require('gulp');
-const runSequence = require('run-sequence');
 const fs = require('fs-extra');
 const exec = require('child_process').exec;
 const PluginError = require('plugin-error');
@@ -20,7 +19,7 @@ let langs = ['es', 'en'];
 let srcDir = './front';
 let modulesDir = './modules';
 let servicesDir = './services';
-let services = require('./modules.yml');
+let modules = require('./modules.yml');
 
 let wpConfig = require('./webpack.config.yml');
 let buildDir = wpConfig.buildDir;
@@ -38,34 +37,52 @@ let defaultPort = proxyConf.defaultPort;
 
 // Development
 
-gulp.task('default', () => {
+const nginx = gulp.series(nginxStart);
-    return gulp.start('client', 'services');
+nginx.description = `Starts/restarts the nginx process`;
-});
 
-gulp.task('client', ['build-clean'], async() => {
+const localesRoutes = gulp.parallel(locales, routes);
-    await runSequenceP(['routes', 'locales'], 'watch', 'webpack-dev-server');
+localesRoutes.description = `Builds locales and routes`;
-});
 
-/**
+const front = gulp.series(buildClean, gulp.parallel(localesRoutes, watch, webpackDevServer));
- * Starts all backend services, including the nginx proxy and the database.
+front.description = `Starts frontend service`;
- */
-gulp.task('services', async() => {
-    await runSequenceP('docker-start', 'services-only', 'nginx');
-});
 
-/**
+const back = gulp.series(dockerStart, backOnly, nginx);
- * Starts backend services.
+back.description = `Starts backend and database service`;
- */
+
-gulp.task('services-only', callback => {
+const defaultTask = gulp.parallel(front, back);
+defaultTask.description = `Starts all application services`;
+
+function backOnly(done) {
     let app = require(`./loopback/server/server`);
     app.start(defaultPort);
-    app.on('started', callback);
+    app.on('started', done);
-});
+}
+backOnly.description = `Starts backend service`;
 
-/**
+function backTestsOnly() {
- * Runs the e2e tests, restoring the fixtures first.
+    serviceRoot = 'vn-loopback';
- */
+    let app = require(`./loopback/server/server`);
-gulp.task('e2e', ['docker'], async() => {
+
+    let specFiles = [
+        `./back/**/*.spec.js`,
+        `./loopback/**/*.spec.js`
+    ];
+    for (let mod of modules)
+        specFiles.push(`./modules/${mod}/back/**/*.spec.js`);
+
+    const jasmine = require('gulp-jasmine');
+    return gulp.src(specFiles)
+        .pipe(jasmine({errorOnFail: false}))
+        .on('jasmineDone', function() {
+            app.disconnect();
+        });
+}
+backTestsOnly.description = `Runs the backend tests only`;
+
+const backTests = gulp.series(docker, backTestsOnly);
+backTests.description = `Restarts database and runs the backend tests`;
+
+function e2eOnly() {
     const jasmine = require('gulp-jasmine');
 
     if (argv.show || argv.s)
@@ -73,31 +90,26 @@ gulp.task('e2e', ['docker'], async() => {
 
     return gulp.src('./e2e/tests.js')
         .pipe(jasmine({reporter: 'none'}));
-});
+}
+e2eOnly.description = `Runs the e2e tests only`;
 
-/**
+e2e = gulp.series(docker, e2eOnly);
- * Runs the smokes tests, restoring the fixtures first.
+e2e.description = `Restarts database and runs the e2e tests`;
- */
+
-gulp.task('smokes', ['docker'], async() => {
+function smokesOnly() {
     const jasmine = require('gulp-jasmine');
     return gulp.src('./e2e/smokes-tests.js')
         .pipe(jasmine({reporter: 'none'}));
-});
+}
+smokesOnly.description = `Runs the smokes tests only`;
 
-/**
+smokes = gulp.series(docker, smokesOnly);
- * Cleans all generated project files.
+smokes.description = `Restarts database and runs the smokes tests`;
- */
-gulp.task('clean', ['build-clean', 'nginx-clean']);
 
-/**
+const clean = gulp.parallel(buildClean, nginxClean);
- * Alias for the 'install' task.
+clean.description = 'Cleans all generated project files';
- */
-gulp.task('i', ['install']);
 
-/**
+function install() {
- * Installs node dependencies in all project directories.
- */
-gulp.task('install', () => {
     const install = require('gulp-install');
     const print = require('gulp-print');
 
@@ -113,182 +125,46 @@ gulp.task('install', () => {
         .pipe(install({
             npm: ['--no-package-lock']
         }));
-});
+}
+install.description = `Installs node dependencies in all directories`;
+
+const i = gulp.series(install);
+i.description = `Alias for the 'install' task`;
 
 // Deployment
 
-gulp.task('build', ['clean'], async() => {
+const build = gulp.series(clean, gulp.parallel(localesRoutes, webpack, nginxConf));
-    await runSequenceP(['routes', 'locales', 'webpack', 'nginx-conf']);
+build.description = `Generates binaries and configuration files`;
-});
 
-gulp.task('docker-compose', async() => {
+function buildClean() {
-    const yaml = require('js-yaml');
-
-    let compose = await fs.readFile('./docker-compose.tpl.yml', 'utf8');
-    let composeYml = yaml.safeLoad(compose);
-
-    let imageTag = 'latest';
-    if (process.env.BUILD_NUMBER)
-        imageTag = process.env.BUILD_NUMBER;
-
-    let namePrefix = '';
-    if (process.env.BRANCH_NAME)
-        namePrefix = `${process.env.BRANCH_NAME}-`;
-
-    for (let service of services) {
-        let dockerFile = `Dockerfile`;
-        let localDockerFile = `${__dirname}/services/${service}/Dockerfile`;
-
-        if (await fs.exists(localDockerFile))
-            dockerFile = localDockerFile;
-
-        composeYml.services[service] = {
-            build: {
-                context: `./services`,
-                dockerfile: dockerFile
-            },
-            ports: [`${service.port}:${defaultPort}`],
-            environment: {
-                NODE_ENV: '${NODE_ENV}'
-            }
-        };
-
-        composeYml.services.nginx.links.push(
-            `${service}:${namePrefix}${service}`
-        );
-    }
-
-    for (let serviceName in composeYml.services) {
-        let service = composeYml.services[serviceName];
-        Object.assign(service, {
-            container_name: `${namePrefix}${serviceName}`,
-            image: `${serviceName}:${imageTag}`,
-            restart: 'unless-stopped',
-            volumes: ['/config:/config']
-        });
-    }
-
-    let ymlString = yaml.safeDump(composeYml);
-    await fs.writeFile('./docker-compose.yml', ymlString);
-});
-
-/**
- * Cleans all files generated by the 'build' task.
- */
-gulp.task('build-clean', () => {
     const del = require('del');
     const files = [
         `${buildDir}/*`
     ];
     return del(files, {force: true});
-});
-
-// Nginx & services
-
-let nginxConf = 'temp/nginx.conf';
-let nginxTemp = `${nginxDir}/temp`;
-
-/**
- * Starts the nginx process, if it is started, restarts it.
- */
-gulp.task('nginx', async() => {
-    await runSequenceP('nginx-start');
-});
-
-/**
- * Starts the nginx process, generating it's configuration file first.
- */
-gulp.task('nginx-start', ['nginx-conf'], async() => {
-    let nginxBin = await nginxGetBin();
-
-    if (isWindows)
-        nginxBin = `start /B ${nginxBin}`;
-
-    log(`Application available at http://${proxyConf.host}:${proxyConf.port}/`);
-    await execP(`${nginxBin} -c "${nginxConf}" -p "${nginxDir}"`);
-});
-
-/**
- * Stops the nginx process.
- */
-gulp.task('nginx-stop', async() => {
-    try {
-        let nginxBin = await nginxGetBin();
-        await fs.stat(`${nginxTemp}/nginx.pid`);
-        await execP(`${nginxBin} -c "${nginxConf}" -p "${nginxDir}" -s stop`);
-    } catch (e) {}
-});
-
-/**
- * Generates the nginx configuration file. If NODE_ENV is defined and the
- * 'nginx.[environment].mst' file exists, it is used as a template, otherwise,
- * the 'nginx.mst' template file is used.
- */
-gulp.task('nginx-conf', ['nginx-stop'], async() => {
-    const mustache = require('mustache');
-
-    if (!await fs.exists(nginxTemp))
-        await fs.mkdir(nginxTemp);
-
-    let params = {
-        services: services,
-        defaultService: defaultService,
-        defaultPort: defaultPort,
-        devServerPort: devServerPort,
-        port: proxyConf.port,
-        host: proxyConf.host
-    };
-
-    let confFile = `${nginxDir}/nginx.${env}.mst`;
-
-    if (!await fs.exists(confFile))
-        confFile = `${nginxDir}/nginx.mst`;
-
-    let template = await fs.readFile(confFile, 'utf8');
-    let nginxConf = mustache.render(template, params);
-
-    await fs.writeFile(`${nginxTemp}/nginx.conf`, nginxConf);
-});
-
-/**
- * Cleans all files generated by nginx.
- */
-gulp.task('nginx-clean', ['nginx-stop'], () => {
-    const del = require('del');
-    return del([`${nginxTemp}/*`], {force: true});
-});
-
-async function nginxGetBin() {
-    if (isWindows)
-        return 'nginx';
-    try {
-        let nginxBin = '/usr/sbin/nginx';
-        await fs.stat(nginxBin);
-        return nginxBin;
-    } catch (e) {
-        return 'nginx';
-    }
 }
+buildClean.description = `Cleans all files generated by the 'build' task`;
 
 // Webpack
 
-gulp.task('webpack', function(callback) {
+function webpack(done) {
-    const webpack = require('webpack');
+    const webpackCompile = require('webpack');
     const merge = require('webpack-merge');
 
     let wpConfig = require('./webpack.config.js');
     wpConfig = merge(wpConfig, {});
 
-    let compiler = webpack(wpConfig);
+    let compiler = webpackCompile(wpConfig);
 
     compiler.run(function(err, stats) {
         if (err) throw new PluginError('webpack', err);
         log('[webpack]', stats.toString(wpConfig.stats));
-        callback();
+        done();
     });
-});
+}
+webpack.description = `Transpiles application into files`;
 
-gulp.task('webpack-dev-server', function(callback) {
+function webpackDevServer(done) {
     const webpack = require('webpack');
     const merge = require('webpack-merge');
     const WebpackDevServer = require('webpack-dev-server');
@@ -313,9 +189,10 @@ gulp.task('webpack-dev-server', function(callback) {
         .listen(devServer.port, devServer.host, function(err) {
             if (err) throw new PluginError('webpack-dev-server', err);
             // TODO: Keep the server alive or continue?
-            callback();
+            done();
         });
-});
+}
+webpackDevServer.description = `Transpiles application into memory`;
 
 // Locale
 
@@ -328,16 +205,18 @@ let localeFiles = [
  * Mixes all locale files into one JSON file per module and language. It looks
  * recursively in all project directories for locale folders with per language
  * yaml translation files.
+ *
+ * @return {Stream} The merged gulp streams
  */
-gulp.task('locales', function() {
+function locales() {
-    const extend = require('gulp-extend');
+    const mergeJson = require('gulp-merge-json');
     const yaml = require('gulp-yaml');
     const merge = require('merge-stream');
 
     let streams = [];
     let localePaths = [];
 
-    for (let mod of services)
+    for (let mod of modules)
         localePaths[mod] = `${modulesDir}/${mod}`;
 
     let baseMods = ['core', 'auth', 'salix'];
@@ -350,19 +229,20 @@ gulp.task('locales', function() {
             let localeFiles = `${path}/**/locale/${lang}.yml`;
             streams.push(gulp.src(localeFiles)
                 .pipe(yaml())
-                .pipe(extend(`${lang}.json`))
+                .pipe(mergeJson({fileName: `${lang}.json`}))
                 .pipe(gulp.dest(`${buildDir}/locale/${mod}`)));
         }
     }
 
     return merge(streams);
-});
+}
+locales.description = `Generates client locale files`;
 
 // Routes
 
 let routeFiles = `${modulesDir}/*/front/routes.json`;
 
-gulp.task('routes', function() {
+function routes() {
     const concat = require('gulp-concat');
     const wrap = require('gulp-wrap');
 
@@ -370,43 +250,55 @@ gulp.task('routes', function() {
         .pipe(concat('routes.js', {newLine: ','}))
         .pipe(wrap('var routes = [<%=contents%>\n];'))
         .pipe(gulp.dest(buildDir));
-});
+}
+routes.description = 'Merges all module routes file into one file';
 
 // Watch
 
-gulp.task('watch', function() {
+function watch(done) {
-    gulp.watch(routeFiles, ['routes']);
+    gulp.watch(routeFiles, gulp.series(routes));
-    gulp.watch(localeFiles, ['locales']);
+    gulp.watch(localeFiles, gulp.series(locales));
-});
+    done();
+}
+watch.description = `Watches for changes in routes and locale files`;
 
 // Docker
 
 /**
- * Rebuilds the docker, if already exists, destroys and rebuild it.
+ * Builds the database image and runs a container. It only rebuilds the
- * Also, if the container or it's image doesn't exists builds them.
+ * image when fixtures have been modified or when the day on which the
+ * image was built is different to today. Some workarounds have been used
+ * to avoid a bug with OverlayFS driver on MacOS.
  */
-gulp.task('docker', async() => {
+async function docker() {
     try {
         await execP('docker rm -fv salix-db');
     } catch (e) {}
 
-    await execP('docker build -t salix-db ./services/db');
+    let d = new Date();
-    await execP('docker run -d --name salix-db -p 3306:3306 salix-db');
+    let pad = v => v < 10 ? '0' + v : v;
-});
+    let stamp = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
+    await execP(`docker build --build-arg STAMP=${stamp} -t salix-db ./services/db`);
+
+    let runChown = process.platform == 'darwin';
+    await execP(`docker run --env RUN_CHOWN=${runChown} -d --name salix-db -p 3306:3306 salix-db`);
+    if (runChown) await dockerWait();
+}
+docker.description = `Builds the database image and runs a container`;
 
 /**
- * Does the minium effort to start the docker, if it doesn't exists calls
+ * Does the minium effort to start the database container, if it doesn't exists
- * the 'docker' task, if it is started does nothing. Keep in mind that when
+ * calls the 'docker' task, if it is started does nothing. Keep in mind that when
  * you do not rebuild the docker you may be using an outdated version of it.
  * See the 'docker' task for more info.
  */
-gulp.task('docker-start', async() => {
+async function dockerStart() {
     let state;
     try {
         let result = await execP('docker container inspect -f "{{json .State}}" salix-db');
         state = JSON.parse(result.stdout);
     } catch (err) {
-        return await runSequenceP('docker');
+        return await docker();
     }
 
     switch (state.Status) {
@@ -414,57 +306,136 @@ gulp.task('docker-start', async() => {
         return;
     case 'exited':
         await execP('docker start salix-db');
-        await runSequenceP('docker-wait');
+        await dockerWait();
         return;
     default:
         throw new Error(`Unknown docker status: ${state.Status}`);
     }
-});
+}
+dockerStart.description = `Starts the database container`;
+
+function dockerWait() {
+    return new Promise((resolve, reject) => {
+        const mysql = require('mysql2');
+
+        let interval = 100;
+        let elapsedTime = 0;
+        let maxInterval = 30 * 60 * 1000;
+
+        log('Waiting for MySQL init process...');
+        checker();
+
+        async function checker() {
+            elapsedTime += interval;
+            let state;
+
+            try {
+                let result = await execP('docker container inspect -f "{{json .State}}" salix-db');
+                state = JSON.parse(result.stdout);
+            } catch (err) {
+                return reject(new Error(err.message));
+            }
+
+            if (state.Status === 'exited')
+                return reject(new Error('Docker exited, please see the docker logs for more info'));
+
+            let conn = mysql.createConnection({
+                host: 'localhost',
+                user: 'root',
+                password: 'root'
+            });
+            conn.on('error', () => {});
+            conn.connect(err => {
+                conn.destroy();
+                if (!err) return resolve();
+
+                if (elapsedTime >= maxInterval)
+                    reject(new Error(`MySQL not initialized whithin ${elapsedTime} secs`));
+                else
+                    setTimeout(checker, interval);
+            });
+        }
+    });
+}
+dockerWait.description = `Waits until database service is ready`;
+
+// Nginx
+
+let nginxConfFile = 'temp/nginx.conf';
+let nginxTemp = `${nginxDir}/temp`;
+
+async function nginxStart() {
+    await nginxConf();
+    let nginxBin = await nginxGetBin();
+
+    if (isWindows)
+        nginxBin = `start /B ${nginxBin}`;
+
+    log(`Application available at http://${proxyConf.host}:${proxyConf.port}/`);
+    await execP(`${nginxBin} -c "${nginxConfFile}" -p "${nginxDir}"`);
+}
+nginxStart.description = `Starts the nginx process`;
+
+async function nginxStop() {
+    try {
+        let nginxBin = await nginxGetBin();
+        await fs.stat(`${nginxTemp}/nginx.pid`);
+        await execP(`${nginxBin} -c "${nginxConfFile}" -p "${nginxDir}" -s stop`);
+    } catch (e) {}
+}
+nginxStop.description = `Stops the nginx process`;
 
 /**
- * Waits until MySQL docker is started and ready to serve connections.
+ * Generates the nginx configuration file. If NODE_ENV is defined and the
+ * 'nginx.[environment].mst' file exists, it is used as a template, otherwise,
+ * the 'nginx.mst' template file is used.
  */
-gulp.task('docker-wait', callback => {
+async function nginxConf() {
-    const mysql = require('mysql2');
+    await nginxStop();
+    const mustache = require('mustache');
 
-    let interval = 100;
+    if (!await fs.exists(nginxTemp))
-    let elapsedTime = 0;
+        await fs.mkdir(nginxTemp);
-    let maxInterval = 30 * 60 * 1000;
 
-    log('Waiting for MySQL init process...');
+    let params = {
-    checker();
+        services: modules,
+        defaultService: defaultService,
+        defaultPort: defaultPort,
+        devServerPort: devServerPort,
+        port: proxyConf.port,
+        host: proxyConf.host
+    };
 
-    async function checker() {
+    let confFile = `${nginxDir}/nginx.${env}.mst`;
-        elapsedTime += interval;
-        let state;
 
-        try {
+    if (!await fs.exists(confFile))
-            let result = await execP('docker container inspect -f "{{json .State}}" salix-db');
+        confFile = `${nginxDir}/nginx.mst`;
-            state = JSON.parse(result.stdout);
-        } catch (err) {
-            return callback(new Error(err.message));
-        }
 
-        if (state.Status === 'exited')
+    let template = await fs.readFile(confFile, 'utf8');
-            return callback(new Error('Docker exited, please see the docker logs for more info'));
+    let nginxConfData = mustache.render(template, params);
 
-        let conn = mysql.createConnection({
+    await fs.writeFile(`${nginxTemp}/nginx.conf`, nginxConfData);
-            host: 'localhost',
+}
-            user: 'root',
+nginxConf.description = `Generates the nginx configuration file`;
-            password: 'root'
-        });
-        conn.on('error', () => {});
-        conn.connect(err => {
-            conn.destroy();
-            if (!err) return callback();
 
-            if (elapsedTime >= maxInterval)
+async function nginxClean() {
-                callback(new Error(`MySQL not initialized whithin ${elapsedTime} secs`));
+    await nginxStop();
-            else
+    const del = require('del');
-                setTimeout(checker, interval);
+    return del([`${nginxTemp}/*`], {force: true});
-        });
+}
+nginxClean.description = `Cleans all files generated by nginx`;
+
+async function nginxGetBin() {
+    if (isWindows)
+        return 'nginx';
+    try {
+        let nginxBin = '/usr/sbin/nginx';
+        await fs.stat(nginxBin);
+        return nginxBin;
+    } catch (e) {
+        return 'nginx';
     }
-});
+}
 
 // Helpers
 
@@ -489,21 +460,32 @@ function execP(command) {
     });
 }
 
-/**
+module.exports = {
- * Promisified version of runSequence().
+    default: defaultTask,
- *
+    front,
- * @param {String} args The list of gulp task names
+    back,
- * @return {Promise} The promise
+    backOnly,
- */
+    backTestsOnly,
-function runSequenceP(...args) {
+    backTests,
-    return new Promise((resolve, reject) => {
+    e2eOnly,
-        args = Array.prototype.slice.call(args);
+    e2e,
-        args.push(err => {
+    smokesOnly,
-            if (err)
+    smokes,
-                reject(err);
+    clean,
-            else
+    build,
-                resolve();
+    buildClean,
-        });
+    nginxStart,
-        runSequence(...args);
+    nginx,
-    });
+    nginxStop,
-}
+    nginxConf,
+    nginxClean,
+    webpack,
+    webpackDevServer,
+    locales,
+    routes,
+    localesRoutes,
+    watch,
+    docker,
+    dockerStart,
+    dockerWait
+};

package.json

@@ -9,7 +9,7 @@
         "cookie-parser": "^1.4.3",
         "cors": "^2.8.5",
         "fs-extra": "^5.0.0",
-        "helmet": "^1.3.0",
+        "helmet": "^3.15.0",
         "i18n": "^0.8.3",
         "loopback": "^3.24.0",
         "loopback-boot": "^2.27.1",
@@ -42,12 +42,12 @@
         "eslint-plugin-jasmine": "^2.10.1",
         "fancy-log": "^1.3.2",
         "file-loader": "^1.1.11",
-        "gulp": "^3.9.1",
+        "gulp": "^4.0.0",
         "gulp-concat": "^2.6.1",
         "gulp-env": "^0.4.0",
-        "gulp-extend": "^0.2.0",
         "gulp-install": "^1.1.0",
         "gulp-jasmine": "^3.0.0",
+        "gulp-merge-json": "^1.3.1",
         "gulp-print": "^2.0.1",
         "gulp-wrap": "^0.13.0",
         "gulp-yaml": "^1.0.1",
@@ -70,14 +70,12 @@
         "nightmare": "^3.0.1",
         "node-sass": "^4.9.3",
         "nodemon": "^1.18.5",
-        "nsp": "^2.1.0",
         "plugin-error": "^1.0.1",
         "raw-loader": "*",
-        "run-sequence": "^2.2.1",
         "sass-loader": "^6.0.7",
         "style-loader": "^0.20.3",
         "webpack": "^4.28.3",
-        "webpack-cli": "^3.2.0",
+        "webpack-cli": "^3.2.1",
         "webpack-dev-server": "^3.1.14",
         "webpack-merge": "^4.2.1",
         "yaml-loader": "^0.5.0"

services/db/Dockerfile

@@ -13,6 +13,7 @@ RUN apt-get update \
     && apt-get purge -y --auto-remove curl ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
+# XXX: Removes the last script line to avoid daemon to be started
 RUN cp /usr/local/bin/docker-entrypoint.sh /usr/local/bin/docker-init.sh \
     && sed -i '$ d' /usr/local/bin/docker-init.sh
 
@@ -24,6 +25,9 @@ RUN mkdir /mysql-data \
     && chown -R mysql:mysql /mysql-data \
     && rm -rf /docker-entrypoint-initdb.d/*
 
+COPY docker-start.sh /usr/local/bin/
+ENTRYPOINT ["docker-start.sh"]
+
 USER mysql
 CMD ["mysqld", "--datadir", "/mysql-data"]
 

services/db/docker-start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# XXX: Workaround to avoid OverlayFS bug on MacOs
+# https://docs.docker.com/storage/storagedriver/overlayfs-driver/#limitations-on-overlayfs-compatibility
+
+if [ "$RUN_CHOWN" = "true" ]; then
+	chown -R mysql:mysql /mysql-data
+fi
+
+exec "$@"

webpack.config.js

@@ -93,6 +93,7 @@ let baseConfig = {
         modules: false,
         assets: false,
         children: false,
+        entrypoints: false,
         colors: true
     }
 };