salix/gulpfile.js

require('require-yaml');
const gulp = require('gulp');
const runSequence = require('run-sequence');
const fs = require('fs-extra');
const exec = require('child_process').exec;
const PluginError = require('plugin-error');
const argv = require('minimist')(process.argv.slice(2));
const log = require('fancy-log');

// Configuration

let isWindows = /^win/.test(process.platform);

if (argv.NODE_ENV)
    process.env.NODE_ENV = argv.NODE_ENV;

let env = process.env.NODE_ENV ? process.env.NODE_ENV : 'development';

let langs = ['es', 'en'];
let srcDir = './client';
let servicesDir = './services';

let wpConfig = require('./webpack.config.yml');
let buildDir = wpConfig.buildDir;
let devServerPort = wpConfig.devServerPort;

let nginxDir = `${servicesDir}/nginx`;
let proxyConf = require(`${nginxDir}/config.yml`);
let proxyEnvFile = `${nginxDir}/config.${env}.yml`;

if (fs.existsSync(proxyEnvFile))
    Object.assign(proxyConf, require(proxyEnvFile));

let defaultService = proxyConf.main;
let defaultPort = proxyConf.defaultPort;

// Development

gulp.task('default', () => {
    return gulp.start('services', 'client');
});

gulp.task('client', ['build-clean'], async () => {
    await runSequenceP(['routes', 'locales'], 'watch', 'webpack-dev-server');
});

/**
 * Starts all backend services, including the nginx proxy and the database.
 */
gulp.task('services', async () => {
    await runSequenceP('docker-start', 'services-only', 'nginx');
});

/**
 * Starts backend services.
 */
gulp.task('services-only', async () => {
    const services = await getServices();
    for (let service of services)
        require(service.index).start(service.port);
});

/**
 * Runs the e2e tests, restoring the fixtures first.
 */
gulp.task('e2e', ['docker'], async () => {
    await runSequenceP('e2e-only');
});

/**
 * Runs the e2e tests.
 */
gulp.task('e2e-only', () => {
    const jasmine = require('gulp-jasmine');

    if (argv.show || argv.s)
        process.env.E2E_SHOW = true;

    return gulp.src('./e2e_tests.js')
        .pipe(jasmine({reporter: 'none'}));
});

gulp.task('smokes', ['docker'], async () => {
    await runSequenceP('smokes-only');
});

gulp.task('smokes-only', () => {
    const jasmine = require('gulp-jasmine');
    return gulp.src('./smokes_tests.js')
        .pipe(jasmine({reporter: 'none'}));
});

/**
 * Runs the backend tests.
 */
// gulp.task('test', ['test-only'], async () => {
//     gulp.watch('./services/**/*', ['test-only']);
//     gulp.unwatch('./services/node_modules');
// });

// gulp.task('test-only', () => {
//     const jasmine = require('gulp-jasmine');
//     gulp.src('./services/loopback/common/**/*[sS]pec.js')
//     .pipe(jasmine(
//         require('./services-test.config')
//     ));
// });

/**
 * Cleans all generated project files.
 */
gulp.task('clean', ['build-clean', 'nginx-clean']);

/**
 * Alias for the 'install' task.
 */
gulp.task('i', ['install']);

/**
 * Installs node dependencies in all project directories.
 */
gulp.task('install', () => {
    const install = require('gulp-install');
    const print = require('gulp-print');

    let packageFiles = [];
    let services = fs.readdirSync(servicesDir);
    services.forEach(service => {
        packageFiles.push(`${servicesDir}/${service}/package.json`);
    });
    return gulp.src(packageFiles)
        .pipe(print(filepath => {
            return `Installing packages in ${filepath}`;
        }))
        .pipe(install({
            npm: ['--no-package-lock']
        }));
});

// Deployment

gulp.task('build', ['clean'], async () => {
    await runSequenceP(['routes', 'locales', 'webpack', 'docker-compose', 'nginx-conf']);
});

gulp.task('docker-compose', async () => {
    const yaml = require('js-yaml');

    let compose = await fs.readFile('./docker-compose.tpl.yml', 'utf8');
    let composeYml = yaml.safeLoad(compose);
    let services = await getServices();

    let imageTag = 'latest';
    if (process.env.BUILD_NUMBER)
        imageTag = process.env.BUILD_NUMBER;

    let namePrefix = '';
    if (process.env.BRANCH_NAME)
        namePrefix = `${process.env.BRANCH_NAME}-`;

    for (let service of services) {
        let dockerFile = `Dockerfile`;
        let localDockerFile = `${__dirname}/services/${service.name}/Dockerfile`;

        if (await fs.exists(localDockerFile))
            dockerFile = localDockerFile;

        composeYml.services[service.name] = {
            build: {
                context: `./services`,
                dockerfile: dockerFile
            },
            ports: [`${service.port}:${defaultPort}`],
            environment: {
                NODE_ENV: '${NODE_ENV}'
            }
        };

        composeYml.services.nginx.links.push(
            `${service.name}:${namePrefix}${service.name}`
        );
    }

    for (let serviceName in composeYml.services) {
        let service = composeYml.services[serviceName];
        Object.assign(service, {
            container_name: `${namePrefix}${serviceName}`,
            image: `${serviceName}:${imageTag}`,
            restart: 'unless-stopped',
            volumes: ['/config:/config']
        });
    }

    let ymlString = yaml.safeDump(composeYml);
    await fs.writeFile('./docker-compose.yml', ymlString);
});

/**
 * Cleans all files generated by the 'build' task.
 */
gulp.task('build-clean', () => {
    const del = require('del');
    const files = [
        `${buildDir}/*`,
        `docker-compose.yml`
    ];
    return del(files, {force: true});
});

// Nginx & services

let nginxConf = 'temp/nginx.conf';
let nginxTemp = `${nginxDir}/temp`;

/**
 * Starts the nginx process, if it is started, restarts it.
 */
gulp.task('nginx', async () => {
    await runSequenceP('nginx-stop', 'nginx-start');
});

/**
 * Starts the nginx process, generating it's configuration file first.
 */
gulp.task('nginx-start', ['nginx-conf'], async () => {
    let nginxBin = await nginxGetBin();

    if (isWindows)
        nginxBin = `start /B ${nginxBin}`;

    log(`Application will be available at http://${proxyConf.host}:${proxyConf.port}/`);
    await execP(`${nginxBin} -c "${nginxConf}" -p "${nginxDir}"`);
});

/**
 * Stops the nginx process.
 */
gulp.task('nginx-stop', async () => {
    try {
        let nginxBin = await nginxGetBin();
        await fs.stat(`${nginxTemp}/nginx.pid`);
        await execP(`${nginxBin} -c "${nginxConf}" -p "${nginxDir}" -s stop`);
    } catch (e) {}
});

/**
 * Generates the nginx configuration file. If NODE_ENV is defined and the
 * 'nginx.[environment].mst' file exists, it is used as a template, otherwise,
 * the 'nginx.mst' template file is used.
 */
gulp.task('nginx-conf', ['nginx-stop'], async () => {
    const mustache = require('mustache');

    if (!await fs.exists(nginxTemp))
        await fs.mkdir(nginxTemp);

    let params = {
        services: await getServices(),
        defaultService: defaultService,
        defaultPort: defaultPort,
        devServerPort: devServerPort,
        port: proxyConf.port,
        host: proxyConf.host
    };

    let confFile = `${nginxDir}/nginx.${env}.mst`;

    if (!await fs.exists(confFile))
        confFile = `${nginxDir}/nginx.mst`;

    let template = await fs.readFile(confFile, 'utf8');
    let nginxConf = mustache.render(template, params);

    await fs.writeFile(`${nginxTemp}/nginx.conf`, nginxConf);
});

/**
 * Cleans all files generated by nginx.
 */
gulp.task('nginx-clean', ['nginx-stop'], () => {
    const del = require('del');
    return del([`${nginxTemp}/*`], {force: true});
});

async function nginxGetBin() {
    if (isWindows)
        return 'nginx';
    try {
        let nginxBin = '/usr/sbin/nginx';
        await fs.stat(nginxBin);
        return nginxBin;
    } catch (e) {
        return 'nginx';
    }
}

async function getServices() {
    let services;
    let startPort = defaultPort + 1;
    services = [];

    const serviceDirs = await fs.readdir(servicesDir);
    const exclude = ['loopback'];

    for (let service of serviceDirs) {
        let index = `${servicesDir}/${service}/server/server.js`;
        if (!await fs.exists(index) || exclude.indexOf(service) !== -1) continue;

        let port = service == defaultService ? defaultPort : startPort++;
        services.push({
            name: service,
            index: index,
            port: port
        });
    }

    return services;
}

// Webpack

gulp.task('webpack', function(callback) {
    const webpack = require('webpack');
    const merge = require('webpack-merge');

    let wpConfig = require('./webpack.config.js');
    wpConfig = merge(wpConfig, {});

    let compiler = webpack(wpConfig);

    compiler.run(function(err, stats) {
        if (err) throw new PluginError('webpack', err);
        log('[webpack]', stats.toString(wpConfig.stats));
        callback();
    });
});

gulp.task('webpack-dev-server', function(callback) {
    const webpack = require('webpack');
    const merge = require('webpack-merge');
    const WebpackDevServer = require('webpack-dev-server');

    let wpConfig = require('./webpack.config.js');
    wpConfig = merge(wpConfig, {});

    let devServer = wpConfig.devServer;

    for (let entryName in wpConfig.entry) {
        let entry = wpConfig.entry[entryName];
        let wdsAssets = [`webpack-dev-server/client?http://127.0.0.1:${devServer.port}/`];
        if (Array.isArray(entry))
            wdsAssets = wdsAssets.concat(entry);
        else
            wdsAssets.push(entry);
        wpConfig.entry[entryName] = wdsAssets;
    }

    let compiler = webpack(wpConfig);
    new WebpackDevServer(compiler, wpConfig.devServer)
        .listen(devServer.port, devServer.host, function(err) {
            if (err) throw new PluginError('webpack-dev-server', err);
            // TODO: Keep the server alive or continue?
            callback();
        });
});

// Locale

let localeFiles = `${srcDir}/**/locale/*.yml`;

/**
 * Mixes all locale files into one JSON file per module and language. It looks
 * recursively in all project directories for locale folders with per language
 * yaml translation files.
 */
gulp.task('locales', function() {
    const extend = require('gulp-extend');
    const yaml = require('gulp-yaml');
    const merge = require('merge-stream');
    const modules = require('./client/modules.yml');

    let streams = [];

    for (let mod in modules) {
        for (let lang of langs) {
            let localeFiles = `./client/${mod}/**/locale/${lang}.yml`;
            streams.push(gulp.src(localeFiles)
                .pipe(yaml())
                .pipe(extend(`${lang}.json`))
                .pipe(gulp.dest(`${buildDir}/locale/${mod}`)));
        }
    }

    return merge(streams);
});

// Routes

let routeFiles = `${srcDir}/**/routes.json`;

gulp.task('routes', function() {
    const concat = require('gulp-concat');
    const wrap = require('gulp-wrap');

    return gulp.src(routeFiles)
        .pipe(concat('routes.js', {newLine: ','}))
        .pipe(wrap('var routes = [<%=contents%>\n];'))
        .pipe(gulp.dest(buildDir));
});

// Watch

gulp.task('watch', function() {
    gulp.watch(routeFiles, ['routes']);
    gulp.watch(localeFiles, ['locales']);
});

// Docker

/**
 * Rebuilds the docker, if already exists, destroys and
 * rebuild it.
 */
gulp.task('docker', async () => {
    try {
        await execP('docker rm -fv dblocal');
    } catch (e) {}

    await runSequenceP('docker-run');
});

/**
 * Rebuilds the docker image, if already exists, destroys and
 * rebuild it. calls upon docker task afterwards.
 */
gulp.task('docker-build', async () => {
    try {
        await execP('docker rm -fv dblocal');
    } catch (e) {}
    try {
        await execP('docker rmi dblocal:latest');
    } catch (e) {}
    try {
        await execP('docker volume rm data');
    } catch (e) {}

    log('Building image...');
    await execP('docker build -t dblocal:latest ./services/db');

    await runSequenceP('docker');
});

/**
 * Does the minium effort to start the docker, if it doesn't exists calls
 * the 'docker-run' task, if it is started does nothing. Keep in mind that when
 * you do not rebuild the docker you may be using an outdated version of it.
 * See the 'docker' task for more info.
 */
gulp.task('docker-start', async () => {
    let state;
    try {
        let result = await execP('docker container inspect -f "{{json .State}}" dblocal');
        state = JSON.parse(result.stdout);
    } catch (err) {
        return await runSequenceP('docker-run');
    }

    switch (state.Status) {
    case 'running':
        return;
    case 'exited':
        return await execP('docker start dblocal');
    default:
        throw new Error(`Unknown docker status: ${status}`);
    }
});

/**
 * Runs the docker, if the container or it's image doesn't exists builds them.
 */
gulp.task('docker-run', async () => {
    try {
        await execP('docker image inspect -f "{{json .Id}}" dblocal');
        await execP('docker run -d --name dblocal --volume data:/data -p 3306:3306 dblocal');
        await runSequenceP('docker-wait');
    } catch (err) {
        await runSequenceP('docker-build');
    }
});

/**
 * Waits until MySQL docker is started and ready to serve connections.
 */
gulp.task('docker-wait', callback => {
    const mysql = require('mysql2');

    let interval = 1;
    let elapsedTime = 0;
    let maxInterval = 30 * 60;

    log('Waiting for MySQL init process...');
    checker();

    async function checker() {
        elapsedTime += interval;
        let state;

        try {
            let result = await execP('docker container inspect -f "{{json .State}}" dblocal');
            state = JSON.parse(result.stdout);
        } catch (err) {
            return callback(new Error(err.message));
        }

        if (state.Status === 'exited')
            return callback(new Error('Docker exited, please see the docker logs for more info'));

        let conn = mysql.createConnection({
            host: 'localhost',
            user: 'root',
            password: 'root'
        });
        conn.on('error', () => {});
        conn.connect(err => {
            conn.destroy();
            if (!err) return callback();

            if (elapsedTime >= maxInterval)
                callback(new Error(`MySQL not initialized whithin ${elapsedTime} secs`));
            else
                setTimeout(checker, interval * 1000);
        });
    }
});

// Helpers

/**
 * Promisified version of exec().
 *
 * @param {String} command The exec command
 * @return {Promise} The promise
 */
function execP(command) {
    return new Promise((resolve, reject) => {
        exec(command, (err, stdout, stderr) => {
            if (err)
                reject(err);
            else {
                resolve({
                    stdout: stdout,
                    stderr: stderr
                });
            }
        });
    });
}

/**
 * Promisified version of runSequence().
 *
 * @param {String} args The list of gulp task names
 * @return {Promise} The promise
 */
function runSequenceP() {
    return new Promise((resolve, reject) => {
        let args = Array.prototype.slice.call(arguments);
        args.push(err => {
            if (err)
                reject(err);
            else
                resolve();
        });
        runSequence(...args);
    });
}