Gulp updated, packages updated, db docker bugs fixed

This commit is contained in:
Juan Ferrer 2019-01-09 10:02:56 +01:00
parent 5130295a0c
commit ab3e588b1e
6 changed files with 993 additions and 1702 deletions

View File

@ -1,6 +1,5 @@
require('require-yaml');
const gulp = require('gulp');
const runSequence = require('run-sequence');
const fs = require('fs-extra');
const exec = require('child_process').exec;
const PluginError = require('plugin-error');
@ -20,7 +19,7 @@ let langs = ['es', 'en'];
let srcDir = './front';
let modulesDir = './modules';
let servicesDir = './services';
let services = require('./modules.yml');
let modules = require('./modules.yml');
let wpConfig = require('./webpack.config.yml');
let buildDir = wpConfig.buildDir;
@ -38,34 +37,52 @@ let defaultPort = proxyConf.defaultPort;
// Development
gulp.task('default', () => {
return gulp.start('client', 'services');
});
const nginx = gulp.series(nginxStart);
nginx.description = `Starts/restarts the nginx process`;
gulp.task('client', ['build-clean'], async() => {
await runSequenceP(['routes', 'locales'], 'watch', 'webpack-dev-server');
});
const localesRoutes = gulp.parallel(locales, routes);
localesRoutes.description = `Builds locales and routes`;
/**
* Starts all backend services, including the nginx proxy and the database.
*/
gulp.task('services', async() => {
await runSequenceP('docker-start', 'services-only', 'nginx');
});
const front = gulp.series(buildClean, gulp.parallel(localesRoutes, watch, webpackDevServer));
front.description = `Starts frontend service`;
/**
* Starts backend services.
*/
gulp.task('services-only', callback => {
const back = gulp.series(dockerStart, backOnly, nginx);
back.description = `Starts backend and database service`;
const defaultTask = gulp.parallel(front, back);
defaultTask.description = `Starts all application services`;
function backOnly(done) {
let app = require(`./loopback/server/server`);
app.start(defaultPort);
app.on('started', callback);
});
app.on('started', done);
}
backOnly.description = `Starts backend service`;
/**
* Runs the e2e tests, restoring the fixtures first.
*/
gulp.task('e2e', ['docker'], async() => {
function backTestsOnly() {
serviceRoot = 'vn-loopback';
let app = require(`./loopback/server/server`);
let specFiles = [
`./back/**/*.spec.js`,
`./loopback/**/*.spec.js`
];
for (let mod of modules)
specFiles.push(`./modules/${mod}/back/**/*.spec.js`);
const jasmine = require('gulp-jasmine');
return gulp.src(specFiles)
.pipe(jasmine({errorOnFail: false}))
.on('jasmineDone', function() {
app.disconnect();
});
}
backTestsOnly.description = `Runs the backend tests only`;
const backTests = gulp.series(docker, backTestsOnly);
backTests.description = `Restarts database and runs the backend tests`;
function e2eOnly() {
const jasmine = require('gulp-jasmine');
if (argv.show || argv.s)
@ -73,31 +90,26 @@ gulp.task('e2e', ['docker'], async() => {
return gulp.src('./e2e/tests.js')
.pipe(jasmine({reporter: 'none'}));
});
}
e2eOnly.description = `Runs the e2e tests only`;
/**
* Runs the smokes tests, restoring the fixtures first.
*/
gulp.task('smokes', ['docker'], async() => {
e2e = gulp.series(docker, e2eOnly);
e2e.description = `Restarts database and runs the e2e tests`;
function smokesOnly() {
const jasmine = require('gulp-jasmine');
return gulp.src('./e2e/smokes-tests.js')
.pipe(jasmine({reporter: 'none'}));
});
}
smokesOnly.description = `Runs the smokes tests only`;
/**
* Cleans all generated project files.
*/
gulp.task('clean', ['build-clean', 'nginx-clean']);
smokes = gulp.series(docker, smokesOnly);
smokes.description = `Restarts database and runs the smokes tests`;
/**
* Alias for the 'install' task.
*/
gulp.task('i', ['install']);
const clean = gulp.parallel(buildClean, nginxClean);
clean.description = 'Cleans all generated project files';
/**
* Installs node dependencies in all project directories.
*/
gulp.task('install', () => {
function install() {
const install = require('gulp-install');
const print = require('gulp-print');
@ -113,182 +125,46 @@ gulp.task('install', () => {
.pipe(install({
npm: ['--no-package-lock']
}));
});
}
install.description = `Installs node dependencies in all directories`;
const i = gulp.series(install);
i.description = `Alias for the 'install' task`;
// Deployment
gulp.task('build', ['clean'], async() => {
await runSequenceP(['routes', 'locales', 'webpack', 'nginx-conf']);
});
const build = gulp.series(clean, gulp.parallel(localesRoutes, webpack, nginxConf));
build.description = `Generates binaries and configuration files`;
gulp.task('docker-compose', async() => {
const yaml = require('js-yaml');
let compose = await fs.readFile('./docker-compose.tpl.yml', 'utf8');
let composeYml = yaml.safeLoad(compose);
let imageTag = 'latest';
if (process.env.BUILD_NUMBER)
imageTag = process.env.BUILD_NUMBER;
let namePrefix = '';
if (process.env.BRANCH_NAME)
namePrefix = `${process.env.BRANCH_NAME}-`;
for (let service of services) {
let dockerFile = `Dockerfile`;
let localDockerFile = `${__dirname}/services/${service}/Dockerfile`;
if (await fs.exists(localDockerFile))
dockerFile = localDockerFile;
composeYml.services[service] = {
build: {
context: `./services`,
dockerfile: dockerFile
},
ports: [`${service.port}:${defaultPort}`],
environment: {
NODE_ENV: '${NODE_ENV}'
}
};
composeYml.services.nginx.links.push(
`${service}:${namePrefix}${service}`
);
}
for (let serviceName in composeYml.services) {
let service = composeYml.services[serviceName];
Object.assign(service, {
container_name: `${namePrefix}${serviceName}`,
image: `${serviceName}:${imageTag}`,
restart: 'unless-stopped',
volumes: ['/config:/config']
});
}
let ymlString = yaml.safeDump(composeYml);
await fs.writeFile('./docker-compose.yml', ymlString);
});
/**
* Cleans all files generated by the 'build' task.
*/
gulp.task('build-clean', () => {
function buildClean() {
const del = require('del');
const files = [
`${buildDir}/*`
];
return del(files, {force: true});
});
// Nginx & services
let nginxConf = 'temp/nginx.conf';
let nginxTemp = `${nginxDir}/temp`;
/**
* Starts the nginx process, if it is started, restarts it.
*/
gulp.task('nginx', async() => {
await runSequenceP('nginx-start');
});
/**
* Starts the nginx process, generating it's configuration file first.
*/
gulp.task('nginx-start', ['nginx-conf'], async() => {
let nginxBin = await nginxGetBin();
if (isWindows)
nginxBin = `start /B ${nginxBin}`;
log(`Application available at http://${proxyConf.host}:${proxyConf.port}/`);
await execP(`${nginxBin} -c "${nginxConf}" -p "${nginxDir}"`);
});
/**
* Stops the nginx process.
*/
gulp.task('nginx-stop', async() => {
try {
let nginxBin = await nginxGetBin();
await fs.stat(`${nginxTemp}/nginx.pid`);
await execP(`${nginxBin} -c "${nginxConf}" -p "${nginxDir}" -s stop`);
} catch (e) {}
});
/**
* Generates the nginx configuration file. If NODE_ENV is defined and the
* 'nginx.[environment].mst' file exists, it is used as a template, otherwise,
* the 'nginx.mst' template file is used.
*/
gulp.task('nginx-conf', ['nginx-stop'], async() => {
const mustache = require('mustache');
if (!await fs.exists(nginxTemp))
await fs.mkdir(nginxTemp);
let params = {
services: services,
defaultService: defaultService,
defaultPort: defaultPort,
devServerPort: devServerPort,
port: proxyConf.port,
host: proxyConf.host
};
let confFile = `${nginxDir}/nginx.${env}.mst`;
if (!await fs.exists(confFile))
confFile = `${nginxDir}/nginx.mst`;
let template = await fs.readFile(confFile, 'utf8');
let nginxConf = mustache.render(template, params);
await fs.writeFile(`${nginxTemp}/nginx.conf`, nginxConf);
});
/**
* Cleans all files generated by nginx.
*/
gulp.task('nginx-clean', ['nginx-stop'], () => {
const del = require('del');
return del([`${nginxTemp}/*`], {force: true});
});
async function nginxGetBin() {
if (isWindows)
return 'nginx';
try {
let nginxBin = '/usr/sbin/nginx';
await fs.stat(nginxBin);
return nginxBin;
} catch (e) {
return 'nginx';
}
}
buildClean.description = `Cleans all files generated by the 'build' task`;
// Webpack
gulp.task('webpack', function(callback) {
const webpack = require('webpack');
function webpack(done) {
const webpackCompile = require('webpack');
const merge = require('webpack-merge');
let wpConfig = require('./webpack.config.js');
wpConfig = merge(wpConfig, {});
let compiler = webpack(wpConfig);
let compiler = webpackCompile(wpConfig);
compiler.run(function(err, stats) {
if (err) throw new PluginError('webpack', err);
log('[webpack]', stats.toString(wpConfig.stats));
callback();
done();
});
});
}
webpack.description = `Transpiles application into files`;
gulp.task('webpack-dev-server', function(callback) {
function webpackDevServer(done) {
const webpack = require('webpack');
const merge = require('webpack-merge');
const WebpackDevServer = require('webpack-dev-server');
@ -313,9 +189,10 @@ gulp.task('webpack-dev-server', function(callback) {
.listen(devServer.port, devServer.host, function(err) {
if (err) throw new PluginError('webpack-dev-server', err);
// TODO: Keep the server alive or continue?
callback();
done();
});
});
}
webpackDevServer.description = `Transpiles application into memory`;
// Locale
@ -328,16 +205,18 @@ let localeFiles = [
* Mixes all locale files into one JSON file per module and language. It looks
* recursively in all project directories for locale folders with per language
* yaml translation files.
*
* @return {Stream} The merged gulp streams
*/
gulp.task('locales', function() {
const extend = require('gulp-extend');
function locales() {
const mergeJson = require('gulp-merge-json');
const yaml = require('gulp-yaml');
const merge = require('merge-stream');
let streams = [];
let localePaths = [];
for (let mod of services)
for (let mod of modules)
localePaths[mod] = `${modulesDir}/${mod}`;
let baseMods = ['core', 'auth', 'salix'];
@ -350,19 +229,20 @@ gulp.task('locales', function() {
let localeFiles = `${path}/**/locale/${lang}.yml`;
streams.push(gulp.src(localeFiles)
.pipe(yaml())
.pipe(extend(`${lang}.json`))
.pipe(mergeJson({fileName: `${lang}.json`}))
.pipe(gulp.dest(`${buildDir}/locale/${mod}`)));
}
}
return merge(streams);
});
}
locales.description = `Generates client locale files`;
// Routes
let routeFiles = `${modulesDir}/*/front/routes.json`;
gulp.task('routes', function() {
function routes() {
const concat = require('gulp-concat');
const wrap = require('gulp-wrap');
@ -370,43 +250,55 @@ gulp.task('routes', function() {
.pipe(concat('routes.js', {newLine: ','}))
.pipe(wrap('var routes = [<%=contents%>\n];'))
.pipe(gulp.dest(buildDir));
});
}
routes.description = 'Merges all module routes file into one file';
// Watch
gulp.task('watch', function() {
gulp.watch(routeFiles, ['routes']);
gulp.watch(localeFiles, ['locales']);
});
function watch(done) {
gulp.watch(routeFiles, gulp.series(routes));
gulp.watch(localeFiles, gulp.series(locales));
done();
}
watch.description = `Watches for changes in routes and locale files`;
// Docker
/**
* Rebuilds the docker, if already exists, destroys and rebuild it.
* Also, if the container or it's image doesn't exists builds them.
* Builds the database image and runs a container. It only rebuilds the
* image when fixtures have been modified or when the day on which the
* image was built is different to today. Some workarounds have been used
* to avoid a bug with OverlayFS driver on MacOS.
*/
gulp.task('docker', async() => {
async function docker() {
try {
await execP('docker rm -fv salix-db');
} catch (e) {}
await execP('docker build -t salix-db ./services/db');
await execP('docker run -d --name salix-db -p 3306:3306 salix-db');
});
let d = new Date();
let pad = v => v < 10 ? '0' + v : v;
let stamp = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
await execP(`docker build --build-arg STAMP=${stamp} -t salix-db ./services/db`);
let runChown = process.platform == 'darwin';
await execP(`docker run --env RUN_CHOWN=${runChown} -d --name salix-db -p 3306:3306 salix-db`);
if (runChown) await dockerWait();
}
docker.description = `Builds the database image and runs a container`;
/**
* Does the minium effort to start the docker, if it doesn't exists calls
* the 'docker' task, if it is started does nothing. Keep in mind that when
* Does the minium effort to start the database container, if it doesn't exists
* calls the 'docker' task, if it is started does nothing. Keep in mind that when
* you do not rebuild the docker you may be using an outdated version of it.
* See the 'docker' task for more info.
*/
gulp.task('docker-start', async() => {
async function dockerStart() {
let state;
try {
let result = await execP('docker container inspect -f "{{json .State}}" salix-db');
state = JSON.parse(result.stdout);
} catch (err) {
return await runSequenceP('docker');
return await docker();
}
switch (state.Status) {
@ -414,57 +306,136 @@ gulp.task('docker-start', async() => {
return;
case 'exited':
await execP('docker start salix-db');
await runSequenceP('docker-wait');
await dockerWait();
return;
default:
throw new Error(`Unknown docker status: ${state.Status}`);
}
});
}
dockerStart.description = `Starts the database container`;
function dockerWait() {
return new Promise((resolve, reject) => {
const mysql = require('mysql2');
let interval = 100;
let elapsedTime = 0;
let maxInterval = 30 * 60 * 1000;
log('Waiting for MySQL init process...');
checker();
async function checker() {
elapsedTime += interval;
let state;
try {
let result = await execP('docker container inspect -f "{{json .State}}" salix-db');
state = JSON.parse(result.stdout);
} catch (err) {
return reject(new Error(err.message));
}
if (state.Status === 'exited')
return reject(new Error('Docker exited, please see the docker logs for more info'));
let conn = mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'root'
});
conn.on('error', () => {});
conn.connect(err => {
conn.destroy();
if (!err) return resolve();
if (elapsedTime >= maxInterval)
reject(new Error(`MySQL not initialized whithin ${elapsedTime} secs`));
else
setTimeout(checker, interval);
});
}
});
}
dockerWait.description = `Waits until database service is ready`;
// Nginx
let nginxConfFile = 'temp/nginx.conf';
let nginxTemp = `${nginxDir}/temp`;
async function nginxStart() {
await nginxConf();
let nginxBin = await nginxGetBin();
if (isWindows)
nginxBin = `start /B ${nginxBin}`;
log(`Application available at http://${proxyConf.host}:${proxyConf.port}/`);
await execP(`${nginxBin} -c "${nginxConfFile}" -p "${nginxDir}"`);
}
nginxStart.description = `Starts the nginx process`;
async function nginxStop() {
try {
let nginxBin = await nginxGetBin();
await fs.stat(`${nginxTemp}/nginx.pid`);
await execP(`${nginxBin} -c "${nginxConfFile}" -p "${nginxDir}" -s stop`);
} catch (e) {}
}
nginxStop.description = `Stops the nginx process`;
/**
* Waits until MySQL docker is started and ready to serve connections.
* Generates the nginx configuration file. If NODE_ENV is defined and the
* 'nginx.[environment].mst' file exists, it is used as a template, otherwise,
* the 'nginx.mst' template file is used.
*/
gulp.task('docker-wait', callback => {
const mysql = require('mysql2');
async function nginxConf() {
await nginxStop();
const mustache = require('mustache');
let interval = 100;
let elapsedTime = 0;
let maxInterval = 30 * 60 * 1000;
if (!await fs.exists(nginxTemp))
await fs.mkdir(nginxTemp);
log('Waiting for MySQL init process...');
checker();
let params = {
services: modules,
defaultService: defaultService,
defaultPort: defaultPort,
devServerPort: devServerPort,
port: proxyConf.port,
host: proxyConf.host
};
async function checker() {
elapsedTime += interval;
let state;
let confFile = `${nginxDir}/nginx.${env}.mst`;
try {
let result = await execP('docker container inspect -f "{{json .State}}" salix-db');
state = JSON.parse(result.stdout);
} catch (err) {
return callback(new Error(err.message));
}
if (!await fs.exists(confFile))
confFile = `${nginxDir}/nginx.mst`;
if (state.Status === 'exited')
return callback(new Error('Docker exited, please see the docker logs for more info'));
let template = await fs.readFile(confFile, 'utf8');
let nginxConfData = mustache.render(template, params);
let conn = mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'root'
});
conn.on('error', () => {});
conn.connect(err => {
conn.destroy();
if (!err) return callback();
await fs.writeFile(`${nginxTemp}/nginx.conf`, nginxConfData);
}
nginxConf.description = `Generates the nginx configuration file`;
if (elapsedTime >= maxInterval)
callback(new Error(`MySQL not initialized whithin ${elapsedTime} secs`));
else
setTimeout(checker, interval);
});
async function nginxClean() {
await nginxStop();
const del = require('del');
return del([`${nginxTemp}/*`], {force: true});
}
nginxClean.description = `Cleans all files generated by nginx`;
async function nginxGetBin() {
if (isWindows)
return 'nginx';
try {
let nginxBin = '/usr/sbin/nginx';
await fs.stat(nginxBin);
return nginxBin;
} catch (e) {
return 'nginx';
}
});
}
// Helpers
@ -489,21 +460,32 @@ function execP(command) {
});
}
/**
* Promisified version of runSequence().
*
* @param {String} args The list of gulp task names
* @return {Promise} The promise
*/
function runSequenceP(...args) {
return new Promise((resolve, reject) => {
args = Array.prototype.slice.call(args);
args.push(err => {
if (err)
reject(err);
else
resolve();
});
runSequence(...args);
});
}
module.exports = {
default: defaultTask,
front,
back,
backOnly,
backTestsOnly,
backTests,
e2eOnly,
e2e,
smokesOnly,
smokes,
clean,
build,
buildClean,
nginxStart,
nginx,
nginxStop,
nginxConf,
nginxClean,
webpack,
webpackDevServer,
locales,
routes,
localesRoutes,
watch,
docker,
dockerStart,
dockerWait
};

2146
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -9,7 +9,7 @@
"cookie-parser": "^1.4.3",
"cors": "^2.8.5",
"fs-extra": "^5.0.0",
"helmet": "^1.3.0",
"helmet": "^3.15.0",
"i18n": "^0.8.3",
"loopback": "^3.24.0",
"loopback-boot": "^2.27.1",
@ -42,12 +42,12 @@
"eslint-plugin-jasmine": "^2.10.1",
"fancy-log": "^1.3.2",
"file-loader": "^1.1.11",
"gulp": "^3.9.1",
"gulp": "^4.0.0",
"gulp-concat": "^2.6.1",
"gulp-env": "^0.4.0",
"gulp-extend": "^0.2.0",
"gulp-install": "^1.1.0",
"gulp-jasmine": "^3.0.0",
"gulp-merge-json": "^1.3.1",
"gulp-print": "^2.0.1",
"gulp-wrap": "^0.13.0",
"gulp-yaml": "^1.0.1",
@ -70,14 +70,12 @@
"nightmare": "^3.0.1",
"node-sass": "^4.9.3",
"nodemon": "^1.18.5",
"nsp": "^2.1.0",
"plugin-error": "^1.0.1",
"raw-loader": "*",
"run-sequence": "^2.2.1",
"sass-loader": "^6.0.7",
"style-loader": "^0.20.3",
"webpack": "^4.28.3",
"webpack-cli": "^3.2.0",
"webpack-cli": "^3.2.1",
"webpack-dev-server": "^3.1.14",
"webpack-merge": "^4.2.1",
"yaml-loader": "^0.5.0"

View File

@ -13,6 +13,7 @@ RUN apt-get update \
&& apt-get purge -y --auto-remove curl ca-certificates \
&& rm -rf /var/lib/apt/lists/*
# XXX: Removes the last script line to avoid daemon to be started
RUN cp /usr/local/bin/docker-entrypoint.sh /usr/local/bin/docker-init.sh \
&& sed -i '$ d' /usr/local/bin/docker-init.sh
@ -24,6 +25,9 @@ RUN mkdir /mysql-data \
&& chown -R mysql:mysql /mysql-data \
&& rm -rf /docker-entrypoint-initdb.d/*
COPY docker-start.sh /usr/local/bin/
ENTRYPOINT ["docker-start.sh"]
USER mysql
CMD ["mysqld", "--datadir", "/mysql-data"]

10
services/db/docker-start.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
# XXX: Workaround to avoid OverlayFS bug on MacOs
# https://docs.docker.com/storage/storagedriver/overlayfs-driver/#limitations-on-overlayfs-compatibility
if [ "$RUN_CHOWN" = "true" ]; then
chown -R mysql:mysql /mysql-data
fi
exec "$@"

View File

@ -93,6 +93,7 @@ let baseConfig = {
modules: false,
assets: false,
children: false,
entrypoints: false,
colors: true
}
};