WIP: 5739-dockerRefactor #1822

Draft
alexm wants to merge 67 commits from 5739-dockerRefactor into dev
22 changed files with 4083 additions and 680 deletions

View File

@ -1,24 +1,15 @@
FROM debian:bookworm-slim
FROM node:20-bullseye-slim
ENV TZ Europe/Madrid
ARG DEBIAN_FRONTEND=noninteractive
# NodeJs
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
curl \
ca-certificates \
gnupg2 \
graphicsmagick \
&& curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
&& apt-get install -y --no-install-recommends nodejs \
&& npm install -g npm@9.6.6
# Puppeteer
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
gnupg2 \
graphicsmagick \
libfontconfig lftp xvfb gconf-service libasound2 libatk1.0-0 libc6 \
libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 \
libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 \

31
Jenkinsfile vendored
View File

@ -7,6 +7,7 @@ pipeline {
environment {
PROJECT_NAME = 'salix'
STACK_NAME = "${env.PROJECT_NAME}-${env.BRANCH_NAME}"
STACK_NAME_LC= "${env.PROJECT_NAME}-${env.BRANCH_NAME}".toLowerCase()
Review

Docker cuando crea la network por defecto le pone el nombre del stack en lowerCase.

Docker cuando crea la network por defecto le pone el nombre del stack en lowerCase.
}
stages {
stage('Checkout') {
@ -39,10 +40,10 @@ pipeline {
NODE_ENV = ""
}
steps {
nodejs('node-v20') {
sh 'npm install --no-audit --prefer-offline'
sh 'gulp install --ci'
}
sh "docker-compose -p testing-${env.STACK_NAME} -f docker-compose.test.yml rm -s -f"
sh "docker-compose -p testing-${env.STACK_NAME} -f docker-compose.test.yml build --force-rm db back"
sh "docker network rm testing-${env.STACK_NAME_LC}_default || true"
sh "docker network create testing-${STACK_NAME_LC}_default || true"
}
}
stage('Test') {
@ -57,20 +58,30 @@ pipeline {
parallel {
stage('Frontend') {
steps {
nodejs('node-v20') {
sh 'jest --ci --reporters=default --reporters=jest-junit --maxWorkers=2'
}
sh "docker-compose -p testing-${env.STACK_NAME} -f docker-compose.test.yml run --rm front"
}
}
stage('Backend') {
steps {
nodejs('node-v20') {
sh 'npm run test:back:ci'
}
sh "docker-compose -p testing-${env.STACK_NAME} -f docker-compose.test.yml run --rm back"
}
}
}
}
stage('ClearTesting') {
when { not { anyOf {
branch 'test'
branch 'master'
}}}
environment {
NODE_ENV = ""
TZ = 'Europe/Madrid'
}
steps {
sh "docker-compose -p testing-${env.STACK_NAME} -f docker-compose.test.yml rm -s -f"
sh "docker network rm testing-${env.STACK_NAME_LC}_default"
}
}
stage('Build') {
when { anyOf {
branch 'test'

View File

@ -8,44 +8,67 @@ Salix is also the scientific name of a beautifull tree! :)
Required applications.
* Node.js
* Docker
* Git
You will need to install globally the following items.
```
$ sudo npm install -g jest gulp-cli
```
Optional.
- NodeJS
## Directory structure recommendation
- /Salix
- /back ← this project
- /front ← salix-front project
## Installing dependencies and launching
Pull from repository.
```sh
$ git clone [url] [directory]
```
Run this commands on project root directory to install Node dependencies.
```sh
$ sh init.sh
```
$ npm install
$ gulp install
```
Launch application in developer environment.
```
$ gulp
```sh
$ docker compose -f docker-compose.local.yml up
# Or with NodeJS
$ npm run start
```
Manually reset fixtures.
Manually reset fixtures if has changes.
```sh
$ sh db/reset.sh
```
$ gulp docker
Manually reset strucutre/fixtures always.
```sh
$ sh db/reset.sh --no-cache
```
## Running the unit tests
For client-side unit tests run from project's root.
```
```sh
$ sh runTest.sh front
# Or with NodeJS
$ npm run test:front
```
For server-side unit tests run from project's root.
```
```sh
$ sh runTest.sh back
# Or with NodeJS
$ npm run test:back
```
Review

En els e2e tenim un problema pq si que fa falta tindre node instalat localment
Per a que puga executar:
node e2e/helpers/tests.js.

Puc seguir intentant vore si puc fer que vaja en docker en local

En els e2e tenim un problema pq si que fa falta tindre node instalat localment Per a que puga executar: `node e2e/helpers/tests.js`. Puc seguir intentant vore si puc fer que vaja en docker en local

59
back/Dockerfile Normal file
View File

@ -0,0 +1,59 @@
FROM node:20-bullseye as base
ENV TZ Europe/Madrid
ARG DEBIAN_FRONTEND=noninteractive
# Puppeteer
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
gnupg2 \
graphicsmagick \
libfontconfig lftp xvfb gconf-service libasound2 libatk1.0-0 libc6 \
libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 \
libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 \
libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 \
libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 \
libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 \
fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget
# Extra dependencies
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
samba-common-bin samba-dsdb-modules\
&& rm -rf /var/lib/apt/lists/*
RUN npm i -g gulp-cli
WORKDIR /salix
COPY print/package.json print/package-lock.json print/
COPY loopback/package.json loopback/
COPY package.json package-lock.json ./
COPY gulpfile.js ./
COPY \
LICENSE \
README.md \
./
FROM base AS local
WORKDIR /salix
RUN cd ./print && npx puppeteer browsers install chrome
CMD ["npx", "gulp", "backWatch"]
FROM base AS test
WORKDIR /salix
RUN cd ./print && npx puppeteer browsers install chrome
COPY loopback/common loopback/common
RUN npm i
COPY loopback/server loopback/server
COPY loopback/util loopback/util
COPY storage storage
COPY back back
COPY print print
COPY modules modules
COPY loopback/locale loopback/locale
CMD ["npx", "gulp", "backOnly"]

View File

@ -1,4 +1,4 @@
const Docker = require('../db/docker.js');
// const Docker = require('../db/docker.js');
let dataSources = require('../loopback/server/datasources.json');
process.on('warning', warning => {
@ -15,23 +15,13 @@ process.on('exit', async function() {
if (container) await container.rm();
});
let container;
async function test() {
let isCI = false;
if (process.argv[2] === 'ci')
isCI = true;
container = new Docker();
await container.run(isCI);
dataSources = JSON.parse(JSON.stringify(dataSources));
Object.assign(dataSources.vn, {
host: container.dbConf.host,
port: container.dbConf.port
});
const bootOptions = {dataSources};
Outdated
Review

Quitar

Quitar
const app = require('vn-loopback/server/server');
await new Promise((resolve, reject) => {
@ -77,7 +67,6 @@ async function test() {
await jasmine.execute();
if (app) await app.disconnect();
if (container) await container.rm();
console.log('App disconnected & container removed');
}

3
db/reset.sh Normal file
View File

@ -0,0 +1,3 @@
docker compose -f docker-compose.local.yml down db
docker compose -f docker-compose.local.yml build db $1
docker compose -f docker-compose.local.yml up db -d

50
docker-compose.local.yml Normal file
View File

@ -0,0 +1,50 @@
version: "3.7"
services:
db:
image: db:local
restart: unless-stopped
build:
context: db
dockerfile: Dockerfile
ports:
- 3306:3306
front:
image: front
command: ["npx", "gulp", "front"]
restart: unless-stopped
build:
context: .
dockerfile: front/Dockerfile
target: local
ports:
- 5000:5000
depends_on:
- back
volumes:
- ./modules:/salix/modules
- ./dist:/salix/dist
- ./front:/salix/front
- ./loopback:/salix/loopback
- ./node_modules:/salix/node_modules
back:
image: back
command: ["npx", "gulp", "backWatch"]
restart: unless-stopped
build:
context: .
dockerfile: back/Dockerfile
target: local
ports:
- 3000:3000
environment:
- NODE_ENV
depends_on:
- db
volumes:
- ./loopback:/salix/loopback
- ./storage:/salix/storage
- ./modules:/salix/modules
- ./back:/salix/back
- ./node_modules:/salix/node_modules
- ./print:/salix/print

28
docker-compose.test.yml Normal file
View File

@ -0,0 +1,28 @@
version: "3.7"
services:
db:
image: db:test
restart: always
build:
context: db
dockerfile: Dockerfile
front:
image: front
restart: always
command: ["npx", "jest", "--ci", "--maxWorkers=2"]
build:
context: .
dockerfile: front/Dockerfile
target: test
back:
image: back
restart: always
command: ["node", "back/tests.js", "ci"]
build:
context: .
dockerfile: back/Dockerfile
target: test
environment:
- NODE_ENV
depends_on:
- db

6
e2e/Dockerfile Normal file
View File

@ -0,0 +1,6 @@
FROM front
Review

Este el borrare era un intento de poder tirar els e2e

Este el borrare era un intento de poder tirar els e2e
WORKDIR /salix
COPY e2e e2e
CMD ["node", "e2e/helpers/tests.js"]

View File

@ -4,7 +4,7 @@ require('regenerator-runtime/runtime');
require('vn-loopback/server/boot/date')();
const axios = require('axios');
const Docker = require('../../db/docker.js');
// const Docker = require('../../db/docker.js');
Review

Quitar antes de subir

Quitar antes de subir
const e2eConfig = require('./config.js');
const log = require('fancy-log');
@ -18,9 +18,9 @@ async function test() {
if (process.argv[2] === 'show')
process.env.E2E_SHOW = true;
const container = new Docker('salix-db');
// const container = new Docker('salix-db');
await container.run();
// await container.run();
const Jasmine = require('jasmine');
const jasmine = new Jasmine();

View File

@ -16,3 +16,36 @@ RUN rm sites-enabled/default && ln -s ../sites-available/salix sites-enabled/sal
COPY dist /salix/dist
CMD ["nginx", "-g", "daemon off;"]
FROM back AS local
EXPOSE 5000
WORKDIR /salix
COPY /front/gulpfile.js ./
COPY /front/webpack.config.js ./
COPY /front/package.json ./front/
CMD ["npx", "gulp", "front"]
FROM back AS test
EXPOSE 5000
WORKDIR /salix
COPY front front
# RUN npm i -g jest
Review

Quitar antes de mergear

Quitar antes de mergear
RUN cd front && npm install --ci
COPY modules modules
# COPY dist dist
COPY jest-front.js ./
COPY jest.front.config.js ./
COPY babel.config.js ./
COPY fileMock.js ./
COPY /front/gulpfile.js ./
COPY /front/webpack.config.js ./
COPY /front/salix ./front/salix
COPY /front/core ./front/core
CMD ["npx", "gulp", "front"]

178
front/gulpfile.js Normal file
View File

@ -0,0 +1,178 @@
require('require-yaml');
const gulp = require('gulp');
const PluginError = require('plugin-error');
const argv = require('minimist')(process.argv.slice(2));
const log = require('fancy-log');
// Configuration
if (argv.NODE_ENV)
process.env.NODE_ENV = argv.NODE_ENV;
let langs = ['es', 'en'];
let srcDir = './front';
let modulesDir = './modules';
let buildDir = 'dist';
// Development
const localesRoutes = gulp.parallel(locales, routes);
localesRoutes.description = `Builds locales and routes`;
const front = gulp.series(clean, gulp.parallel(localesRoutes, watch, webpackDevServer));
front.description = `Starts frontend service`;
function clean() {
const del = require('del');
const files = [
`${buildDir}/*`
];
return del(files, {force: true});
}
clean.description = `Cleans all files generated by the 'build' task`;
// Webpack
function webpack(done) {
const webpackCompile = require('webpack');
const merge = require('webpack-merge');
let wpConfig = require('./webpack.config.js');
wpConfig = merge(wpConfig, {});
let compiler = webpackCompile(wpConfig);
compiler.run(function(err, stats) {
if (err) throw new PluginError('webpack', err);
log('[webpack]', stats.toString(wpConfig.stats));
done();
});
}
webpack.description = `Transpiles application into files`;
function webpackDevServer(done) {
const webpack = require('webpack');
const merge = require('webpack-merge');
const WebpackDevServer = require('webpack-dev-server');
let wpConfig = require('./webpack.config.js');
wpConfig = merge(wpConfig, {});
let devServer = wpConfig.devServer;
for (let entryName in wpConfig.entry) {
let entry = wpConfig.entry[entryName];
if (!Array.isArray(entry))
entry = [entry];
let wdsAssets = [
`webpack-dev-server/client?http://localhost:${devServer.port}/`,
`webpack/hot/dev-server`
];
wpConfig.entry[entryName] = wdsAssets.concat(entry);
}
let compiler = webpack(wpConfig);
new WebpackDevServer(compiler, wpConfig.devServer)
.listen(devServer.port, devServer.host, function(err) {
if (err) throw new PluginError('webpack-dev-server', err);
// XXX: Keep the server alive or continue?
done();
});
}
webpackDevServer.description = `Transpiles application into memory`;
// Locale
let localeFiles = [
`${srcDir}/**/locale/*.yml`,
`${modulesDir}/*/front/**/locale/*.yml`
];
/**
* Mixes all locale files into one JSON file per module and language. It looks
* recursively in all project directories for locale folders with per language
* yaml translation files.
*
* @return {Stream} The merged gulp streams
*/
function locales() {
const mergeJson = require('gulp-merge-json');
const gulpFile = require('gulp-file');
const yaml = require('gulp-yaml');
const merge = require('merge-stream');
const fs = require('fs-extra');
let streams = [];
let localePaths = [];
let modules = fs.readdirSync(modulesDir);
for (let mod of modules)
localePaths[mod] = `${modulesDir}/${mod}`;
let baseMods = ['core', 'salix'];
for (let mod of baseMods)
localePaths[mod] = `${srcDir}/${mod}`;
for (let mod in localePaths) {
let path = localePaths[mod];
for (let lang of langs) {
let localeFiles = `${path}/**/locale/${lang}.yml`;
streams.push(gulp.src(localeFiles)
.pipe(yaml())
.pipe(mergeJson({fileName: `${lang}.json`}))
.pipe(gulp.dest(`${buildDir}/locale/${mod}`)));
}
}
for (let mod in localePaths) {
for (let lang of langs) {
let file = `${buildDir}/locale/${mod}/${lang}.json`;
if (fs.existsSync(file)) continue;
streams.push(gulpFile('en.json', '{}', {src: true})
.pipe(gulp.dest(`${buildDir}/locale/${mod}`)));
}
}
return merge(streams);
}
locales.description = `Generates client locale files`;
// Routes
let routeFiles = `${modulesDir}/*/front/routes.json`;
function routes() {
const concat = require('gulp-concat');
const wrap = require('gulp-wrap');
return gulp.src(routeFiles)
.pipe(concat('routes.js', {newLine: ','}))
.pipe(wrap('var routes = [<%=contents%>\n];'))
.pipe(gulp.dest(buildDir));
}
routes.description = 'Merges all module routes file into one file';
// Watch
function watch(done) {
gulp.watch(routeFiles, gulp.series(routes));
gulp.watch(localeFiles, gulp.series(locales));
done();
}
watch.description = `Watches for changes in routes and locale files`;
const build = gulp.series(clean, gulp.parallel(localesRoutes, webpack));
build.description = `Generates binaries and configuration files`;
module.exports = {
front,
build,
clean,
webpack,
webpackDevServer,
routes,
locales,
localesRoutes,
watch
};

View File

@ -109,7 +109,8 @@ let baseConfig = {
modules: false,
children: false,
entrypoints: false,
colors: true
colors: true,
errorDetails: true
}
};
@ -146,9 +147,9 @@ let devConfig = {
inline: true,
stats: baseConfig.stats,
proxy: {
'/api': 'http://localhost:3000',
'/api': 'http://back:3000',
'/*/api/**': {
target: 'http://localhost:3000',
target: 'http://back:3000',
pathRewrite: {'^/[\\w-]+': ''}
}
}

View File

@ -3,7 +3,6 @@ const gulp = require('gulp');
const PluginError = require('plugin-error');
const argv = require('minimist')(process.argv.slice(2));
const log = require('fancy-log');
const Docker = require('./db/docker.js');
// Configuration
@ -60,6 +59,13 @@ function backWatch(done) {
}
backWatch.description = `Starts backend in watcher mode`;
function backTest(done) {
let app = require(`./loopback/server/server`);
app.start();
app.on('started', done);
}
backOnly.description = `Starts backend service`;
const back = gulp.series(dockerStart, backWatch);
back.description = `Starts backend and database service`;

7
init.sh Normal file
View File

@ -0,0 +1,7 @@
sudo docker run --rm -it \
Review

Genera los node_modules sin tener node instalado

Genera los node_modules sin tener node instalado
-w /salix \
-v $PWD/../salix:/salix \
node:20-bullseye \
npm i -g gulp-cli && \
npm i && \
gulp i

View File

@ -7,7 +7,7 @@
"connector": "vn-mysql",
"database": "vn",
"debug": false,
"host": "localhost",
"host": "db",
"port": "3306",
"username": "root",
"password": "root",

View File

@ -1,10 +1,8 @@
{
"name": "RoutesMonitor",
"base": "Loggable",
"options": {
"mysql": {
"table": "routesMonitor"
}
"base": "VnModel",
"mixins": {
"Loggable": true
},
"properties": {
"routeFk": {

1497
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -16,6 +16,7 @@
"base64url": "^3.0.1",
"bcrypt": "^5.0.1",
"bmp-js": "^0.1.0",
"chrome": "^0.1.0",
"compression": "^1.7.3",
"form-data": "^4.0.0",
"fs-extra": "^5.0.0",
@ -41,7 +42,7 @@
"node-ssh": "^11.0.0",
"object-diff": "0.0.4",
"object.pick": "^1.3.0",
"puppeteer": "^20.3.0",
"puppeteer": "^21.7.0",
"read-chunk": "^3.2.0",
"require-yaml": "0.0.1",
"smbhash": "0.0.1",
@ -107,13 +108,14 @@
},
"scripts": {
"dbtest": "nodemon -q db/tests.js -w db/tests",
"test:back": "nodemon -q back/tests.js --config back/nodemonConfig.json",
"test:back:ci": "node back/tests.js ci",
"test:back": "sh runTest.sh back",
"test:e2e": "node e2e/helpers/tests.js",
"test:front": "jest --watch",
"test:front": "sh runTest.sh front",
"test:front:ci": "npx jest --ci --maxWorkers=2",
"back": "nodemon --inspect -w modules ./node_modules/gulp/bin/gulp.js back",
"lint": "eslint ./ --cache --ignore-pattern .gitignore",
"docker": "docker build --progress=plain -t salix-db ./db"
"docker": "docker build --progress=plain -t salix-db ./db",
"start": "docker compose -f docker-compose.local.yml up"
},
"jest": {
"projects": [

2737
print/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

10
runTest.sh Normal file
View File

@ -0,0 +1,10 @@
if [ $1 = "front" ]; then
docker-compose -p salix-testing -f docker-compose.test.yml build back
fi
docker-compose -p salix-testing -f docker-compose.test.yml build $1
if [ $1 = "back" ]; then
docker-compose -p salix-testing -f docker-compose.test.yml build db
fi
docker-compose -p salix-testing -f docker-compose.test.yml run --rm $1
docker-compose -p salix-testing -f docker-compose.test.yml rm -s -v -f