From a947666838f302d085f8616190609f31f984057d Mon Sep 17 00:00:00 2001 From: joan Date: Wed, 15 Mar 2023 14:24:30 +0100 Subject: [PATCH] refactor(imageQueue): remplaced sharp with gm --- Dockerfile | 1 + .../back/methods/item-image-queue/download.js | 181 ++++++++++++++++++ modules/item/back/models/item-image-queue.js | 2 +- package-lock.json | 143 +++++++++++++- package.json | 1 + 5 files changed, 325 insertions(+), 3 deletions(-) create mode 100644 modules/item/back/methods/item-image-queue/download.js diff --git a/Dockerfile b/Dockerfile index a59725f770..ee87cd0d05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update \ curl \ ca-certificates \ gnupg2 \ + graphicsmagick \ && curl -fsSL https://deb.nodesource.com/setup_14.x | bash - \ && apt-get install -y --no-install-recommends nodejs \ && npm install -g npm@8.19.2 diff --git a/modules/item/back/methods/item-image-queue/download.js b/modules/item/back/methods/item-image-queue/download.js new file mode 100644 index 0000000000..527f9ba65f --- /dev/null +++ b/modules/item/back/methods/item-image-queue/download.js @@ -0,0 +1,181 @@ +const axios = require('axios'); +const uuid = require('uuid'); +const fs = require('fs/promises'); +const {createWriteStream} = require('fs'); +const path = require('path'); +const gm = require('gm'); + +module.exports = Self => { + Self.remoteMethod('download', { + description: 'Processes the image download queue', + accessType: 'WRITE', + http: { + path: `/download`, + verb: 'POST' + } + }); + + Self.download = async() => { + const models = Self.app.models; + const tempContainer = await models.TempContainer.container('salix-image'); + const tempPath = path.join(tempContainer.client.root, tempContainer.name); + const maxAttempts = 3; + const collectionName = 'catalog'; + + const tx = await Self.beginTransaction({}); + + let tempFilePath; + let queueRow; + try { + const myOptions = {transaction: tx}; + + queueRow = await Self.findOne({ + fields: [ + 'id', + 'itemFk', + 'url', + 'attempts' + ], + where: { + url: {neq: null}, + attempts: { + lt: maxAttempts + } + }, + order: 'priority, attempts, updated' + }, myOptions); + + if (!queueRow) return; + + const collection = await models.ImageCollection.findOne({ + fields: [ + 'id', + 'maxWidth', + 'maxHeight', + 'model', + 'property' + ], + where: {name: collectionName}, + include: { + relation: 'sizes', + scope: { + fields: ['width', 'height', 'crop'] + } + } + }, myOptions); + + const fileName = `${uuid.v4()}.png`; + tempFilePath = path.join(tempPath, fileName); + + // Insert image row + await models.Image.create({ + name: fileName, + collectionFk: collectionName, + updated: Date.vnNow() + }, myOptions); + + // Update item + const model = models[collection.model]; + if (!model) + throw new Error('No matching model found'); + + const item = await model.findById(queueRow.itemFk, null, myOptions); + if (item) { + await item.updateAttribute( + collection.property, + fileName, + myOptions + ); + } + + // Download remote image + const response = await axios.get(queueRow.url, { + responseType: 'stream', + }); + + const writeStream = createWriteStream(tempFilePath); + await new Promise((resolve, reject) => { + writeStream.on('open', () => response.data.pipe(writeStream)); + writeStream.on('finish', () => resolve()); + writeStream.on('error', error => reject(error)); + }); + + // Resize + const container = await models.ImageContainer.container(collectionName); + const rootPath = container.client.root; + const collectionDir = path.join(rootPath, collectionName); + + // To max size + const {maxWidth, maxHeight} = collection; + const fullSizePath = path.join(collectionDir, 'full'); + const toFullSizePath = `${fullSizePath}/${fileName}`; + + await fs.mkdir(fullSizePath, {recursive: true}); + await new Promise((resolve, reject) => { + gm(tempFilePath) + .resize(maxWidth, maxHeight, '>') + .setFormat('png') + .write(toFullSizePath, function(err) { + if (err) reject(err); + if (!err) resolve(); + }); + }); + + // To collection sizes + for (const size of collection.sizes()) { + const {width, height} = size; + + const sizePath = path.join(collectionDir, `${width}x${height}`); + const toSizePath = `${sizePath}/${fileName}`; + + await fs.mkdir(sizePath, {recursive: true}); + await new Promise((resolve, reject) => { + const gmInstance = gm(tempFilePath); + + if (size.crop) { + gmInstance + .resize(width, height, '^') + .gravity('Center') + .crop(width, height); + } + + if (!size.crop) + gmInstance.resize(width, height, '>'); + + gmInstance + .setFormat('png') + .write(toSizePath, function(err) { + if (err) reject(err); + if (!err) resolve(); + }); + }); + } + + if (await fs.stat(tempFilePath)) + await fs.unlink(tempFilePath); + + await queueRow.destroy(myOptions); + + // Restart queue + Self.download(); + + await tx.commit(); + } catch (error) { + await tx.rollback(); + + if (queueRow.attempts < maxAttempts) { + await queueRow.updateAttributes({ + error: error, + attempts: queueRow.attempts + 1, + updated: Date.vnNew() + }); + } + + try { + await fs.unlink(tempFilePath); + } catch (error) {} + + Self.download(); + } + }; +}; diff --git a/modules/item/back/models/item-image-queue.js b/modules/item/back/models/item-image-queue.js index e2059ddac7..35a467693a 100644 --- a/modules/item/back/models/item-image-queue.js +++ b/modules/item/back/models/item-image-queue.js @@ -1,3 +1,3 @@ module.exports = Self => { - require('../methods/item-image-queue/downloadImages')(Self); + require('../methods/item-image-queue/download')(Self); }; diff --git a/package-lock.json b/package-lock.json index 8a39bd9024..7e86dbba9c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "salix-back", - "version": "23.04.01", + "version": "23.08.01", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "salix-back", - "version": "23.04.01", + "version": "23.08.01", "license": "GPL-3.0", "dependencies": { "axios": "^1.2.2", @@ -17,6 +17,7 @@ "form-data": "^4.0.0", "fs-extra": "^5.0.0", "ftps": "^1.2.0", + "gm": "^1.25.0", "got": "^10.7.0", "helmet": "^3.21.2", "i18n": "^0.8.4", @@ -3625,6 +3626,16 @@ "node": ">=0.10.0" } }, + "node_modules/array-parallel": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/array-parallel/-/array-parallel-0.1.3.tgz", + "integrity": "sha512-TDPTwSWW5E4oiFiKmz6RGJ/a80Y91GuLgUYuLd49+XBS75tYo8PNgaT2K/OxuQYqkoI852MDGBorg9OcUSTQ8w==" + }, + "node_modules/array-series": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/array-series/-/array-series-0.1.5.tgz", + "integrity": "sha512-L0XlBwfx9QetHOsbLDrE/vh2t018w9462HM3iaFfxRiK83aJjAt/Ja3NMkOW7FICwWTlQBa3ZbL5FKhuQWkDrg==" + }, "node_modules/array-slice": { "version": "1.1.0", "dev": true, @@ -9284,6 +9295,67 @@ "node": ">= 0.10" } }, + "node_modules/gm": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/gm/-/gm-1.25.0.tgz", + "integrity": "sha512-4kKdWXTtgQ4biIo7hZA396HT062nDVVHPjQcurNZ3o/voYN+o5FUC5kOwuORbpExp3XbTJ3SU7iRipiIhQtovw==", + "dependencies": { + "array-parallel": "~0.1.3", + "array-series": "~0.1.5", + "cross-spawn": "^4.0.0", + "debug": "^3.1.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gm/node_modules/cross-spawn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", + "integrity": "sha512-yAXz/pA1tD8Gtg2S98Ekf/sewp3Lcp3YoFKJ4Hkp5h5yLWnKVTDU0kwjKJ8NDCYcfTLfyGkzTikst+jWypT1iA==", + "dependencies": { + "lru-cache": "^4.0.1", + "which": "^1.2.9" + } + }, + "node_modules/gm/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/gm/node_modules/lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dependencies": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "node_modules/gm/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/gm/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/gm/node_modules/yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" + }, "node_modules/google-auth-library": { "version": "3.1.2", "license": "Apache-2.0", @@ -28673,6 +28745,16 @@ } } }, + "array-parallel": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/array-parallel/-/array-parallel-0.1.3.tgz", + "integrity": "sha512-TDPTwSWW5E4oiFiKmz6RGJ/a80Y91GuLgUYuLd49+XBS75tYo8PNgaT2K/OxuQYqkoI852MDGBorg9OcUSTQ8w==" + }, + "array-series": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/array-series/-/array-series-0.1.5.tgz", + "integrity": "sha512-L0XlBwfx9QetHOsbLDrE/vh2t018w9462HM3iaFfxRiK83aJjAt/Ja3NMkOW7FICwWTlQBa3ZbL5FKhuQWkDrg==" + }, "array-slice": { "version": "1.1.0", "dev": true @@ -32611,6 +32693,63 @@ "sparkles": "^1.0.0" } }, + "gm": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/gm/-/gm-1.25.0.tgz", + "integrity": "sha512-4kKdWXTtgQ4biIo7hZA396HT062nDVVHPjQcurNZ3o/voYN+o5FUC5kOwuORbpExp3XbTJ3SU7iRipiIhQtovw==", + "requires": { + "array-parallel": "~0.1.3", + "array-series": "~0.1.5", + "cross-spawn": "^4.0.0", + "debug": "^3.1.0" + }, + "dependencies": { + "cross-spawn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", + "integrity": "sha512-yAXz/pA1tD8Gtg2S98Ekf/sewp3Lcp3YoFKJ4Hkp5h5yLWnKVTDU0kwjKJ8NDCYcfTLfyGkzTikst+jWypT1iA==", + "requires": { + "lru-cache": "^4.0.1", + "which": "^1.2.9" + } + }, + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "requires": { + "isexe": "^2.0.0" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" + } + } + }, "google-auth-library": { "version": "3.1.2", "requires": { diff --git a/package.json b/package.json index f4415f99b0..9d782bcc56 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "form-data": "^4.0.0", "fs-extra": "^5.0.0", "ftps": "^1.2.0", + "gm": "^1.25.0", "got": "^10.7.0", "helmet": "^3.21.2", "i18n": "^0.8.4",