refactor(imageQueue): remplaced sharp with gm

2023-03-15 14:24:30 +01:00 · 2023-03-15 14:24:30 +01:00 · a947666838
parent 1b984f4325
commit a947666838
5 changed files with 325 additions and 3 deletions
--- a/1
+++ b/1
@ -10,6 +10,7 @@ RUN apt-get update \
        curl \
        ca-certificates \
        gnupg2 \
+        graphicsmagick \
    && curl -fsSL https://deb.nodesource.com/setup_14.x | bash - \
    && apt-get install -y --no-install-recommends nodejs \
    && npm install -g npm@8.19.2
--- a/modules/item/back/methods/item-image-queue/download.js
+++ b/modules/item/back/methods/item-image-queue/download.js
@ -0,0 +1,181 @@
+const axios = require('axios');
+const uuid = require('uuid');
+const fs = require('fs/promises');
+const {createWriteStream} = require('fs');
+const path = require('path');
+const gm = require('gm');
+
+module.exports = Self => {
+    Self.remoteMethod('download', {
+        description: 'Processes the image download queue',
+        accessType: 'WRITE',
+        http: {
+            path: `/download`,
+            verb: 'POST'
+        }
+    });
+
+    Self.download = async() => {
+        const models = Self.app.models;
+        const tempContainer = await models.TempContainer.container('salix-image');
+        const tempPath = path.join(tempContainer.client.root, tempContainer.name);
+        const maxAttempts = 3;
+        const collectionName = 'catalog';
+
+        const tx = await Self.beginTransaction({});
+
+        let tempFilePath;
+        let queueRow;
+        try {
+            const myOptions = {transaction: tx};
+
+            queueRow = await Self.findOne({
+                fields: [
+                    'id',
+                    'itemFk',
+                    'url',
+                    'attempts'
+                ],
+                where: {
+                    url: {neq: null},
+                    attempts: {
+                        lt: maxAttempts
+                    }
+                },
+                order: 'priority, attempts, updated'
+            }, myOptions);
+
+            if (!queueRow) return;
+
+            const collection = await models.ImageCollection.findOne({
+                fields: [
+                    'id',
+                    'maxWidth',
+                    'maxHeight',
+                    'model',
+                    'property'
+                ],
+                where: {name: collectionName},
+                include: {
+                    relation: 'sizes',
+                    scope: {
+                        fields: ['width', 'height', 'crop']
+                    }
+                }
+            }, myOptions);
+
+            const fileName = `${uuid.v4()}.png`;
+            tempFilePath = path.join(tempPath, fileName);
+
+            // Insert image row
+            await models.Image.create({
+                name: fileName,
+                collectionFk: collectionName,
+                updated: Date.vnNow()
+            }, myOptions);
+
+            // Update item
+            const model = models[collection.model];
+            if (!model)
+                throw new Error('No matching model found');
+
+            const item = await model.findById(queueRow.itemFk, null, myOptions);
+            if (item) {
+                await item.updateAttribute(
+                    collection.property,
+                    fileName,
+                    myOptions
+                );
+            }
+
+            // Download remote image
+            const response = await axios.get(queueRow.url, {
+                responseType: 'stream',
+            });
+
+            const writeStream = createWriteStream(tempFilePath);
+            await new Promise((resolve, reject) => {
+                writeStream.on('open', () => response.data.pipe(writeStream));
+                writeStream.on('finish', () => resolve());
+                writeStream.on('error', error => reject(error));
+            });
+
+            // Resize
+            const container = await models.ImageContainer.container(collectionName);
+            const rootPath = container.client.root;
+            const collectionDir = path.join(rootPath, collectionName);
+
+            // To max size
+            const {maxWidth, maxHeight} = collection;
+            const fullSizePath = path.join(collectionDir, 'full');
+            const toFullSizePath = `${fullSizePath}/${fileName}`;
+
+            await fs.mkdir(fullSizePath, {recursive: true});
+            await new Promise((resolve, reject) => {
+                gm(tempFilePath)
+                    .resize(maxWidth, maxHeight, '>')
+                    .setFormat('png')
+                    .write(toFullSizePath, function(err) {
+                        if (err) reject(err);
+                        if (!err) resolve();
+                    });
+            });
+
+            // To collection sizes
+            for (const size of collection.sizes()) {
+                const {width, height} = size;
+
+                const sizePath = path.join(collectionDir, `${width}x${height}`);
+                const toSizePath = `${sizePath}/${fileName}`;
+
+                await fs.mkdir(sizePath, {recursive: true});
+                await new Promise((resolve, reject) => {
+                    const gmInstance = gm(tempFilePath);
+
+                    if (size.crop) {
+                        gmInstance
+                            .resize(width, height, '^')
+                            .gravity('Center')
+                            .crop(width, height);
+                    }
+
+                    if (!size.crop)
+                        gmInstance.resize(width, height, '>');
+
+                    gmInstance
+                        .setFormat('png')
+                        .write(toSizePath, function(err) {
+                            if (err) reject(err);
+                            if (!err) resolve();
+                        });
+                });
+            }
+
+            if (await fs.stat(tempFilePath))
+                await fs.unlink(tempFilePath);
+
+            await queueRow.destroy(myOptions);
+
+            // Restart queue
+            Self.download();
+
+            await tx.commit();
+        } catch (error) {
+            await tx.rollback();
+
+            if (queueRow.attempts < maxAttempts) {
+                await queueRow.updateAttributes({
+                    error: error,
+                    attempts: queueRow.attempts + 1,
+                    updated: Date.vnNew()
+                });
+            }
+
+            try {
+                await fs.unlink(tempFilePath);
+            } catch (error) {}
+
+            Self.download();
+        }
+    };
+};
--- a/modules/item/back/models/item-image-queue.js
+++ b/modules/item/back/models/item-image-queue.js
@ -1,3 +1,3 @@
 module.exports = Self => {
-    require('../methods/item-image-queue/downloadImages')(Self);
+    require('../methods/item-image-queue/download')(Self);
 };
--- a/package-lock.json
+++ b/package-lock.json
@ -1,12 +1,12 @@
 {
    "name": "salix-back",
-    "version": "23.04.01",
+    "version": "23.08.01",
    "lockfileVersion": 2,
    "requires": true,
    "packages": {
        "": {
            "name": "salix-back",
-            "version": "23.04.01",
+            "version": "23.08.01",
            "license": "GPL-3.0",
            "dependencies": {
                "axios": "^1.2.2",
@ -17,6 +17,7 @@
                "form-data": "^4.0.0",
                "fs-extra": "^5.0.0",
                "ftps": "^1.2.0",
+                "gm": "^1.25.0",
                "got": "^10.7.0",
                "helmet": "^3.21.2",
                "i18n": "^0.8.4",
@ -3625,6 +3626,16 @@
                "node": ">=0.10.0"
            }
        },
+        "node_modules/array-parallel": {
+            "version": "0.1.3",
+            "resolved": "https://registry.npmjs.org/array-parallel/-/array-parallel-0.1.3.tgz",
+            "integrity": "sha512-TDPTwSWW5E4oiFiKmz6RGJ/a80Y91GuLgUYuLd49+XBS75tYo8PNgaT2K/OxuQYqkoI852MDGBorg9OcUSTQ8w=="
+        },
+        "node_modules/array-series": {
+            "version": "0.1.5",
+            "resolved": "https://registry.npmjs.org/array-series/-/array-series-0.1.5.tgz",
+            "integrity": "sha512-L0XlBwfx9QetHOsbLDrE/vh2t018w9462HM3iaFfxRiK83aJjAt/Ja3NMkOW7FICwWTlQBa3ZbL5FKhuQWkDrg=="
+        },
        "node_modules/array-slice": {
            "version": "1.1.0",
            "dev": true,
@ -9284,6 +9295,67 @@
                "node": ">= 0.10"
            }
        },
+        "node_modules/gm": {
+            "version": "1.25.0",
+            "resolved": "https://registry.npmjs.org/gm/-/gm-1.25.0.tgz",
+            "integrity": "sha512-4kKdWXTtgQ4biIo7hZA396HT062nDVVHPjQcurNZ3o/voYN+o5FUC5kOwuORbpExp3XbTJ3SU7iRipiIhQtovw==",
+            "dependencies": {
+                "array-parallel": "~0.1.3",
+                "array-series": "~0.1.5",
+                "cross-spawn": "^4.0.0",
+                "debug": "^3.1.0"
+            },
+            "engines": {
+                "node": ">=14"
+            }
+        },
+        "node_modules/gm/node_modules/cross-spawn": {
+            "version": "4.0.2",
+            "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz",
+            "integrity": "sha512-yAXz/pA1tD8Gtg2S98Ekf/sewp3Lcp3YoFKJ4Hkp5h5yLWnKVTDU0kwjKJ8NDCYcfTLfyGkzTikst+jWypT1iA==",
+            "dependencies": {
+                "lru-cache": "^4.0.1",
+                "which": "^1.2.9"
+            }
+        },
+        "node_modules/gm/node_modules/debug": {
+            "version": "3.2.7",
+            "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+            "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+            "dependencies": {
+                "ms": "^2.1.1"
+            }
+        },
+        "node_modules/gm/node_modules/lru-cache": {
+            "version": "4.1.5",
+            "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
+            "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
+            "dependencies": {
+                "pseudomap": "^1.0.2",
+                "yallist": "^2.1.2"
+            }
+        },
+        "node_modules/gm/node_modules/ms": {
+            "version": "2.1.3",
+            "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+            "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+        },
+        "node_modules/gm/node_modules/which": {
+            "version": "1.3.1",
+            "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
+            "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
+            "dependencies": {
+                "isexe": "^2.0.0"
+            },
+            "bin": {
+                "which": "bin/which"
+            }
+        },
+        "node_modules/gm/node_modules/yallist": {
+            "version": "2.1.2",
+            "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+            "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A=="
+        },
        "node_modules/google-auth-library": {
            "version": "3.1.2",
            "license": "Apache-2.0",
@ -28673,6 +28745,16 @@
                }
            }
        },
+        "array-parallel": {
+            "version": "0.1.3",
+            "resolved": "https://registry.npmjs.org/array-parallel/-/array-parallel-0.1.3.tgz",
+            "integrity": "sha512-TDPTwSWW5E4oiFiKmz6RGJ/a80Y91GuLgUYuLd49+XBS75tYo8PNgaT2K/OxuQYqkoI852MDGBorg9OcUSTQ8w=="
+        },
+        "array-series": {
+            "version": "0.1.5",
+            "resolved": "https://registry.npmjs.org/array-series/-/array-series-0.1.5.tgz",
+            "integrity": "sha512-L0XlBwfx9QetHOsbLDrE/vh2t018w9462HM3iaFfxRiK83aJjAt/Ja3NMkOW7FICwWTlQBa3ZbL5FKhuQWkDrg=="
+        },
        "array-slice": {
            "version": "1.1.0",
            "dev": true
@ -32611,6 +32693,63 @@
                "sparkles": "^1.0.0"
            }
        },
+        "gm": {
+            "version": "1.25.0",
+            "resolved": "https://registry.npmjs.org/gm/-/gm-1.25.0.tgz",
+            "integrity": "sha512-4kKdWXTtgQ4biIo7hZA396HT062nDVVHPjQcurNZ3o/voYN+o5FUC5kOwuORbpExp3XbTJ3SU7iRipiIhQtovw==",
+            "requires": {
+                "array-parallel": "~0.1.3",
+                "array-series": "~0.1.5",
+                "cross-spawn": "^4.0.0",
+                "debug": "^3.1.0"
+            },
+            "dependencies": {
+                "cross-spawn": {
+                    "version": "4.0.2",
+                    "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz",
+                    "integrity": "sha512-yAXz/pA1tD8Gtg2S98Ekf/sewp3Lcp3YoFKJ4Hkp5h5yLWnKVTDU0kwjKJ8NDCYcfTLfyGkzTikst+jWypT1iA==",
+                    "requires": {
+                        "lru-cache": "^4.0.1",
+                        "which": "^1.2.9"
+                    }
+                },
+                "debug": {
+                    "version": "3.2.7",
+                    "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+                    "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+                    "requires": {
+                        "ms": "^2.1.1"
+                    }
+                },
+                "lru-cache": {
+                    "version": "4.1.5",
+                    "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
+                    "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
+                    "requires": {
+                        "pseudomap": "^1.0.2",
+                        "yallist": "^2.1.2"
+                    }
+                },
+                "ms": {
+                    "version": "2.1.3",
+                    "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+                    "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+                },
+                "which": {
+                    "version": "1.3.1",
+                    "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
+                    "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
+                    "requires": {
+                        "isexe": "^2.0.0"
+                    }
+                },
+                "yallist": {
+                    "version": "2.1.2",
+                    "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+                    "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A=="
+                }
+            }
+        },
        "google-auth-library": {
            "version": "3.1.2",
            "requires": {
--- a/package.json
+++ b/package.json
@ -20,6 +20,7 @@
        "form-data": "^4.0.0",
        "fs-extra": "^5.0.0",
        "ftps": "^1.2.0",
+        "gm": "^1.25.0",
        "got": "^10.7.0",
        "helmet": "^3.21.2",
        "i18n": "^0.8.4",