From 378400c6d2c8447e743e77ef69d8bca7ac3b2b1f Mon Sep 17 00:00:00 2001 From: Juan Ferrer Toribio Date: Mon, 17 Apr 2023 18:19:25 +0200 Subject: [PATCH] refs #5553 Image scrubing method improved Remove & dryRun options, check that collection exists, performance improved, exclusive lock --- back/methods/image/scrub.js | 135 +++++++++++++++++++++++++++--------- loopback/locale/es.json | 6 +- 2 files changed, 105 insertions(+), 36 deletions(-) diff --git a/back/methods/image/scrub.js b/back/methods/image/scrub.js index a9c2c6125..893c536ff 100644 --- a/back/methods/image/scrub.js +++ b/back/methods/image/scrub.js @@ -1,9 +1,10 @@ const fs = require('fs-extra'); const path = require('path'); +const UserError = require('vn-loopback/util/user-error'); module.exports = Self => { Self.remoteMethod('scrub', { - description: 'Cleans collection images directory', + description: 'Deletes images without database reference', accessType: 'WRITE', accepts: [ { @@ -11,14 +12,22 @@ module.exports = Self => { type: 'string', description: 'The collection name', required: true + }, { + arg: 'remove', + type: 'boolean', + description: 'Delete instead of move images to trash' }, { arg: 'limit', - type: 'number', - description: 'Maximun number of image to clean' + type: 'integer', + description: 'Maximum number of images to clean' + }, { + arg: 'dryRun', + type: 'boolean', + description: 'Simulate actions' } ], returns: { - type: 'number', + type: 'integer', root: true }, http: { @@ -27,44 +36,102 @@ module.exports = Self => { } }); - Self.scrub = async function(collection, limit) { - const models = Self.app.models; - const container = await models.ImageContainer.container( - collection - ); + Self.scrub = async function(collection, remove, limit, dryRun) { + const $ = Self.app.models; + + const env = process.env.NODE_ENV; + dryRun = dryRun || (env && env !== 'production'); + + const instance = await $.ImageCollection.findOne({ + fields: ['id'], + where: {name: collection} + }); + if (!instance) + throw new UserError('Collection does not exist'); + + const container = await $.ImageContainer.container(collection); const rootPath = container.client.root; - const now = Date.vnNew().toJSON(); - const scrubDir = path.join(rootPath, '.scrub', now); + const conn = await getConnection(); + const lockName = 'salix.Image.scrub'; + let lockObtained; - const collectionDir = path.join(rootPath, collection); - const sizes = await fs.readdir(collectionDir); - let cleanCount = 0; + try { + const [row] = await query(conn, + `SELECT GET_LOCK(?, 0) hasLock`, + [lockName] + ); + lockObtained = !!row.hasLock; + if (!lockObtained) + throw new UserError('Cannot obtain exclusive lock'); - mainLoop: for (const size of sizes) { - const sizeDir = path.join(collectionDir, size); - const scrubSizeDir = path.join(scrubDir, collection, size); - const images = await fs.readdir(sizeDir); - for (const image of images) { - const imageName = path.parse(image).name; - const count = await models.Image.count({ - collectionFk: collection, - name: imageName - }); - const exists = count > 0; - if (!exists) { - cleanCount++; - const srcDir = path.join(sizeDir, image); - const dstDir = path.join(scrubSizeDir, image); - await fs.mkdir(scrubSizeDir, {recursive: true}); - await fs.rename(srcDir, dstDir); + const now = Date.vnNew().toJSON(); + const scrubDir = path.join(rootPath, '.scrub', now); - if (limit && cleanCount == limit) - break mainLoop; + const collectionDir = path.join(rootPath, collection); + const sizes = await fs.readdir(collectionDir); + let cleanCount = 0; + + mainLoop: for (const size of sizes) { + const sizeDir = path.join(collectionDir, size); + const scrubSizeDir = path.join(scrubDir, collection, size); + const images = await fs.readdir(sizeDir); + for (const image of images) { + const imageName = path.parse(image).name; + const count = await Self.count({ + collectionFk: collection, + name: imageName + }); + const exists = count > 0; + let scrubDirCreated = false; + if (!exists) { + const srcFile = path.join(sizeDir, image); + if (remove !== true) { + if (!scrubDirCreated) { + if (!dryRun) + await fs.mkdir(scrubSizeDir, {recursive: true}); + scrubDirCreated = true; + } + const dstFile = path.join(scrubSizeDir, image); + if (!dryRun) + await fs.rename(srcFile, dstFile); + } else { + if (!dryRun) + await fs.unlink(srcFile); + } + + cleanCount++; + if (limit && cleanCount == limit) + break mainLoop; + } } } + + return cleanCount; + } finally { + if (lockObtained) + await query(conn, `DO RELEASE_LOCK(?)`, [lockName]); + conn.release(); } - return cleanCount; + // Promisified datasource functions + + function getConnection() { + return new Promise((resolve, reject) => { + Self.dataSource.connector.client.getConnection((err, conn) => { + if (err) return reject(err); + resolve(conn); + }); + }); + } + + function query(conn, sql, params) { + return new Promise((resolve, reject) => { + conn.query(sql, params, (err, res) => { + if (err) return reject(err); + resolve(res); + }); + }); + } }; }; diff --git a/loopback/locale/es.json b/loopback/locale/es.json index 95bf16d66..e4dc74aa3 100644 --- a/loopback/locale/es.json +++ b/loopback/locale/es.json @@ -268,8 +268,10 @@ "Exists an invoice with a previous date": "Existe una factura con fecha anterior", "Invoice date can't be less than max date": "La fecha de factura no puede ser inferior a la fecha límite", "Warehouse inventory not set": "El almacén inventario no está establecido", - "This locker has already been assigned": "Esta taquilla ya ha sido asignada", + "This locker has already been assigned": "Esta taquilla ya ha sido asignada", "Tickets with associated refunds": "No se pueden borrar tickets con abonos asociados. Este ticket está asociado al abono Nº {{id}}", "Not exist this branch": "La rama no existe", - "This ticket cannot be signed because it has not been boxed": "Este ticket no puede firmarse porque no ha sido encajado" + "This ticket cannot be signed because it has not been boxed": "Este ticket no puede firmarse porque no ha sido encajado", + "Collection does not exist": "La colección no existe", + "Cannot obtain exclusive lock": "No se puede obtener un bloqueo exclusivo" }