From ca47fe68275a84c2ea75f88e114385622978fe0d Mon Sep 17 00:00:00 2001 From: joan Date: Thu, 28 Jan 2021 12:23:18 +0100 Subject: [PATCH] 2476 - downloadImages() method refactor --- back/models/image.js | 12 +++- .../10280-valentineDay/00-itemImageQueue.sql | 2 + .../item-image-queue/downloadImages.js | 71 +++++++++++-------- .../item/back/models/item-image-queue.json | 15 +++- 4 files changed, 66 insertions(+), 34 deletions(-) create mode 100644 db/changes/10280-valentineDay/00-itemImageQueue.sql diff --git a/back/models/image.js b/back/models/image.js index 78d159940..6723b59d2 100644 --- a/back/models/image.js +++ b/back/models/image.js @@ -57,7 +57,11 @@ module.exports = Self => { await sharp(srcFilePath, {failOnError: false}) .resize(collection.maxWidth, collection.maxHeight, resizeOpts) .png() - .toFile(dstFile); + .toFile(dstFile) + .catch(error => { + console.log('[ERROR] Image download failed: ' + error); + console.log('File => ' + srcFilePath); + }); const sizes = collection.sizes(); for (let size of sizes) { @@ -72,7 +76,11 @@ module.exports = Self => { await sharp(srcFilePath, {failOnError: false}) .resize(size.width, size.height, resizeOpts) .png() - .toFile(dstFile); + .toFile(dstFile) + .catch(error => { + console.log('[ERROR] Image download failed: ' + error); + console.log('File => ' + srcFilePath); + }); } const model = models[collection.model]; diff --git a/db/changes/10280-valentineDay/00-itemImageQueue.sql b/db/changes/10280-valentineDay/00-itemImageQueue.sql new file mode 100644 index 000000000..41c55552d --- /dev/null +++ b/db/changes/10280-valentineDay/00-itemImageQueue.sql @@ -0,0 +1,2 @@ +ALTER TABLE `vn`.`itemImageQueue` + ADD attempts INT default 0 NULL AFTER error; diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index ce52c103b..d99109738 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -18,39 +18,37 @@ module.exports = Self => { Self.downloadImages = async() => { const models = Self.app.models; + const container = await models.TempContainer.container('salix-image'); + const tempPath = path.join(container.client.root, container.name); + const maxAttempts = 3; - try { - const tempPath = path.join('/tmp/salix-image'); + prune(); + download(); - // Create temporary path - await fs.mkdir(tempPath, {recursive: true}); + async function download() { + const image = await Self.findOne({ + where: {url: {neq: null}, attempts: {lt: maxAttempts}}, + order: 'attempts, updated' + }); - const timer = setInterval(async() => { - const image = await Self.findOne({ - where: {error: null, url: {neq: null}} - }); + if (!image) return; - // Exit loop - if (!image) return clearInterval(timer); + const srcFile = image.url.split('/').pop(); + const dotIndex = srcFile.lastIndexOf('.'); + const fileName = srcFile.substring(0, dotIndex); + const file = `${fileName}.png`; + const filePath = path.join(tempPath, file); - const srcFile = image.url.split('/').pop(); - const fileName = srcFile.split('.')[0]; - const file = `${fileName}.png`; - const filePath = path.join(tempPath, file); + https.get(image.url, async response => { + if (response.statusCode != 200) { + const error = new Error(`Could not download the image. Status code ${response.statusCode}`); + + return await errorHandler(image.itemFk, error, filePath); + } const writeStream = fs.createWriteStream(filePath); writeStream.on('open', () => { - https.get(image.url, async response => { - if (response.statusCode != 200) { - const error = new Error(`Could not download the image. Status code ${response.statusCode}`); - - return await errorHandler(image.itemFk, error, filePath); - } - - response.pipe(writeStream); - }).on('error', async error => { - await errorHandler(image.itemFk, error, filePath); - }); + response.pipe(writeStream); }); writeStream.on('error', async error => { @@ -58,16 +56,23 @@ module.exports = Self => { }); writeStream.on('finish', async function() { + writeStream.end(); + }); + + writeStream.on('close', async function() { + console.log('stream closed'); try { await models.Image.registerImage('catalog', filePath, fileName, image.itemFk); await image.destroy(); + + download(); } catch (error) { await errorHandler(image.itemFk, error, filePath); } }); - }, 1000); - } catch (error) { - throw new Error('Try-catch error: ', error); + }).on('error', async error => { + await errorHandler(image.itemFk, error, filePath); + }); } async function errorHandler(rowId, error, filePath) { @@ -77,10 +82,18 @@ module.exports = Self => { if (!row) throw new Error(`Could not update due error ${error}`); - await row.updateAttribute('error', error); + if (row.attempts < maxAttempts) { + await row.updateAttributes({ + error: error, + attempts: row.attempts + 1, + updated: new Date() + }); + } if (filePath && fs.existsSync(filePath)) await fs.unlink(filePath); + + download(); } catch (err) { throw new Error(`ErrorHandler error: ${err}`); } diff --git a/modules/item/back/models/item-image-queue.json b/modules/item/back/models/item-image-queue.json index 6e248ac96..a4b65b204 100644 --- a/modules/item/back/models/item-image-queue.json +++ b/modules/item/back/models/item-image-queue.json @@ -9,17 +9,26 @@ }, "properties": { "itemFk": { - "type": "Number", + "type": "number", "id": true, "description": "Identifier" }, "url": { - "type": "String", + "type": "string", "required": true }, "error": { - "type": "String", + "type": "string", "required": true + }, + "attempts": { + "type": "number" + }, + "created": { + "type": "date" + }, + "updated": { + "type": "date" } }, "relations": {