From ca47fe68275a84c2ea75f88e114385622978fe0d Mon Sep 17 00:00:00 2001 From: joan Date: Thu, 28 Jan 2021 12:23:18 +0100 Subject: [PATCH 1/4] 2476 - downloadImages() method refactor --- back/models/image.js | 12 +++- .../10280-valentineDay/00-itemImageQueue.sql | 2 + .../item-image-queue/downloadImages.js | 71 +++++++++++-------- .../item/back/models/item-image-queue.json | 15 +++- 4 files changed, 66 insertions(+), 34 deletions(-) create mode 100644 db/changes/10280-valentineDay/00-itemImageQueue.sql diff --git a/back/models/image.js b/back/models/image.js index 78d159940..6723b59d2 100644 --- a/back/models/image.js +++ b/back/models/image.js @@ -57,7 +57,11 @@ module.exports = Self => { await sharp(srcFilePath, {failOnError: false}) .resize(collection.maxWidth, collection.maxHeight, resizeOpts) .png() - .toFile(dstFile); + .toFile(dstFile) + .catch(error => { + console.log('[ERROR] Image download failed: ' + error); + console.log('File => ' + srcFilePath); + }); const sizes = collection.sizes(); for (let size of sizes) { @@ -72,7 +76,11 @@ module.exports = Self => { await sharp(srcFilePath, {failOnError: false}) .resize(size.width, size.height, resizeOpts) .png() - .toFile(dstFile); + .toFile(dstFile) + .catch(error => { + console.log('[ERROR] Image download failed: ' + error); + console.log('File => ' + srcFilePath); + }); } const model = models[collection.model]; diff --git a/db/changes/10280-valentineDay/00-itemImageQueue.sql b/db/changes/10280-valentineDay/00-itemImageQueue.sql new file mode 100644 index 000000000..41c55552d --- /dev/null +++ b/db/changes/10280-valentineDay/00-itemImageQueue.sql @@ -0,0 +1,2 @@ +ALTER TABLE `vn`.`itemImageQueue` + ADD attempts INT default 0 NULL AFTER error; diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index ce52c103b..d99109738 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -18,39 +18,37 @@ module.exports = Self => { Self.downloadImages = async() => { const models = Self.app.models; + const container = await models.TempContainer.container('salix-image'); + const tempPath = path.join(container.client.root, container.name); + const maxAttempts = 3; - try { - const tempPath = path.join('/tmp/salix-image'); + prune(); + download(); - // Create temporary path - await fs.mkdir(tempPath, {recursive: true}); + async function download() { + const image = await Self.findOne({ + where: {url: {neq: null}, attempts: {lt: maxAttempts}}, + order: 'attempts, updated' + }); - const timer = setInterval(async() => { - const image = await Self.findOne({ - where: {error: null, url: {neq: null}} - }); + if (!image) return; - // Exit loop - if (!image) return clearInterval(timer); + const srcFile = image.url.split('/').pop(); + const dotIndex = srcFile.lastIndexOf('.'); + const fileName = srcFile.substring(0, dotIndex); + const file = `${fileName}.png`; + const filePath = path.join(tempPath, file); - const srcFile = image.url.split('/').pop(); - const fileName = srcFile.split('.')[0]; - const file = `${fileName}.png`; - const filePath = path.join(tempPath, file); + https.get(image.url, async response => { + if (response.statusCode != 200) { + const error = new Error(`Could not download the image. Status code ${response.statusCode}`); + + return await errorHandler(image.itemFk, error, filePath); + } const writeStream = fs.createWriteStream(filePath); writeStream.on('open', () => { - https.get(image.url, async response => { - if (response.statusCode != 200) { - const error = new Error(`Could not download the image. Status code ${response.statusCode}`); - - return await errorHandler(image.itemFk, error, filePath); - } - - response.pipe(writeStream); - }).on('error', async error => { - await errorHandler(image.itemFk, error, filePath); - }); + response.pipe(writeStream); }); writeStream.on('error', async error => { @@ -58,16 +56,23 @@ module.exports = Self => { }); writeStream.on('finish', async function() { + writeStream.end(); + }); + + writeStream.on('close', async function() { + console.log('stream closed'); try { await models.Image.registerImage('catalog', filePath, fileName, image.itemFk); await image.destroy(); + + download(); } catch (error) { await errorHandler(image.itemFk, error, filePath); } }); - }, 1000); - } catch (error) { - throw new Error('Try-catch error: ', error); + }).on('error', async error => { + await errorHandler(image.itemFk, error, filePath); + }); } async function errorHandler(rowId, error, filePath) { @@ -77,10 +82,18 @@ module.exports = Self => { if (!row) throw new Error(`Could not update due error ${error}`); - await row.updateAttribute('error', error); + if (row.attempts < maxAttempts) { + await row.updateAttributes({ + error: error, + attempts: row.attempts + 1, + updated: new Date() + }); + } if (filePath && fs.existsSync(filePath)) await fs.unlink(filePath); + + download(); } catch (err) { throw new Error(`ErrorHandler error: ${err}`); } diff --git a/modules/item/back/models/item-image-queue.json b/modules/item/back/models/item-image-queue.json index 6e248ac96..a4b65b204 100644 --- a/modules/item/back/models/item-image-queue.json +++ b/modules/item/back/models/item-image-queue.json @@ -9,17 +9,26 @@ }, "properties": { "itemFk": { - "type": "Number", + "type": "number", "id": true, "description": "Identifier" }, "url": { - "type": "String", + "type": "string", "required": true }, "error": { - "type": "String", + "type": "string", "required": true + }, + "attempts": { + "type": "number" + }, + "created": { + "type": "date" + }, + "updated": { + "type": "date" } }, "relations": { From 9d5d2e9177a3e8a627465140f08bca56ecaf6c7a Mon Sep 17 00:00:00 2001 From: joan Date: Thu, 28 Jan 2021 15:57:33 +0100 Subject: [PATCH 2/4] Added method prune again --- .../item-image-queue/downloadImages.js | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index d99109738..3207a57ed 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -22,7 +22,20 @@ module.exports = Self => { const tempPath = path.join(container.client.root, container.name); const maxAttempts = 3; - prune(); + const images = await Self.find({ + where: {url: {neq: null}, attempts: {eq: maxAttempts}} + }); + + for (let image of images) { + const currentStamp = new Date().getTime(); + const updatedStamp = image.updated.getTime(); + const graceTime = Math.abs(currentStamp - updatedStamp); + const maxTTL = 3600 * 48 * 1000; // 48 hours in ms; + + if (graceTime >= maxTTL) + await Self.destroyById(image.itemFk); + } + download(); async function download() { @@ -60,7 +73,6 @@ module.exports = Self => { }); writeStream.on('close', async function() { - console.log('stream closed'); try { await models.Image.registerImage('catalog', filePath, fileName, image.itemFk); await image.destroy(); @@ -79,8 +91,7 @@ module.exports = Self => { try { const row = await Self.findById(rowId); - if (!row) - throw new Error(`Could not update due error ${error}`); + if (!row) return; if (row.attempts < maxAttempts) { await row.updateAttributes({ @@ -95,7 +106,7 @@ module.exports = Self => { download(); } catch (err) { - throw new Error(`ErrorHandler error: ${err}`); + throw new Error(`Image download failed: ${err}`); } } }; From ade2d9f4a2c7015d29a7e687f1473684cb10b7bf Mon Sep 17 00:00:00 2001 From: joan Date: Thu, 28 Jan 2021 16:01:49 +0100 Subject: [PATCH 3/4] Removed catch --- back/models/image.js | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/back/models/image.js b/back/models/image.js index 6723b59d2..78d159940 100644 --- a/back/models/image.js +++ b/back/models/image.js @@ -57,11 +57,7 @@ module.exports = Self => { await sharp(srcFilePath, {failOnError: false}) .resize(collection.maxWidth, collection.maxHeight, resizeOpts) .png() - .toFile(dstFile) - .catch(error => { - console.log('[ERROR] Image download failed: ' + error); - console.log('File => ' + srcFilePath); - }); + .toFile(dstFile); const sizes = collection.sizes(); for (let size of sizes) { @@ -76,11 +72,7 @@ module.exports = Self => { await sharp(srcFilePath, {failOnError: false}) .resize(size.width, size.height, resizeOpts) .png() - .toFile(dstFile) - .catch(error => { - console.log('[ERROR] Image download failed: ' + error); - console.log('File => ' + srcFilePath); - }); + .toFile(dstFile); } const model = models[collection.model]; From 7bcbf5303e005d19510c81a2c2ad41383aa3241c Mon Sep 17 00:00:00 2001 From: joan Date: Thu, 28 Jan 2021 16:08:57 +0100 Subject: [PATCH 4/4] Fix where --- modules/item/back/methods/item-image-queue/downloadImages.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index 3207a57ed..b8a03e1c1 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -23,7 +23,7 @@ module.exports = Self => { const maxAttempts = 3; const images = await Self.find({ - where: {url: {neq: null}, attempts: {eq: maxAttempts}} + where: {attempts: {eq: maxAttempts}} }); for (let image of images) {