From 8968f41b753414e793e0e5fc1af4de5b28ac200d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Tue, 18 Aug 2020 08:10:40 +0200 Subject: [PATCH 1/4] Queue changes --- .../item/back/methods/item-image-queue/downloadImages.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index e27b4516a..d6e38d97d 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -20,7 +20,6 @@ module.exports = Self => { const models = Self.app.models; try { - const imageQueue = await Self.find({where: {error: null}, limit: 25}); /* const tempPath = path.join('/tmp/salix-image'); */ const rootPath = models.Image.getPath(); const tempPath = path.join(rootPath, 'temp'); @@ -28,7 +27,8 @@ module.exports = Self => { // Create temporary path await fs.mkdir(tempPath, {recursive: true}); - for (let image of imageQueue) { + setInterval(async() => { + const image = await Self.findOne({where: {error: null}}); const fileName = `${image.itemFk}.png`; const filePath = path.join(tempPath, fileName); const file = fs.createWriteStream(filePath); @@ -58,7 +58,7 @@ module.exports = Self => { }).on('error', async error => { await errorHandler(image.itemFk, error, filePath); }); - } + }, 500); } catch (error) { await errorHandler(image.itemFk, error); } From 1ef6637ad3496213a7512f428b73bc9944a4e0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Tue, 18 Aug 2020 08:37:39 +0200 Subject: [PATCH 2/4] Added clearInterval() --- .../item/back/methods/item-image-queue/downloadImages.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index d6e38d97d..dc6318813 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -27,12 +27,15 @@ module.exports = Self => { // Create temporary path await fs.mkdir(tempPath, {recursive: true}); - setInterval(async() => { + const timer = setInterval(async() => { const image = await Self.findOne({where: {error: null}}); const fileName = `${image.itemFk}.png`; const filePath = path.join(tempPath, fileName); - const file = fs.createWriteStream(filePath); + // Exit loop + if (!image) clearInterval(timer); + + const file = fs.createWriteStream(filePath); https.get(image.url, async response => { if (response.statusCode != 200) { const error = new Error(`Could not download the image. Status code ${response.statusCode}`); From 2a7495bac902116934009e98639f91071fd016de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Tue, 18 Aug 2020 09:45:26 +0200 Subject: [PATCH 3/4] Image queue fixes --- back/models/image.js | 9 ++-- .../item-image-queue/downloadImages.js | 52 ++++++++++--------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/back/models/image.js b/back/models/image.js index 079acd293..f1eb09c9e 100644 --- a/back/models/image.js +++ b/back/models/image.js @@ -4,7 +4,8 @@ const path = require('path'); module.exports = Self => { Self.getPath = function() { - return '/var/lib/salix/image'; + // return '/var/lib/salix/image'; + return 'C:\\Users\\jsanc\\Desktop\\image'; }; Self.registerImage = async(collectionName, file, srcFilePath) => { @@ -55,8 +56,9 @@ module.exports = Self => { }; await fs.mkdir(dstDir, {recursive: true}); - await sharp(srcFilePath) + await sharp(srcFilePath, {failOnError: false}) .resize(collection.maxWidth, collection.maxHeight, resizeOpts) + .png() .toFile(dstFile); const sizes = collection.sizes(); @@ -69,8 +71,9 @@ module.exports = Self => { }; await fs.mkdir(dstDir, {recursive: true}); - await sharp(srcFilePath) + await sharp(srcFilePath, {failOnError: false}) .resize(size.width, size.height, resizeOpts) + .png() .toFile(dstFile); } diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index dc6318813..f295f420e 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -29,39 +29,41 @@ module.exports = Self => { const timer = setInterval(async() => { const image = await Self.findOne({where: {error: null}}); + + // Exit loop + if (!image) return clearInterval(timer); + const fileName = `${image.itemFk}.png`; const filePath = path.join(tempPath, fileName); - // Exit loop - if (!image) clearInterval(timer); + const writeStream = fs.createWriteStream(filePath); + writeStream.on('open', () => { + https.get(image.url, async response => { + if (response.statusCode != 200) { + const error = new Error(`Could not download the image. Status code ${response.statusCode}`); - const file = fs.createWriteStream(filePath); - https.get(image.url, async response => { - if (response.statusCode != 200) { - const error = new Error(`Could not download the image. Status code ${response.statusCode}`); - - file.close(); - await errorHandler(image.itemFk, error, filePath); - } - - response.pipe(file); - - file.on('error', async error => { - await errorHandler(image.itemFk, error, filePath); - }); - - file.on('finish', async function() { - try { - await models.Image.registerImage('catalog', fileName, filePath); - await image.destroy(); - } catch (error) { - await errorHandler(image.itemFk, error, filePath); + return await errorHandler(image.itemFk, error, filePath); } + + response.pipe(writeStream); + }).on('error', async error => { + await errorHandler(image.itemFk, error, filePath); }); - }).on('error', async error => { + }); + + writeStream.on('error', async error => { await errorHandler(image.itemFk, error, filePath); }); - }, 500); + + writeStream.on('finish', async function() { + try { + await models.Image.registerImage('catalog', fileName, filePath); + await image.destroy(); + } catch (error) { + await errorHandler(image.itemFk, error, filePath); + } + }); + }, 1000); } catch (error) { await errorHandler(image.itemFk, error); } From 4ef540c8f3e8b94dfedca96574a8e67122a69feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Tue, 18 Aug 2020 09:48:02 +0200 Subject: [PATCH 4/4] Changed download path --- back/models/image.js | 3 +-- modules/item/back/methods/item-image-queue/downloadImages.js | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/back/models/image.js b/back/models/image.js index f1eb09c9e..340b2e5a6 100644 --- a/back/models/image.js +++ b/back/models/image.js @@ -4,8 +4,7 @@ const path = require('path'); module.exports = Self => { Self.getPath = function() { - // return '/var/lib/salix/image'; - return 'C:\\Users\\jsanc\\Desktop\\image'; + return '/var/lib/salix/image'; }; Self.registerImage = async(collectionName, file, srcFilePath) => { diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index f295f420e..bd22bc281 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -20,9 +20,7 @@ module.exports = Self => { const models = Self.app.models; try { - /* const tempPath = path.join('/tmp/salix-image'); */ - const rootPath = models.Image.getPath(); - const tempPath = path.join(rootPath, 'temp'); + const tempPath = path.join('/tmp/salix-image'); // Create temporary path await fs.mkdir(tempPath, {recursive: true});