From c9ed17d981defac7e36080ef64c1135d6d59e5e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Thu, 13 Aug 2020 12:16:25 +0200 Subject: [PATCH 1/6] Changes to dowload item photos endpoint --- .../item-image-queue/downloadImages.js | 46 +++++++++++++------ .../item/back/models/item-image-queue.json | 4 ++ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index 372648dd6..c7054b340 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -20,9 +20,8 @@ module.exports = Self => { const models = Self.app.models; try { - const imageQueue = await Self.find({limit: 25}); - const rootPath = models.Image.getPath(); - const tempPath = path.join(rootPath, 'temp'); + const imageQueue = await Self.find({where: {error: null}}); + const tempPath = path.join('/temp/salix-image'); // Create temporary path await fs.mkdir(tempPath, {recursive: true}); @@ -33,24 +32,43 @@ module.exports = Self => { const file = fs.createWriteStream(filePath); https.get(image.url, async response => { + if (response.statusCode != 200) { + const error = new Error(`Could not download the image. Status code ${response.statusCode}`); + + file.close(); + await errorHandler(image.itemFk, error, filePath); + } + response.pipe(file); - }); - file.on('finish', async function() { - await models.Image.registerImage('catalog', fileName, filePath); - await image.destroy(); - }); + file.on('error', async error => { + await errorHandler(image.itemFk, error, filePath); + }); - file.on('error', err => { - fs.unlink(filePath); - - throw err; + file.on('finish', async function() { + try { + await models.Image.registerImage('catalog', fileName, filePath); + await image.destroy(); + } catch (error) { + await errorHandler(image.itemFk, error, filePath); + } + }); + }).on('error', async error => { + await errorHandler(image.itemFk, error, filePath); }); } return imageQueue; - } catch (e) { - throw e; + } catch (error) { + await errorHandler(image.itemFk, error); + } + + async function errorHandler(rowId, error, filePath) { + const row = await Self.findById(rowId); + await row.updateAttribute('error', error); + + if (filePath) + await fs.unlink(filePath); } }; }; diff --git a/modules/item/back/models/item-image-queue.json b/modules/item/back/models/item-image-queue.json index 61cb7b018..6e248ac96 100644 --- a/modules/item/back/models/item-image-queue.json +++ b/modules/item/back/models/item-image-queue.json @@ -16,6 +16,10 @@ "url": { "type": "String", "required": true + }, + "error": { + "type": "String", + "required": true } }, "relations": { From c0794808f03cd8db22c482fc3d509bf2a9baa2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Thu, 13 Aug 2020 13:09:53 +0200 Subject: [PATCH 2/6] Updated tmp directory --- modules/item/back/methods/item-image-queue/downloadImages.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index c7054b340..2e32a57b8 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -21,7 +21,7 @@ module.exports = Self => { try { const imageQueue = await Self.find({where: {error: null}}); - const tempPath = path.join('/temp/salix-image'); + const tempPath = path.join('/tmp/salix-image'); // Create temporary path await fs.mkdir(tempPath, {recursive: true}); From bbf15da16fa893cd603e790e8990d0ef364fea6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Thu, 13 Aug 2020 13:38:52 +0200 Subject: [PATCH 3/6] Changed tmp uri --- back/models/image.js | 2 +- .../item/back/methods/item-image-queue/downloadImages.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/back/models/image.js b/back/models/image.js index 079acd293..8d79d1d5f 100644 --- a/back/models/image.js +++ b/back/models/image.js @@ -4,7 +4,7 @@ const path = require('path'); module.exports = Self => { Self.getPath = function() { - return '/var/lib/salix/image'; + return 'C:\\Users\\jsanc\\Desktop\\image'; }; Self.registerImage = async(collectionName, file, srcFilePath) => { diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index 2e32a57b8..ab242fdad 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -21,7 +21,9 @@ module.exports = Self => { try { const imageQueue = await Self.find({where: {error: null}}); - const tempPath = path.join('/tmp/salix-image'); + /* const tempPath = path.join('/tmp/salix-image'); */ + const rootPath = models.Image.getPath(); + const tempPath = path.join(rootPath, 'temp'); // Create temporary path await fs.mkdir(tempPath, {recursive: true}); @@ -57,8 +59,6 @@ module.exports = Self => { await errorHandler(image.itemFk, error, filePath); }); } - - return imageQueue; } catch (error) { await errorHandler(image.itemFk, error); } From 49af185976685a0a8007d51746e374dee459e602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Thu, 13 Aug 2020 13:47:11 +0200 Subject: [PATCH 4/6] updated path --- back/models/image.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/back/models/image.js b/back/models/image.js index 8d79d1d5f..079acd293 100644 --- a/back/models/image.js +++ b/back/models/image.js @@ -4,7 +4,7 @@ const path = require('path'); module.exports = Self => { Self.getPath = function() { - return 'C:\\Users\\jsanc\\Desktop\\image'; + return '/var/lib/salix/image'; }; Self.registerImage = async(collectionName, file, srcFilePath) => { From 7a3de746cca84b782e19dfe063d0518b190f7520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Thu, 13 Aug 2020 14:00:40 +0200 Subject: [PATCH 5/6] Added limit --- modules/item/back/methods/item-image-queue/downloadImages.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index ab242fdad..45c1b0bd6 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -20,7 +20,7 @@ module.exports = Self => { const models = Self.app.models; try { - const imageQueue = await Self.find({where: {error: null}}); + const imageQueue = await Self.find({where: {error: null}, limit: 100}); /* const tempPath = path.join('/tmp/salix-image'); */ const rootPath = models.Image.getPath(); const tempPath = path.join(rootPath, 'temp'); From a10e091b628f5a7eb87c1b0a442542754b6a762a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Thu, 13 Aug 2020 16:26:35 +0200 Subject: [PATCH 6/6] Added limit --- modules/item/back/methods/item-image-queue/downloadImages.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index 45c1b0bd6..e27b4516a 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -20,7 +20,7 @@ module.exports = Self => { const models = Self.app.models; try { - const imageQueue = await Self.find({where: {error: null}, limit: 100}); + const imageQueue = await Self.find({where: {error: null}, limit: 25}); /* const tempPath = path.join('/tmp/salix-image'); */ const rootPath = models.Image.getPath(); const tempPath = path.join(rootPath, 'temp');