2476 - downloadImages() method refactor

back/models/image.js

@@ -57,7 +57,11 @@ module.exports = Self => {
             await sharp(srcFilePath, {failOnError: false})
                 .resize(collection.maxWidth, collection.maxHeight, resizeOpts)
                 .png()
-                .toFile(dstFile);
+                .toFile(dstFile)
+                .catch(error => {
+                    console.log('[ERROR] Image download failed: ' + error);
+                    console.log('File => ' + srcFilePath);
+                });
 
             const sizes = collection.sizes();
             for (let size of sizes) {
@@ -72,7 +76,11 @@ module.exports = Self => {
                 await sharp(srcFilePath, {failOnError: false})
                     .resize(size.width, size.height, resizeOpts)
                     .png()
-                    .toFile(dstFile);
+                    .toFile(dstFile)
+                    .catch(error => {
+                        console.log('[ERROR] Image download failed: ' + error);
+                        console.log('File => ' + srcFilePath);
+                    });
             }
 
             const model = models[collection.model];

db/changes/10280-valentineDay/00-itemImageQueue.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `vn`.`itemImageQueue`
+	ADD attempts INT default 0 NULL AFTER error;

modules/item/back/methods/item-image-queue/downloadImages.js

@@ -18,28 +18,27 @@ module.exports = Self => {
 
     Self.downloadImages = async() => {
         const models = Self.app.models;
+        const container = await models.TempContainer.container('salix-image');
+        const tempPath = path.join(container.client.root, container.name);
+        const maxAttempts = 3;
 
-        try {
+        prune();
-            const tempPath = path.join('/tmp/salix-image');
+        download();
 
-            // Create temporary path
+        async function download() {
-            await fs.mkdir(tempPath, {recursive: true});
-
-            const timer = setInterval(async() => {
             const image = await Self.findOne({
-                    where: {error: null, url: {neq: null}}
+                where: {url: {neq: null}, attempts: {lt: maxAttempts}},
+                order: 'attempts, updated'
             });
 
-                // Exit loop
+            if (!image) return;
-                if (!image) return clearInterval(timer);
 
             const srcFile = image.url.split('/').pop();
-                const fileName = srcFile.split('.')[0];
+            const dotIndex = srcFile.lastIndexOf('.');
+            const fileName = srcFile.substring(0, dotIndex);
             const file = `${fileName}.png`;
             const filePath = path.join(tempPath, file);
 
-                const writeStream = fs.createWriteStream(filePath);
-                writeStream.on('open', () => {
             https.get(image.url, async response => {
                 if (response.statusCode != 200) {
                     const error = new Error(`Could not download the image. Status code ${response.statusCode}`);
@@ -47,10 +46,9 @@ module.exports = Self => {
                     return await errorHandler(image.itemFk, error, filePath);
                 }
 
+                const writeStream = fs.createWriteStream(filePath);
+                writeStream.on('open', () => {
                     response.pipe(writeStream);
-                    }).on('error', async error => {
-                        await errorHandler(image.itemFk, error, filePath);
-                    });
                 });
 
                 writeStream.on('error', async error => {
@@ -58,16 +56,23 @@ module.exports = Self => {
                 });
 
                 writeStream.on('finish', async function() {
+                    writeStream.end();
+                });
+
+                writeStream.on('close', async function() {
+                    console.log('stream closed');
                     try {
                         await models.Image.registerImage('catalog', filePath, fileName, image.itemFk);
                         await image.destroy();
+
+                        download();
                     } catch (error) {
                         await errorHandler(image.itemFk, error, filePath);
                     }
                 });
-            }, 1000);
+            }).on('error', async error => {
-        } catch (error) {
+                await errorHandler(image.itemFk, error, filePath);
-            throw new Error('Try-catch error: ', error);
+            });
         }
 
         async function errorHandler(rowId, error, filePath) {
@@ -77,10 +82,18 @@ module.exports = Self => {
                 if (!row)
                     throw new Error(`Could not update due error ${error}`);
 
-                await row.updateAttribute('error', error);
+                if (row.attempts < maxAttempts) {
+                    await row.updateAttributes({
+                        error: error,
+                        attempts: row.attempts + 1,
+                        updated: new Date()
+                    });
+                }
 
                 if (filePath && fs.existsSync(filePath))
                     await fs.unlink(filePath);
+
+                download();
             } catch (err) {
                 throw new Error(`ErrorHandler error: ${err}`);
             }

modules/item/back/models/item-image-queue.json

@@ -9,17 +9,26 @@
     },
     "properties": {
         "itemFk": {
-            "type": "Number",
+            "type": "number",
             "id": true,
             "description": "Identifier"
         },
         "url": {
-            "type": "String",
+            "type": "string",
             "required": true
         },
         "error": {
-            "type": "String",
+            "type": "string",
             "required": true
+        },
+        "attempts": {
+            "type": "number"
+        },
+        "created": {
+            "type": "date"
+        },
+        "updated": {
+            "type": "date"
         }
     },
     "relations": {