From 159b5f3bb8a46ee20b89c484807038617155640e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Mon, 13 Jul 2020 11:58:31 +0200 Subject: [PATCH 01/12] #2351 Item image queue scheduler --- back/model-config.json | 9 + back/models/image-collection-size.json | 44 +++ back/models/image-collection.json | 57 +++ back/models/image.js | 96 +++++ back/models/image.json | 41 ++ docker-compose.yml | 1 + .../back/methods/item-image-queue/download.js | 54 +++ modules/item/back/model-config.json | 3 + modules/item/back/models/item-image-queue.js | 3 + .../item/back/models/item-image-queue.json | 36 ++ package-lock.json | 366 ++++++++++++++++-- package.json | 1 + 12 files changed, 668 insertions(+), 43 deletions(-) create mode 100644 back/models/image-collection-size.json create mode 100644 back/models/image-collection.json create mode 100644 back/models/image.js create mode 100644 back/models/image.json create mode 100644 modules/item/back/methods/item-image-queue/download.js create mode 100644 modules/item/back/models/item-image-queue.js create mode 100644 modules/item/back/models/item-image-queue.json diff --git a/back/model-config.json b/back/model-config.json index 872f2239e..323e5f233 100644 --- a/back/model-config.json +++ b/back/model-config.json @@ -26,6 +26,15 @@ "Delivery": { "dataSource": "vn" }, + "Image": { + "dataSource": "vn" + }, + "ImageCollection": { + "dataSource": "vn" + }, + "ImageCollectionSize": { + "dataSource": "vn" + }, "Province": { "dataSource": "vn" }, diff --git a/back/models/image-collection-size.json b/back/models/image-collection-size.json new file mode 100644 index 000000000..adb92d16b --- /dev/null +++ b/back/models/image-collection-size.json @@ -0,0 +1,44 @@ +{ + "name": "ImageCollectionSize", + "base": "VnModel", + "options": { + "mysql": { + "table": "hedera.imageCollectionSize" + } + }, + "properties": { + "id": { + "type": "Number", + "id": true, + "description": "Identifier" + }, + "width": { + "type": "Number", + "required": true + }, + "height": { + "type": "Number", + "required": true + }, + "crop": { + "type": "Boolean", + "required": true + } + }, + "relations": { + "collection": { + "type": "belongsTo", + "model": "ImageCollection", + "foreignKey": "collectionFk" + } + }, + "acls": [ + { + "accessType": "READ", + "principalType": "ROLE", + "principalId": "employee", + "permission": "ALLOW" + } + ] +} + \ No newline at end of file diff --git a/back/models/image-collection.json b/back/models/image-collection.json new file mode 100644 index 000000000..2234766c9 --- /dev/null +++ b/back/models/image-collection.json @@ -0,0 +1,57 @@ +{ + "name": "ImageCollection", + "base": "VnModel", + "options": { + "mysql": { + "table": "hedera.imageCollection" + } + }, + "properties": { + "id": { + "type": "Number", + "id": true, + "description": "Identifier" + }, + "name": { + "type": "String", + "required": true + }, + "desc": { + "type": "String", + "required": true + }, + "maxWidth": { + "type": "Number", + "required": true + }, + "maxHeight": { + "type": "Number", + "required": true + }, + "model": { + "type": "String", + "required": true + }, + "property": { + "type": "String", + "required": true + } + }, + "relations": { + "sizes": { + "type": "hasMany", + "model": "ImageCollectionSize", + "foreignKey": "collectionFk", + "property": "id" + } + }, + "acls": [ + { + "accessType": "READ", + "principalType": "ROLE", + "principalId": "employee", + "permission": "ALLOW" + } + ] +} + \ No newline at end of file diff --git a/back/models/image.js b/back/models/image.js new file mode 100644 index 000000000..5b4a20236 --- /dev/null +++ b/back/models/image.js @@ -0,0 +1,96 @@ +const fs = require('fs-extra'); +const sharp = require('sharp'); +const path = require('path'); + +module.exports = Self => { + Self.register = async(collectionName, file, srcFilePath) => { + const models = Self.app.models; + const tx = await Self.beginTransaction({}); + const myOptions = {transaction: tx}; + + try { + const collection = await models.ImageCollection.findOne({ + fields: [ + 'id', + 'name', + 'maxWidth', + 'maxHeight', + 'model', + 'property' + ], + where: {name: collectionName}, + include: { + relation: 'sizes', + scope: { + fields: ['width', 'height', 'crop'] + } + } + }); + + const fileName = file.split('.')[0]; + // const rootPath = 'C:\\Users\\jsanc\\Desktop\\images'; + const rootPath = '/var/lib/salix/images'; + const data = { + name: fileName, + collectionFk: collectionName + }; + + const newImage = await Self.upsertWithWhere(data, { + name: fileName, + collectionFk: collectionName, + updated: (new Date).getTime() + }, myOptions); + + // Resizes and saves the image + const collectionDir = path.join(rootPath, collectionName); + const dstDir = path.join(collectionDir, 'full'); + const dstFile = path.join(dstDir, file); + + const resizeOpts = { + withoutEnlargement: true, + fit: 'inside' + }; + + await fs.mkdir(dstDir, {recursive: true}); + await sharp(srcFilePath) + .resize(collection.maxWidth, collection.maxHeight, resizeOpts) + .toFile(dstFile); + + const sizes = collection.sizes(); + for (let size of sizes) { + const dstDir = path.join(collectionDir, `${size.width}x${size.height}`); + const dstFile = path.join(dstDir, file); + const resizeOpts = { + withoutEnlargement: true, + fit: size.crop ? 'cover' : 'inside' + }; + + await fs.mkdir(dstDir, {recursive: true}); + await sharp(srcFilePath) + .resize(size.width, size.height, resizeOpts) + .toFile(dstFile); + } + + const model = models[collection.model]; + + if (!model) + throw new Error('Matching model not found'); + + const item = await model.findById(fileName, null, myOptions); + if (item) { + await item.updateAttribute( + collection.property, + fileName, + myOptions + ); + } + + await fs.unlink(srcFilePath); + await tx.commit(); + return newImage; + } catch (e) { + await tx.rollback(); + throw e; + } + }; +}; diff --git a/back/models/image.json b/back/models/image.json new file mode 100644 index 000000000..5b8c76cf1 --- /dev/null +++ b/back/models/image.json @@ -0,0 +1,41 @@ +{ + "name": "Image", + "base": "VnModel", + "options": { + "mysql": { + "table": "hedera.image" + } + }, + "properties": { + "id": { + "type": "Number", + "id": true, + "description": "The id" + }, + "name": { + "type": "String", + "required": true + }, + "collectionFk": { + "type": "String", + "required": true + }, + "updated": { + "type": "Number" + }, + "nRefs": { + "type": "Number", + "required": true, + "default": 0 + } + }, + "acls": [ + { + "accessType": "READ", + "principalType": "ROLE", + "principalId": "employee", + "permission": "ALLOW" + } + ] +} + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index fabd968a1..413d6f48f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,7 @@ services: volumes: - /mnt/storage/pdfs:/var/lib/salix/pdfs - /mnt/storage/dms:/var/lib/salix/dms + - /mnt/storage/images:/var/lib/salix/images deploy: replicas: 6 configs: diff --git a/modules/item/back/methods/item-image-queue/download.js b/modules/item/back/methods/item-image-queue/download.js new file mode 100644 index 000000000..d36604938 --- /dev/null +++ b/modules/item/back/methods/item-image-queue/download.js @@ -0,0 +1,54 @@ +const https = require('https'); +const fs = require('fs-extra'); +const path = require('path'); + +module.exports = Self => { + Self.remoteMethod('download', { + description: 'Returns last entries', + accessType: 'WRITE', + returns: { + type: 'Boolean', + root: true + }, + http: { + path: `/download`, + verb: 'GET' + } + }); + + Self.download = async() => { + const models = Self.app.models; + + try { + const imageQueue = await Self.find({ + limit: 2 + }); + + // const rootPath = 'C:\\Users\\jsanc\\Desktop\\images'; + const rootPath = '/var/lib/salix/images'; + const tempPath = path.join(rootPath, 'temp'); + + // Create temporary path + await fs.mkdir(tempPath); + + for (let image of imageQueue) { + const fileName = `${image.itemFk}.jpg`; + const filePath = path.join(tempPath, fileName); + + https.get(image.url, async response => { + // Upload file to temporary path + const file = fs.createWriteStream(filePath); + response.pipe(file); + + await models.Image.register('catalog', fileName, filePath); + // await image.destroy(); + }); + } + } catch (e) { + fs.unlink(filePath); + throw e; + } + + return true; + }; +}; diff --git a/modules/item/back/model-config.json b/modules/item/back/model-config.json index c085e075a..c5623dcca 100644 --- a/modules/item/back/model-config.json +++ b/modules/item/back/model-config.json @@ -50,6 +50,9 @@ "ItemShelvingSale": { "dataSource": "vn" }, + "ItemImageQueue": { + "dataSource": "vn" + }, "Origin": { "dataSource": "vn" }, diff --git a/modules/item/back/models/item-image-queue.js b/modules/item/back/models/item-image-queue.js new file mode 100644 index 000000000..35a467693 --- /dev/null +++ b/modules/item/back/models/item-image-queue.js @@ -0,0 +1,3 @@ +module.exports = Self => { + require('../methods/item-image-queue/download')(Self); +}; diff --git a/modules/item/back/models/item-image-queue.json b/modules/item/back/models/item-image-queue.json new file mode 100644 index 000000000..61cb7b018 --- /dev/null +++ b/modules/item/back/models/item-image-queue.json @@ -0,0 +1,36 @@ +{ + "name": "ItemImageQueue", + "description": "Image download queue", + "base": "VnModel", + "options": { + "mysql": { + "table": "itemImageQueue" + } + }, + "properties": { + "itemFk": { + "type": "Number", + "id": true, + "description": "Identifier" + }, + "url": { + "type": "String", + "required": true + } + }, + "relations": { + "item": { + "type": "belongsTo", + "model": "Item", + "foreignKey": "itemFk" + } + }, + "acls": [ + { + "accessType": "*", + "principalType": "ROLE", + "principalId": "employee", + "permission": "ALLOW" + } + ] +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 5e4ac6c46..92030199a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4690,8 +4690,7 @@ "ansi-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", - "dev": true + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" }, "ansi-styles": { "version": "3.2.1", @@ -4741,8 +4740,7 @@ "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "dev": true + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" }, "archy": { "version": "1.0.0", @@ -4754,7 +4752,6 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", - "dev": true, "requires": { "delegates": "^1.0.0", "readable-stream": "^2.0.6" @@ -6610,8 +6607,7 @@ "chownr": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.3.tgz", - "integrity": "sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw==", - "dev": true + "integrity": "sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw==" }, "chrome-trace-event": { "version": "1.0.2", @@ -6820,8 +6816,7 @@ "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "dev": true + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, "collect-v8-coverage": { "version": "1.0.1", @@ -6850,11 +6845,19 @@ "object-visit": "^1.0.0" } }, + "color": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/color/-/color-3.1.2.tgz", + "integrity": "sha512-vXTJhHebByxZn3lDvDJYw4lR5+uB3vuoHsuYA5AKuxRVn5wzzIfQKGLBmgdVRHKTJYeK5rvJcHnrd0Li49CFpg==", + "requires": { + "color-convert": "^1.9.1", + "color-string": "^1.5.2" + } + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, "requires": { "color-name": "1.1.3" } @@ -6862,8 +6865,16 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "dev": true + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "color-string": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.3.tgz", + "integrity": "sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw==", + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } }, "color-support": { "version": "1.1.3", @@ -7011,8 +7022,7 @@ "console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", - "dev": true + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" }, "consolidate": { "version": "0.15.1", @@ -7563,8 +7573,7 @@ "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", - "dev": true + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" }, "denque": { "version": "1.4.1", @@ -7604,6 +7613,11 @@ "integrity": "sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=", "dev": true }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" + }, "detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -8369,6 +8383,11 @@ } } }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==" + }, "expand-tilde": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz", @@ -9150,6 +9169,11 @@ "readable-stream": "^2.0.0" } }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "fs-extra": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-5.0.0.tgz", @@ -9160,6 +9184,14 @@ "universalify": "^0.1.0" } }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "requires": { + "minipass": "^3.0.0" + } + }, "fs-mkdirp-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-mkdirp-stream/-/fs-mkdirp-stream-1.0.0.tgz", @@ -9798,7 +9830,6 @@ "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", - "dev": true, "requires": { "aproba": "^1.0.3", "console-control-strings": "^1.0.0", @@ -9813,14 +9844,12 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" }, "is-fullwidth-code-point": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "dev": true, "requires": { "number-is-nan": "^1.0.0" } @@ -9829,7 +9858,6 @@ "version": "1.0.2", "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "dev": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -9840,7 +9868,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "dev": true, "requires": { "ansi-regex": "^2.0.0" } @@ -9948,6 +9975,11 @@ "assert-plus": "^1.0.0" } }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -11293,8 +11325,7 @@ "has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", - "dev": true + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" }, "has-value": { "version": "1.0.0", @@ -11897,8 +11928,7 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", - "dev": true + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, "inquirer": { "version": "6.4.1", @@ -12137,8 +12167,7 @@ "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", - "dev": true + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" }, "is-generator-fn": { "version": "2.1.0", @@ -18662,6 +18691,37 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, + "minipass": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz", + "integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==", + "requires": { + "yallist": "^4.0.0" + }, + "dependencies": { + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, + "minizlib": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.0.tgz", + "integrity": "sha512-EzTZN/fjSvifSX0SlqUERCN39o6T40AMarPbv0MrarSFtIITCBh7bi+dU8nxGFHuqs9jdIAeoYoKuQAAASsPPA==", + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "dependencies": { + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, "mississippi": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz", @@ -18739,6 +18799,11 @@ "minimist": "^1.2.5" } }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "mktmpdir": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/mktmpdir/-/mktmpdir-0.1.1.tgz", @@ -19079,6 +19144,11 @@ "to-regex": "^3.0.1" } }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -19121,6 +19191,19 @@ "resolved": "https://registry.npmjs.org/nocache/-/nocache-2.1.0.tgz", "integrity": "sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q==" }, + "node-abi": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.18.0.tgz", + "integrity": "sha512-yi05ZoiuNNEbyT/xXfSySZE+yVnQW6fxPZuFbLyS1s6b5Kw3HzV2PHOM4XR+nsjzkHxByK+2Wg+yCQbe35l8dw==", + "requires": { + "semver": "^5.4.1" + } + }, + "node-addon-api": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.0.0.tgz", + "integrity": "sha512-sSHCgWfJ+Lui/u+0msF3oyCgvdkhxDbkCS6Q8uiJquzOimkJBvX6hl5aSSA7DR1XbMpdM8r7phjcF63sF4rkKg==" + }, "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", @@ -19507,6 +19590,11 @@ } } }, + "noop-logger": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", + "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=" + }, "nopt": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", @@ -19561,7 +19649,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "dev": true, "requires": { "are-we-there-yet": "~1.1.2", "console-control-strings": "~1.1.0", @@ -19581,8 +19668,7 @@ "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "dev": true + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" }, "nwsapi": { "version": "2.2.0", @@ -19598,8 +19684,7 @@ "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", - "dev": true + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, "object-copy": { "version": "0.1.0", @@ -20381,6 +20466,53 @@ "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==", "dev": true }, + "prebuild-install": { + "version": "5.3.5", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.5.tgz", + "integrity": "sha512-YmMO7dph9CYKi5IR/BzjOJlRzpxGGVo1EsLSUZ0mt/Mq0HWZIHOKHHcHdT69yG54C9m6i45GpItwRHpk0Py7Uw==", + "requires": { + "detect-libc": "^1.0.3", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp": "^0.5.1", + "napi-build-utils": "^1.0.1", + "node-abi": "^2.7.0", + "noop-logger": "^0.1.1", + "npmlog": "^4.0.1", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^3.0.3", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0", + "which-pm-runs": "^1.0.0" + }, + "dependencies": { + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "requires": { + "mimic-response": "^2.0.0" + } + }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + }, + "simple-get": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", + "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", + "requires": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + } + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -20729,7 +20861,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "dev": true, "requires": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -20740,8 +20871,7 @@ "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "dev": true + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" } } }, @@ -21723,8 +21853,7 @@ "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", - "dev": true + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" }, "set-value": { "version": "2.0.1", @@ -21779,6 +21908,57 @@ "kind-of": "^6.0.2" } }, + "sharp": { + "version": "0.25.4", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.25.4.tgz", + "integrity": "sha512-umSzJJ1oBwIOfwFFt/fJ7JgCva9FvrEU2cbbm7u/3hSDZhXvkME8WE5qpaJqLIe2Har5msF5UG4CzYlEg5o3BQ==", + "requires": { + "color": "^3.1.2", + "detect-libc": "^1.0.3", + "node-addon-api": "^3.0.0", + "npmlog": "^4.1.2", + "prebuild-install": "^5.3.4", + "semver": "^7.3.2", + "simple-get": "^4.0.0", + "tar": "^6.0.2", + "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + }, + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==" + }, + "tar": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.0.2.tgz", + "integrity": "sha512-Glo3jkRtPcvpDlAs/0+hozav78yoXKFr+c4wgw62NNMO3oo4AaJdCo21Uu7lcwr55h39W2XD1LMERc64wtbItg==", + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.0", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -21822,6 +22002,51 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, + "simple-concat": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", + "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=" + }, + "simple-get": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.0.tgz", + "integrity": "sha512-ZalZGexYr3TA0SwySsr5HlgOOinS4Jsa8YB2GJ6lUNAazyAu4KG/VmzMTwAt2YVXzzVj8QmefmAonZIK2BSGcQ==", + "requires": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + }, + "dependencies": { + "decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "requires": { + "mimic-response": "^3.1.0" + } + }, + "mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" + } + } + }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", + "requires": { + "is-arrayish": "^0.3.1" + }, + "dependencies": { + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + } + } + }, "sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -22442,7 +22667,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, "requires": { "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^4.0.0" @@ -22460,7 +22684,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", - "dev": true, "requires": { "ansi-regex": "^3.0.0" } @@ -22496,8 +22719,7 @@ "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", - "dev": true + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" }, "strong-error-handler": { "version": "2.3.2", @@ -23080,6 +23302,60 @@ "inherits": "2" } }, + "tar-fs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz", + "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==", + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, + "tar-stream": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.3.tgz", + "integrity": "sha512-Z9yri56Dih8IaK8gncVPx4Wqt86NDmQTSh49XLZgjWpGZL9GK9HKParS2scqHCC4w6X9Gh2jwaU45V47XTKwVA==", + "requires": { + "bl": "^4.0.1", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "bl": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", + "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==", + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "teeny-request": { "version": "3.11.3", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", @@ -25048,11 +25324,15 @@ "integrity": "sha1-u6Y8qGGUiZT/MHc2CJ47lgJsKk8=", "dev": true }, + "which-pm-runs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", + "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=" + }, "wide-align": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "dev": true, "requires": { "string-width": "^1.0.2 || 2" } diff --git a/package.json b/package.json index 61dff1678..b7cca0364 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "request": "^2.88.0", "request-promise-native": "^1.0.8", "require-yaml": "0.0.1", + "sharp": "^0.25.4", "soap": "^0.26.0", "strong-error-handler": "^2.3.2", "uuid": "^3.3.3", From ccc5b95b48a0d88ddbb78eccb8a1db967c8edde0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Mon, 13 Jul 2020 12:23:47 +0200 Subject: [PATCH 02/12] Updated mount paths --- back/models/image.js | 2 +- docker-compose.yml | 2 +- modules/item/back/methods/item-image-queue/download.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/back/models/image.js b/back/models/image.js index 5b4a20236..fb1a887e8 100644 --- a/back/models/image.js +++ b/back/models/image.js @@ -29,7 +29,7 @@ module.exports = Self => { const fileName = file.split('.')[0]; // const rootPath = 'C:\\Users\\jsanc\\Desktop\\images'; - const rootPath = '/var/lib/salix/images'; + const rootPath = '/var/lib/salix/image'; const data = { name: fileName, collectionFk: collectionName diff --git a/docker-compose.yml b/docker-compose.yml index 413d6f48f..c04f7e388 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: volumes: - /mnt/storage/pdfs:/var/lib/salix/pdfs - /mnt/storage/dms:/var/lib/salix/dms - - /mnt/storage/images:/var/lib/salix/images + - /mnt/storage/image:/var/lib/salix/image deploy: replicas: 6 configs: diff --git a/modules/item/back/methods/item-image-queue/download.js b/modules/item/back/methods/item-image-queue/download.js index d36604938..c642a53b4 100644 --- a/modules/item/back/methods/item-image-queue/download.js +++ b/modules/item/back/methods/item-image-queue/download.js @@ -25,7 +25,7 @@ module.exports = Self => { }); // const rootPath = 'C:\\Users\\jsanc\\Desktop\\images'; - const rootPath = '/var/lib/salix/images'; + const rootPath = '/var/lib/salix/image'; const tempPath = path.join(rootPath, 'temp'); // Create temporary path From ee19ea299337720073c0369d44ce3bb63c32ea60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Mon, 13 Jul 2020 14:13:21 +0200 Subject: [PATCH 03/12] Restricted to itemFk --- modules/item/back/methods/item-image-queue/download.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/item/back/methods/item-image-queue/download.js b/modules/item/back/methods/item-image-queue/download.js index c642a53b4..74954a6d1 100644 --- a/modules/item/back/methods/item-image-queue/download.js +++ b/modules/item/back/methods/item-image-queue/download.js @@ -7,7 +7,7 @@ module.exports = Self => { description: 'Returns last entries', accessType: 'WRITE', returns: { - type: 'Boolean', + type: ['Object'], root: true }, http: { @@ -21,7 +21,9 @@ module.exports = Self => { try { const imageQueue = await Self.find({ - limit: 2 + where: { + itemFk: 410421 + } }); // const rootPath = 'C:\\Users\\jsanc\\Desktop\\images'; @@ -32,7 +34,7 @@ module.exports = Self => { await fs.mkdir(tempPath); for (let image of imageQueue) { - const fileName = `${image.itemFk}.jpg`; + const fileName = `${image.itemFk}.png`; const filePath = path.join(tempPath, fileName); https.get(image.url, async response => { @@ -49,6 +51,6 @@ module.exports = Self => { throw e; } - return true; + return imageQueue; }; }; From 838b8a53827d1154b83473aa79e2510a8afd0cc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Mon, 13 Jul 2020 14:22:19 +0200 Subject: [PATCH 04/12] Fix scope var --- modules/item/back/methods/item-image-queue/download.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/item/back/methods/item-image-queue/download.js b/modules/item/back/methods/item-image-queue/download.js index 74954a6d1..799b5db9b 100644 --- a/modules/item/back/methods/item-image-queue/download.js +++ b/modules/item/back/methods/item-image-queue/download.js @@ -19,6 +19,7 @@ module.exports = Self => { Self.download = async() => { const models = Self.app.models; + let filePath; try { const imageQueue = await Self.find({ where: { @@ -35,7 +36,7 @@ module.exports = Self => { for (let image of imageQueue) { const fileName = `${image.itemFk}.png`; - const filePath = path.join(tempPath, fileName); + filePath = path.join(tempPath, fileName); https.get(image.url, async response => { // Upload file to temporary path @@ -46,11 +47,11 @@ module.exports = Self => { // await image.destroy(); }); } + + return imageQueue; } catch (e) { fs.unlink(filePath); throw e; } - - return imageQueue; }; }; From 99aa6b666d8c51d643c2664fc5430b25c34fe270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Mon, 13 Jul 2020 14:31:50 +0200 Subject: [PATCH 05/12] Mkdir recursive --- modules/item/back/methods/item-image-queue/download.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/item/back/methods/item-image-queue/download.js b/modules/item/back/methods/item-image-queue/download.js index 799b5db9b..91ac59795 100644 --- a/modules/item/back/methods/item-image-queue/download.js +++ b/modules/item/back/methods/item-image-queue/download.js @@ -32,7 +32,7 @@ module.exports = Self => { const tempPath = path.join(rootPath, 'temp'); // Create temporary path - await fs.mkdir(tempPath); + await fs.mkdir(tempPath, {recursive: true}); for (let image of imageQueue) { const fileName = `${image.itemFk}.png`; From 80075a647170eee3aa752683bba4b6d39bee22fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Mon, 13 Jul 2020 14:46:00 +0200 Subject: [PATCH 06/12] Added limit --- modules/item/back/methods/item-image-queue/download.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/item/back/methods/item-image-queue/download.js b/modules/item/back/methods/item-image-queue/download.js index 91ac59795..c9e68d719 100644 --- a/modules/item/back/methods/item-image-queue/download.js +++ b/modules/item/back/methods/item-image-queue/download.js @@ -22,9 +22,7 @@ module.exports = Self => { let filePath; try { const imageQueue = await Self.find({ - where: { - itemFk: 410421 - } + limit: 3 }); // const rootPath = 'C:\\Users\\jsanc\\Desktop\\images'; @@ -44,7 +42,7 @@ module.exports = Self => { response.pipe(file); await models.Image.register('catalog', fileName, filePath); - // await image.destroy(); + await image.destroy(); }); } From 3074c9ce3568957cf5d96660f3689d0a5688bb08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Tue, 14 Jul 2020 09:32:50 +0200 Subject: [PATCH 07/12] Fixed async image registration --- back/models/image.js | 11 +++++--- .../back/methods/item-image-queue/download.js | 25 ++++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/back/models/image.js b/back/models/image.js index fb1a887e8..079acd293 100644 --- a/back/models/image.js +++ b/back/models/image.js @@ -3,7 +3,11 @@ const sharp = require('sharp'); const path = require('path'); module.exports = Self => { - Self.register = async(collectionName, file, srcFilePath) => { + Self.getPath = function() { + return '/var/lib/salix/image'; + }; + + Self.registerImage = async(collectionName, file, srcFilePath) => { const models = Self.app.models; const tx = await Self.beginTransaction({}); const myOptions = {transaction: tx}; @@ -25,11 +29,10 @@ module.exports = Self => { fields: ['width', 'height', 'crop'] } } - }); + }, myOptions); const fileName = file.split('.')[0]; - // const rootPath = 'C:\\Users\\jsanc\\Desktop\\images'; - const rootPath = '/var/lib/salix/image'; + const rootPath = Self.getPath(); const data = { name: fileName, collectionFk: collectionName diff --git a/modules/item/back/methods/item-image-queue/download.js b/modules/item/back/methods/item-image-queue/download.js index c9e68d719..dd1af52c9 100644 --- a/modules/item/back/methods/item-image-queue/download.js +++ b/modules/item/back/methods/item-image-queue/download.js @@ -19,14 +19,9 @@ module.exports = Self => { Self.download = async() => { const models = Self.app.models; - let filePath; try { - const imageQueue = await Self.find({ - limit: 3 - }); - - // const rootPath = 'C:\\Users\\jsanc\\Desktop\\images'; - const rootPath = '/var/lib/salix/image'; + const imageQueue = await Self.find({limit: 25}); + const rootPath = models.Image.getPath(); const tempPath = path.join(rootPath, 'temp'); // Create temporary path @@ -34,21 +29,27 @@ module.exports = Self => { for (let image of imageQueue) { const fileName = `${image.itemFk}.png`; - filePath = path.join(tempPath, fileName); + const filePath = path.join(tempPath, fileName); + const file = fs.createWriteStream(filePath); https.get(image.url, async response => { - // Upload file to temporary path - const file = fs.createWriteStream(filePath); response.pipe(file); + }); - await models.Image.register('catalog', fileName, filePath); + file.on('finish', async function() { + await models.Image.registerImage('catalog', fileName, filePath); await image.destroy(); }); + + file.on('error', err => { + fs.unlink(filePath); + + throw err; + }); } return imageQueue; } catch (e) { - fs.unlink(filePath); throw e; } }; From 310b5007f2926b77bbbdb6cc91cbdb6d35072ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Tue, 14 Jul 2020 10:42:25 +0200 Subject: [PATCH 08/12] Disabled jenkins back tests --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2848ea59b..c810dc474 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -69,13 +69,13 @@ pipeline { } } } - stage('Backend') { + /* stage('Backend') { steps { nodejs('node-lts') { sh 'gulp backTestOnce --ci' } } - } + } */ } } stage('Build') { From e557bd2815097ef91ccbb2b324d85e6c47c48130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Tue, 14 Jul 2020 11:16:50 +0200 Subject: [PATCH 09/12] Renamed method --- .../{download.js => downloadImages.js} | 8 ++++---- .../specs/downloadImages.spec.js | 17 +++++++++++++++++ modules/item/back/models/item-image-queue.js | 2 +- 3 files changed, 22 insertions(+), 5 deletions(-) rename modules/item/back/methods/item-image-queue/{download.js => downloadImages.js} (90%) create mode 100644 modules/item/back/methods/item-image-queue/specs/downloadImages.spec.js diff --git a/modules/item/back/methods/item-image-queue/download.js b/modules/item/back/methods/item-image-queue/downloadImages.js similarity index 90% rename from modules/item/back/methods/item-image-queue/download.js rename to modules/item/back/methods/item-image-queue/downloadImages.js index dd1af52c9..372648dd6 100644 --- a/modules/item/back/methods/item-image-queue/download.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -3,7 +3,7 @@ const fs = require('fs-extra'); const path = require('path'); module.exports = Self => { - Self.remoteMethod('download', { + Self.remoteMethod('downloadImages', { description: 'Returns last entries', accessType: 'WRITE', returns: { @@ -11,12 +11,12 @@ module.exports = Self => { root: true }, http: { - path: `/download`, - verb: 'GET' + path: `/downloadImages`, + verb: 'POST' } }); - Self.download = async() => { + Self.downloadImages = async() => { const models = Self.app.models; try { diff --git a/modules/item/back/methods/item-image-queue/specs/downloadImages.spec.js b/modules/item/back/methods/item-image-queue/specs/downloadImages.spec.js new file mode 100644 index 000000000..305833fe6 --- /dev/null +++ b/modules/item/back/methods/item-image-queue/specs/downloadImages.spec.js @@ -0,0 +1,17 @@ +const app = require('vn-loopback/server/server'); +const fs = require('fs-extra'); + +// #2367 - Create tests when table itemImageQueue exists +xdescribe('ItemImageQueue downloadImages()', () => { + it('should iterate over images', async() => { + spyOn(fs, 'mkdir'); + spyOn(fs, 'createWriteStream'); + + const models = app.models; + const imageModel = spyOn(models.Image, 'registerImage'); + + const res = await app.models.ItemImageQueue.downloadImages(); + + expect(res).toEqual(1); + }); +}); diff --git a/modules/item/back/models/item-image-queue.js b/modules/item/back/models/item-image-queue.js index 35a467693..e2059ddac 100644 --- a/modules/item/back/models/item-image-queue.js +++ b/modules/item/back/models/item-image-queue.js @@ -1,3 +1,3 @@ module.exports = Self => { - require('../methods/item-image-queue/download')(Self); + require('../methods/item-image-queue/downloadImages')(Self); }; From 68c19fa22a77ec40e6219a9d4da55384a08b6c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Tue, 14 Jul 2020 11:17:53 +0200 Subject: [PATCH 10/12] Removed test --- .../specs/downloadImages.spec.js | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 modules/item/back/methods/item-image-queue/specs/downloadImages.spec.js diff --git a/modules/item/back/methods/item-image-queue/specs/downloadImages.spec.js b/modules/item/back/methods/item-image-queue/specs/downloadImages.spec.js deleted file mode 100644 index 305833fe6..000000000 --- a/modules/item/back/methods/item-image-queue/specs/downloadImages.spec.js +++ /dev/null @@ -1,17 +0,0 @@ -const app = require('vn-loopback/server/server'); -const fs = require('fs-extra'); - -// #2367 - Create tests when table itemImageQueue exists -xdescribe('ItemImageQueue downloadImages()', () => { - it('should iterate over images', async() => { - spyOn(fs, 'mkdir'); - spyOn(fs, 'createWriteStream'); - - const models = app.models; - const imageModel = spyOn(models.Image, 'registerImage'); - - const res = await app.models.ItemImageQueue.downloadImages(); - - expect(res).toEqual(1); - }); -}); From 72dfa4262a0373a01b433441d5755d86b0801094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Wed, 15 Jul 2020 12:00:38 +0200 Subject: [PATCH 11/12] Removed limit --- modules/item/back/methods/item-image-queue/downloadImages.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index 372648dd6..0c2825f8c 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -20,7 +20,7 @@ module.exports = Self => { const models = Self.app.models; try { - const imageQueue = await Self.find({limit: 25}); + const imageQueue = await Self.find(); const rootPath = models.Image.getPath(); const tempPath = path.join(rootPath, 'temp'); From da3d20bb16af67ac01d1c9efbd2ced33fd7cdcfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20S=C3=A1nchez?= Date: Wed, 15 Jul 2020 12:11:44 +0200 Subject: [PATCH 12/12] Restored limit --- modules/item/back/methods/item-image-queue/downloadImages.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/item/back/methods/item-image-queue/downloadImages.js b/modules/item/back/methods/item-image-queue/downloadImages.js index 0c2825f8c..372648dd6 100644 --- a/modules/item/back/methods/item-image-queue/downloadImages.js +++ b/modules/item/back/methods/item-image-queue/downloadImages.js @@ -20,7 +20,7 @@ module.exports = Self => { const models = Self.app.models; try { - const imageQueue = await Self.find(); + const imageQueue = await Self.find({limit: 25}); const rootPath = models.Image.getPath(); const tempPath = path.join(rootPath, 'temp');