Merge pull request '2476 - downloadImages() method refactor' (#528) from 2476-floramondo_refactor into dev
gitea/salix/pipeline/head This commit looks good Details

Reviewed-on: #528
Reviewed-by: Carlos Jimenez Ruiz <carlosjr@verdnatura.es>
This commit is contained in:
Carlos Jimenez Ruiz 2021-01-28 16:03:28 +00:00
commit 5384c2b217
3 changed files with 70 additions and 35 deletions

View File

@ -0,0 +1,2 @@
ALTER TABLE `vn`.`itemImageQueue`
ADD attempts INT default 0 NULL AFTER error;

View File

@ -18,39 +18,50 @@ module.exports = Self => {
Self.downloadImages = async() => {
const models = Self.app.models;
const container = await models.TempContainer.container('salix-image');
const tempPath = path.join(container.client.root, container.name);
const maxAttempts = 3;
try {
const tempPath = path.join('/tmp/salix-image');
const images = await Self.find({
where: {attempts: {eq: maxAttempts}}
});
// Create temporary path
await fs.mkdir(tempPath, {recursive: true});
for (let image of images) {
const currentStamp = new Date().getTime();
const updatedStamp = image.updated.getTime();
const graceTime = Math.abs(currentStamp - updatedStamp);
const maxTTL = 3600 * 48 * 1000; // 48 hours in ms;
const timer = setInterval(async() => {
const image = await Self.findOne({
where: {error: null, url: {neq: null}}
});
if (graceTime >= maxTTL)
await Self.destroyById(image.itemFk);
}
// Exit loop
if (!image) return clearInterval(timer);
download();
const srcFile = image.url.split('/').pop();
const fileName = srcFile.split('.')[0];
const file = `${fileName}.png`;
const filePath = path.join(tempPath, file);
async function download() {
const image = await Self.findOne({
where: {url: {neq: null}, attempts: {lt: maxAttempts}},
order: 'attempts, updated'
});
if (!image) return;
const srcFile = image.url.split('/').pop();
const dotIndex = srcFile.lastIndexOf('.');
const fileName = srcFile.substring(0, dotIndex);
const file = `${fileName}.png`;
const filePath = path.join(tempPath, file);
https.get(image.url, async response => {
if (response.statusCode != 200) {
const error = new Error(`Could not download the image. Status code ${response.statusCode}`);
return await errorHandler(image.itemFk, error, filePath);
}
const writeStream = fs.createWriteStream(filePath);
writeStream.on('open', () => {
https.get(image.url, async response => {
if (response.statusCode != 200) {
const error = new Error(`Could not download the image. Status code ${response.statusCode}`);
return await errorHandler(image.itemFk, error, filePath);
}
response.pipe(writeStream);
}).on('error', async error => {
await errorHandler(image.itemFk, error, filePath);
});
response.pipe(writeStream);
});
writeStream.on('error', async error => {
@ -58,31 +69,44 @@ module.exports = Self => {
});
writeStream.on('finish', async function() {
writeStream.end();
});
writeStream.on('close', async function() {
try {
await models.Image.registerImage('catalog', filePath, fileName, image.itemFk);
await image.destroy();
download();
} catch (error) {
await errorHandler(image.itemFk, error, filePath);
}
});
}, 1000);
} catch (error) {
throw new Error('Try-catch error: ', error);
}).on('error', async error => {
await errorHandler(image.itemFk, error, filePath);
});
}
async function errorHandler(rowId, error, filePath) {
try {
const row = await Self.findById(rowId);
if (!row)
throw new Error(`Could not update due error ${error}`);
if (!row) return;
await row.updateAttribute('error', error);
if (row.attempts < maxAttempts) {
await row.updateAttributes({
error: error,
attempts: row.attempts + 1,
updated: new Date()
});
}
if (filePath && fs.existsSync(filePath))
await fs.unlink(filePath);
download();
} catch (err) {
throw new Error(`ErrorHandler error: ${err}`);
throw new Error(`Image download failed: ${err}`);
}
}
};

View File

@ -9,17 +9,26 @@
},
"properties": {
"itemFk": {
"type": "Number",
"type": "number",
"id": true,
"description": "Identifier"
},
"url": {
"type": "String",
"type": "string",
"required": true
},
"error": {
"type": "String",
"type": "string",
"required": true
},
"attempts": {
"type": "number"
},
"created": {
"type": "date"
},
"updated": {
"type": "date"
}
},
"relations": {