Merge pull request '2476 - downloadImages() method refactor' (#528) from 2476-floramondo_refactor into dev
gitea/salix/pipeline/head This commit looks good
Details
gitea/salix/pipeline/head This commit looks good
Details
Reviewed-on: #528 Reviewed-by: Carlos Jimenez Ruiz <carlosjr@verdnatura.es>
This commit is contained in:
commit
5384c2b217
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE `vn`.`itemImageQueue`
|
||||
ADD attempts INT default 0 NULL AFTER error;
|
|
@ -18,39 +18,50 @@ module.exports = Self => {
|
|||
|
||||
Self.downloadImages = async() => {
|
||||
const models = Self.app.models;
|
||||
const container = await models.TempContainer.container('salix-image');
|
||||
const tempPath = path.join(container.client.root, container.name);
|
||||
const maxAttempts = 3;
|
||||
|
||||
try {
|
||||
const tempPath = path.join('/tmp/salix-image');
|
||||
const images = await Self.find({
|
||||
where: {attempts: {eq: maxAttempts}}
|
||||
});
|
||||
|
||||
// Create temporary path
|
||||
await fs.mkdir(tempPath, {recursive: true});
|
||||
for (let image of images) {
|
||||
const currentStamp = new Date().getTime();
|
||||
const updatedStamp = image.updated.getTime();
|
||||
const graceTime = Math.abs(currentStamp - updatedStamp);
|
||||
const maxTTL = 3600 * 48 * 1000; // 48 hours in ms;
|
||||
|
||||
const timer = setInterval(async() => {
|
||||
const image = await Self.findOne({
|
||||
where: {error: null, url: {neq: null}}
|
||||
});
|
||||
if (graceTime >= maxTTL)
|
||||
await Self.destroyById(image.itemFk);
|
||||
}
|
||||
|
||||
// Exit loop
|
||||
if (!image) return clearInterval(timer);
|
||||
download();
|
||||
|
||||
const srcFile = image.url.split('/').pop();
|
||||
const fileName = srcFile.split('.')[0];
|
||||
const file = `${fileName}.png`;
|
||||
const filePath = path.join(tempPath, file);
|
||||
async function download() {
|
||||
const image = await Self.findOne({
|
||||
where: {url: {neq: null}, attempts: {lt: maxAttempts}},
|
||||
order: 'attempts, updated'
|
||||
});
|
||||
|
||||
if (!image) return;
|
||||
|
||||
const srcFile = image.url.split('/').pop();
|
||||
const dotIndex = srcFile.lastIndexOf('.');
|
||||
const fileName = srcFile.substring(0, dotIndex);
|
||||
const file = `${fileName}.png`;
|
||||
const filePath = path.join(tempPath, file);
|
||||
|
||||
https.get(image.url, async response => {
|
||||
if (response.statusCode != 200) {
|
||||
const error = new Error(`Could not download the image. Status code ${response.statusCode}`);
|
||||
|
||||
return await errorHandler(image.itemFk, error, filePath);
|
||||
}
|
||||
|
||||
const writeStream = fs.createWriteStream(filePath);
|
||||
writeStream.on('open', () => {
|
||||
https.get(image.url, async response => {
|
||||
if (response.statusCode != 200) {
|
||||
const error = new Error(`Could not download the image. Status code ${response.statusCode}`);
|
||||
|
||||
return await errorHandler(image.itemFk, error, filePath);
|
||||
}
|
||||
|
||||
response.pipe(writeStream);
|
||||
}).on('error', async error => {
|
||||
await errorHandler(image.itemFk, error, filePath);
|
||||
});
|
||||
response.pipe(writeStream);
|
||||
});
|
||||
|
||||
writeStream.on('error', async error => {
|
||||
|
@ -58,31 +69,44 @@ module.exports = Self => {
|
|||
});
|
||||
|
||||
writeStream.on('finish', async function() {
|
||||
writeStream.end();
|
||||
});
|
||||
|
||||
writeStream.on('close', async function() {
|
||||
try {
|
||||
await models.Image.registerImage('catalog', filePath, fileName, image.itemFk);
|
||||
await image.destroy();
|
||||
|
||||
download();
|
||||
} catch (error) {
|
||||
await errorHandler(image.itemFk, error, filePath);
|
||||
}
|
||||
});
|
||||
}, 1000);
|
||||
} catch (error) {
|
||||
throw new Error('Try-catch error: ', error);
|
||||
}).on('error', async error => {
|
||||
await errorHandler(image.itemFk, error, filePath);
|
||||
});
|
||||
}
|
||||
|
||||
async function errorHandler(rowId, error, filePath) {
|
||||
try {
|
||||
const row = await Self.findById(rowId);
|
||||
|
||||
if (!row)
|
||||
throw new Error(`Could not update due error ${error}`);
|
||||
if (!row) return;
|
||||
|
||||
await row.updateAttribute('error', error);
|
||||
if (row.attempts < maxAttempts) {
|
||||
await row.updateAttributes({
|
||||
error: error,
|
||||
attempts: row.attempts + 1,
|
||||
updated: new Date()
|
||||
});
|
||||
}
|
||||
|
||||
if (filePath && fs.existsSync(filePath))
|
||||
await fs.unlink(filePath);
|
||||
|
||||
download();
|
||||
} catch (err) {
|
||||
throw new Error(`ErrorHandler error: ${err}`);
|
||||
throw new Error(`Image download failed: ${err}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
|
@ -9,17 +9,26 @@
|
|||
},
|
||||
"properties": {
|
||||
"itemFk": {
|
||||
"type": "Number",
|
||||
"type": "number",
|
||||
"id": true,
|
||||
"description": "Identifier"
|
||||
},
|
||||
"url": {
|
||||
"type": "String",
|
||||
"type": "string",
|
||||
"required": true
|
||||
},
|
||||
"error": {
|
||||
"type": "String",
|
||||
"type": "string",
|
||||
"required": true
|
||||
},
|
||||
"attempts": {
|
||||
"type": "number"
|
||||
},
|
||||
"created": {
|
||||
"type": "date"
|
||||
},
|
||||
"updated": {
|
||||
"type": "date"
|
||||
}
|
||||
},
|
||||
"relations": {
|
||||
|
|
Loading…
Reference in New Issue