diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index 3dae2d47d..86a9e1f31 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -20,83 +20,93 @@ module.exports = Self => { const models = Self.app.models; // Get files checksum - const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig'); + const tx = await Self.beginTransaction({}); - const updatableFiles = []; - for (const file of files) { - const fileChecksum = await getChecksum(file); + try { + const options = {transaction: tx}; + const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options); - if (file.checksum != fileChecksum) { - updatableFiles.push({ - name: file.name, - checksum: fileChecksum - }); - } else - console.debug(`File already updated, skipping...`); - } + const updatableFiles = []; + for (const file of files) { + const fileChecksum = await getChecksum(file); - if (updatableFiles.length === 0) - return false; - - // Download files - const container = await models.TempContainer.container('edi'); - const tempPath = path.join(container.client.root, container.name); - - let remoteFile; - let tempDir; - let tempFile; - - const fileNames = updatableFiles.map(file => file.name); - - const tables = await Self.rawSql(` - SELECT fileName, toTable, file - FROM edi.tableConfig - WHERE file IN (?)`, [fileNames]); - - for (const table of tables) { - const fileName = table.file; - - console.debug(`Downloading file ${fileName}...`); - - remoteFile = `codes/${fileName}.ZIP`; - tempDir = `${tempPath}/${fileName}`; - tempFile = `${tempPath}/${fileName}.zip`; - - try { - await fs.readFile(tempFile); - } catch (error) { - if (error.code === 'ENOENT') { - const downloadOutput = await downloadFile(remoteFile, tempFile); - if (downloadOutput.error) - continue; - } + if (file.checksum != fileChecksum) { + updatableFiles.push({ + name: file.name, + checksum: fileChecksum + }); + } else + console.debug(`File already updated, skipping...`); } - console.debug(`Extracting file ${fileName}...`); - await extractFile(tempFile, tempDir); + if (updatableFiles.length === 0) + return false; - console.debug(`Updating table ${table.toTable}...`); - await dumpData(tempDir, table); - } + // Download files + const container = await models.TempContainer.container('edi'); + const tempPath = path.join(container.client.root, container.name); - // Update files checksum - for (const file of updatableFiles) { - await Self.rawSql(` - UPDATE edi.fileConfig - SET checksum = ? - WHERE name = ?`, - [file.checksum, file.name]); - } + let remoteFile; + let tempDir; + let tempFile; - // Clean files - try { - await fs.remove(tempPath); + const fileNames = updatableFiles.map(file => file.name); + + const tables = await Self.rawSql(` + SELECT fileName, toTable, file + FROM edi.tableConfig + WHERE file IN (?)`, [fileNames], options); + + for (const table of tables) { + const fileName = table.file; + + remoteFile = `codes/${fileName}.ZIP`; + tempDir = `${tempPath}/${fileName}`; + tempFile = `${tempPath}/${fileName}.zip`; + + try { + await fs.readFile(tempFile); + } catch (error) { + if (error.code === 'ENOENT') { + console.debug(`Downloading file ${fileName}...`); + const downloadOutput = await downloadFile(remoteFile, tempFile); + if (downloadOutput.error) + continue; + } + } + + await extractFile(fileName, tempFile, tempDir); + + console.debug(`Updating table ${table.toTable}...`); + await dumpData(tempDir, table, options); + } + + // Update files checksum + for (const file of updatableFiles) { + console.log(`Updating file ${file.name} checksum...`); + await Self.rawSql(` + UPDATE edi.fileConfig + SET checksum = ? + WHERE name = ?`, + [file.checksum, file.name], options); + } + + await tx.commit(); + + // Clean files + try { + console.debug(`Cleaning files...`); + await fs.remove(tempPath); + } catch (error) { + if (error.code !== 'ENOENT') + throw e; + } + + return true; } catch (error) { - if (error.code !== 'ENOENT') - throw e; + await tx.rollback(); + throw error; } - - return true; }; let ftpClient; @@ -126,9 +136,9 @@ module.exports = Self => { const response = await new Promise((resolve, reject) => { ftpClient.exec((err, response) => { - if (response.error) { + if (err || response.error) { console.debug(`Error downloading checksum file... ${response.error}`); - reject(err); + return reject(err); } resolve(response); @@ -159,9 +169,9 @@ module.exports = Self => { return new Promise((resolve, reject) => { ftpClient.exec((err, response) => { - if (response.error) { + if (err || response.error) { console.debug(`Error downloading file... ${response.error}`); - reject(err); + return reject(err); } resolve(response); @@ -169,11 +179,12 @@ module.exports = Self => { }); } - async function extractFile(tempFile, tempDir) { + async function extractFile(fileName, tempFile, tempDir) { const JSZip = require('jszip'); try { await fs.mkdir(tempDir); + console.debug(`Extracting file ${fileName}...`); } catch (error) { if (error.code !== 'EEXIST') throw e; @@ -196,66 +207,32 @@ module.exports = Self => { } } - async function dumpData(tempDir, table) { + async function dumpData(tempDir, table, options) { const toTable = table.toTable; const baseName = table.fileName; - const firstEntry = entries[0]; - const entryName = firstEntry.entryName; - const startIndex = (entryName.length - 10); - const endIndex = (entryName.length - 4); - const dateString = entryName.substring(startIndex, endIndex); + console.log(`Emptying table ${toTable}...`); + const tableName = `edi.${toTable}`; + await Self.rawSql(`DELETE FROM ??`, [tableName]); - const lastUpdated = new Date(); + const dirFiles = await fs.readdir(tempDir); + const files = dirFiles.filter(file => file.startsWith(baseName)); - let updated = null; - if (file.updated) { - updated = new Date(file.updated); - updated.setHours(0, 0, 0, 0); - } + for (const file of files) { + console.log(`Dumping data from file ${file}...`); - // Format string date to a date object - lastUpdated.setFullYear(`20${dateString.substring(4, 6)}`); - lastUpdated.setMonth(parseInt(dateString.substring(2, 4)) - 1); - lastUpdated.setDate(dateString.substring(0, 2)); - lastUpdated.setHours(0, 0, 0, 0); + const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); + const sqlTemplate = await fs.readFile(templatePath, 'utf8'); + const filePath = path.join(tempDir, file); - if (updated && lastUpdated <= updated) { - console.debug(`Table ${toTable} already updated, skipping...`); - return; - } - - const tx = await Self.beginTransaction({}); - - try { - const options = {transaction: tx}; - - const tableName = `edi.${toTable}`; - await Self.rawSql(`DELETE FROM ??`, [tableName], options); - - const dirFiles = await fs.readdir(tempDir); - const files = dirFiles.filter(file => file.startsWith(baseName)); - - for (const file of files) { - console.log(`Dumping data from file ${file}...`); - - const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); - const sqlTemplate = await fs.readFile(templatePath, 'utf8'); - const filePath = path.join(tempDir, file); - - await Self.rawSql(sqlTemplate, [filePath], options); - await Self.rawSql(` + await Self.rawSql(sqlTemplate, [filePath], options); + await Self.rawSql(` UPDATE edi.tableConfig SET updated = ? WHERE fileName = ? `, [new Date(), baseName], options); - } - - tx.commit(); - } catch (error) { - tx.rollback(); - throw error; } + console.log(`Updated table ${toTable}\n`); } }; diff --git a/front/core/components/smart-table/index.html b/front/core/components/smart-table/index.html index a3295c47e..f26a6b4a2 100644 --- a/front/core/components/smart-table/index.html +++ b/front/core/components/smart-table/index.html @@ -10,9 +10,9 @@