/* eslint no-console: "off" */ const path = require('path'); const fs = require('fs-extra'); module.exports = Self => { Self.remoteMethodCtx('updateData', { description: 'Updates schema data from external provider', accessType: 'WRITE', returns: { type: 'object', root: true }, http: { path: `/updateData`, verb: 'GET' } }); Self.updateData = async() => { const models = Self.app.models; // Get files checksum const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig'); const updatableFiles = []; for (const file of files) { const fileChecksum = await getChecksum(file); if (file.checksum != fileChecksum) { updatableFiles.push({ name: file.name, checksum: fileChecksum }); } else console.debug(`File already updated, skipping...`); } if (updatableFiles.length === 0) return false; // Download files const container = await models.TempContainer.container('edi'); const tempPath = path.join(container.client.root, container.name); let remoteFile; let tempDir; let tempFile; const fileNames = updatableFiles.map(file => file.name); const tables = await Self.rawSql(` SELECT fileName, toTable, file FROM edi.tableConfig WHERE file IN (?)`, [fileNames]); for (const table of tables) { const fileName = table.file; console.debug(`Downloading file ${fileName}...`); remoteFile = `codes/${fileName}.ZIP`; tempDir = `${tempPath}/${fileName}`; tempFile = `${tempPath}/${fileName}.zip`; try { await fs.readFile(tempFile); } catch (error) { if (error.code === 'ENOENT') { const downloadOutput = await downloadFile(remoteFile, tempFile); if (downloadOutput.error) continue; } } console.debug(`Extracting file ${fileName}...`); await extractFile(tempFile, tempDir); console.debug(`Updating table ${table.toTable}...`); await dumpData(tempDir, table); } // Update files checksum for (const file of updatableFiles) { await Self.rawSql(` UPDATE edi.fileConfig SET checksum = ? WHERE name = ?`, [file.checksum, file.name]); } // Clean files try { await fs.remove(tempPath); } catch (error) { if (error.code !== 'ENOENT') throw e; } return true; }; let ftpClient; async function getFtpClient() { if (!ftpClient) { const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig'); console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`); const FtpClient = require('ftps'); ftpClient = new FtpClient({ host: ftpConfig.host, username: ftpConfig.user, password: ftpConfig.password, procotol: 'ftp' }); } return ftpClient; } async function getChecksum(file) { const ftpClient = await getFtpClient(); console.debug(`Checking checksum for file ${file.name}...`); ftpClient.cat(`codes/${file.name}.txt`); const response = await new Promise((resolve, reject) => { ftpClient.exec((err, response) => { if (response.error) { console.debug(`Error downloading checksum file... ${response.error}`); reject(err); } resolve(response); }); }); if (response && response.data) { const fileContents = response.data; const rows = fileContents.split('\n'); const row = rows[4]; const columns = row.split(/\s+/); let fileChecksum; if (file.keyValue) fileChecksum = columns[1]; if (!file.keyValue) fileChecksum = columns[0]; return fileChecksum; } } async function downloadFile(remoteFile, tempFile) { const ftpClient = await getFtpClient(); ftpClient.get(remoteFile, tempFile); return new Promise((resolve, reject) => { ftpClient.exec((err, response) => { if (response.error) { console.debug(`Error downloading file... ${response.error}`); reject(err); } resolve(response); }); }); } async function extractFile(tempFile, tempDir) { const JSZip = require('jszip'); try { await fs.mkdir(tempDir); } catch (error) { if (error.code !== 'EEXIST') throw e; } const fileStream = await fs.readFile(tempFile); if (fileStream) { const zip = new JSZip(); const zipContents = await zip.loadAsync(fileStream); if (!zipContents) return; const fileNames = Object.keys(zipContents.files); for (const fileName of fileNames) { const fileContent = await zip.file(fileName).async('nodebuffer'); const dest = path.join(tempDir, fileName); await fs.writeFile(dest, fileContent); } } } async function dumpData(tempDir, table) { const toTable = table.toTable; const baseName = table.fileName; const firstEntry = entries[0]; const entryName = firstEntry.entryName; const startIndex = (entryName.length - 10); const endIndex = (entryName.length - 4); const dateString = entryName.substring(startIndex, endIndex); const lastUpdated = new Date(); let updated = null; if (file.updated) { updated = new Date(file.updated); updated.setHours(0, 0, 0, 0); } // Format string date to a date object lastUpdated.setFullYear(`20${dateString.substring(4, 6)}`); lastUpdated.setMonth(parseInt(dateString.substring(2, 4)) - 1); lastUpdated.setDate(dateString.substring(0, 2)); lastUpdated.setHours(0, 0, 0, 0); if (updated && lastUpdated <= updated) { console.debug(`Table ${toTable} already updated, skipping...`); return; } const tx = await Self.beginTransaction({}); try { const options = {transaction: tx}; const tableName = `edi.${toTable}`; await Self.rawSql(`DELETE FROM ??`, [tableName], options); const dirFiles = await fs.readdir(tempDir); const files = dirFiles.filter(file => file.startsWith(baseName)); for (const file of files) { console.log(`Dumping data from file ${file}...`); const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); const sqlTemplate = await fs.readFile(templatePath, 'utf8'); const filePath = path.join(tempDir, file); await Self.rawSql(sqlTemplate, [filePath], options); await Self.rawSql(` UPDATE edi.tableConfig SET updated = ? WHERE fileName = ? `, [new Date(), baseName], options); } tx.commit(); } catch (error) { tx.rollback(); throw error; } console.log(`Updated table ${toTable}\n`); } };