/* eslint no-console: "off" */ const path = require('path'); const fs = require('fs-extra'); module.exports = Self => { Self.remoteMethodCtx('updateData', { description: 'Updates schema data from external provider', accessType: 'WRITE', returns: { type: 'object', root: true }, http: { path: `/updateData`, verb: 'GET' } }); Self.updateData = async() => { const models = Self.app.models; // Get files checksum const tx = await Self.beginTransaction({}); try { const options = {transaction: tx}; const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options); const updatableFiles = []; for (const file of files) { const fileChecksum = await getChecksum(file); if (file.checksum != fileChecksum) { updatableFiles.push({ name: file.name, checksum: fileChecksum }); } else console.debug(`File already updated, skipping...`); } if (updatableFiles.length === 0) return false; // Download files const container = await models.TempContainer.container('edi'); const tempPath = path.join(container.client.root, container.name); let remoteFile; let tempDir; let tempFile; const fileNames = updatableFiles.map(file => file.name); const tables = await Self.rawSql(` SELECT fileName, toTable, file FROM edi.tableConfig WHERE file IN (?)`, [fileNames], options); for (const table of tables) { const fileName = table.file; remoteFile = `codes/${fileName}.ZIP`; tempDir = `${tempPath}/${fileName}`; tempFile = `${tempPath}/${fileName}.zip`; try { await fs.readFile(tempFile); } catch (error) { if (error.code === 'ENOENT') { console.debug(`Downloading file ${fileName}...`); const downloadOutput = await downloadFile(remoteFile, tempFile); if (downloadOutput.error) continue; } } await extractFile(fileName, tempFile, tempDir); console.debug(`Updating table ${table.toTable}...`); await dumpData(tempDir, table, options); } // Update files checksum for (const file of updatableFiles) { console.log(`Updating file ${file.name} checksum...`); await Self.rawSql(` UPDATE edi.fileConfig SET checksum = ? WHERE name = ?`, [file.checksum, file.name], options); } await tx.commit(); // Clean files try { console.debug(`Cleaning files...`); await fs.remove(tempPath); } catch (error) { if (error.code !== 'ENOENT') throw e; } return true; } catch (error) { await tx.rollback(); throw error; } }; let ftpClient; async function getFtpClient() { if (!ftpClient) { const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig'); console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`); const FtpClient = require('ftps'); ftpClient = new FtpClient({ host: ftpConfig.host, username: ftpConfig.user, password: ftpConfig.password, procotol: 'ftp' }); } return ftpClient; } async function getChecksum(file) { const ftpClient = await getFtpClient(); console.debug(`Checking checksum for file ${file.name}...`); ftpClient.cat(`codes/${file.name}.txt`); const response = await new Promise((resolve, reject) => { ftpClient.exec((err, response) => { if (err || response.error) { console.debug(`Error downloading checksum file... ${response.error}`); return reject(err); } resolve(response); }); }); if (response && response.data) { const fileContents = response.data; const rows = fileContents.split('\n'); const row = rows[4]; const columns = row.split(/\s+/); let fileChecksum; if (file.keyValue) fileChecksum = columns[1]; if (!file.keyValue) fileChecksum = columns[0]; return fileChecksum; } } async function downloadFile(remoteFile, tempFile) { const ftpClient = await getFtpClient(); ftpClient.get(remoteFile, tempFile); return new Promise((resolve, reject) => { ftpClient.exec((err, response) => { if (err || response.error) { console.debug(`Error downloading file... ${response.error}`); return reject(err); } resolve(response); }); }); } async function extractFile(fileName, tempFile, tempDir) { const JSZip = require('jszip'); try { await fs.mkdir(tempDir); console.debug(`Extracting file ${fileName}...`); } catch (error) { if (error.code !== 'EEXIST') throw e; } const fileStream = await fs.readFile(tempFile); if (fileStream) { const zip = new JSZip(); const zipContents = await zip.loadAsync(fileStream); if (!zipContents) return; const fileNames = Object.keys(zipContents.files); for (const fileName of fileNames) { const fileContent = await zip.file(fileName).async('nodebuffer'); const dest = path.join(tempDir, fileName); await fs.writeFile(dest, fileContent); } } } async function dumpData(tempDir, table, options) { const toTable = table.toTable; const baseName = table.fileName; console.log(`Emptying table ${toTable}...`); const tableName = `edi.${toTable}`; await Self.rawSql(`DELETE FROM ??`, [tableName]); const dirFiles = await fs.readdir(tempDir); const files = dirFiles.filter(file => file.startsWith(baseName)); for (const file of files) { console.log(`Dumping data from file ${file}...`); const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); const sqlTemplate = await fs.readFile(templatePath, 'utf8'); const filePath = path.join(tempDir, file); await Self.rawSql(sqlTemplate, [filePath], options); await Self.rawSql(` UPDATE edi.tableConfig SET updated = ? WHERE fileName = ? `, [new Date(), baseName], options); } console.log(`Updated table ${toTable}\n`); } };