diff --git a/back/methods/edi/sql/item.sql b/back/methods/edi/sql/item.sql index 8d794c7cf..cd1bef88d 100644 --- a/back/methods/edi/sql/item.sql +++ b/back/methods/edi/sql/item.sql @@ -1,9 +1,9 @@ LOAD DATA LOCAL INFILE ? INTO TABLE `edi`.`item` + CHARACTER SET ascii FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12) - SET - id = @col2, + SET id = @col2, product_name = @col4, name = @col5, plant_id = @col7, @@ -11,3 +11,4 @@ LOAD DATA LOCAL INFILE ? entry_date = STR_TO_DATE(@col10, '%Y%m%d'), expiry_date = IFNULL(NULL,STR_TO_DATE(@col11, '%Y%m%d')), change_date_time = STR_TO_DATE(@col12, '%Y%m%d%H%i') + \ No newline at end of file diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index 232695f4e..1057be39b 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -3,236 +3,237 @@ const path = require('path'); const fs = require('fs-extra'); module.exports = Self => { - Self.remoteMethodCtx('updateData', { - description: 'Updates schema data from external provider', - accessType: 'WRITE', - returns: { - type: 'object', - root: true - }, - http: { - path: `/updateData`, - verb: 'POST' - } - }); + Self.remoteMethodCtx('updateData', { + description: 'Updates schema data from external provider', + accessType: 'WRITE', + returns: { + type: 'object', + root: true + }, + http: { + path: `/updateData`, + verb: 'POST' + } + }); - Self.updateData = async() => { - const models = Self.app.models; + Self.updateData = async() => { + const models = Self.app.models; - // Get files checksum - const tx = await Self.beginTransaction({}); + // Get files checksum + const tx = await Self.beginTransaction({}); - try { - const options = {transaction: tx}; - const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options); + try { + const options = {transaction: tx}; + const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options); - const updatableFiles = []; - for (const file of files) { - const fileChecksum = await getChecksum(file); + const updatableFiles = []; + for (const file of files) { + const fileChecksum = await getChecksum(file); - if (file.checksum != fileChecksum) { - updatableFiles.push({ - name: file.name, - checksum: fileChecksum - }); - } else - console.debug(`File already updated, skipping...`); - } + if (file.checksum != fileChecksum) { + updatableFiles.push({ + name: file.name, + checksum: fileChecksum + }); + } else + console.debug(`File already updated, skipping...`); + } - if (updatableFiles.length === 0) - return false; + if (updatableFiles.length === 0) + return false; - // Download files - const container = await models.TempContainer.container('edi'); - const tempPath = path.join(container.client.root, container.name); + // Download files + const container = await models.TempContainer.container('edi'); + const tempPath = path.join(container.client.root, container.name); - let remoteFile; - let tempDir; - let tempFile; + let remoteFile; + let tempDir; + let tempFile; - const fileNames = updatableFiles.map(file => file.name); + const fileNames = updatableFiles.map(file => file.name); - const tables = await Self.rawSql(` - SELECT fileName, toTable, file - FROM edi.tableConfig - WHERE file IN (?)`, [fileNames], options); + const tables = await Self.rawSql(` + SELECT fileName, toTable, file + FROM edi.tableConfig + WHERE file IN (?)`, [fileNames], options); - for (const table of tables) { - const fileName = table.file; + for (const table of tables) { + const fileName = table.file; - remoteFile = `codes/${fileName}.ZIP`; - tempDir = `${tempPath}/${fileName}`; - tempFile = `${tempPath}/${fileName}.zip`; + remoteFile = `codes/${fileName}.ZIP`; + tempDir = `${tempPath}/${fileName}`; + tempFile = `${tempPath}/${fileName}.zip`; - try { - await fs.readFile(tempFile); - } catch (error) { - if (error.code === 'ENOENT') { - console.debug(`Downloading file ${fileName}...`); - const downloadOutput = await downloadFile(remoteFile, tempFile); - if (downloadOutput.error) - continue; - } - } + try { + await fs.readFile(tempFile); + } catch (error) { + if (error.code === 'ENOENT') { + console.debug(`Downloading file ${fileName}...`); + const downloadOutput = await downloadFile(remoteFile, tempFile); + if (downloadOutput.error) + continue; + } + } - await extractFile(fileName, tempFile, tempDir); + await extractFile(fileName, tempFile, tempDir); - console.debug(`Updating table ${table.toTable}...`); - await dumpData(tempDir, table, options); - } + console.debug(`Updating table ${table.toTable}...`); + await dumpData(tempDir, table, options); + } - // Update files checksum - for (const file of updatableFiles) { - console.log(`Updating file ${file.name} checksum...`); - await Self.rawSql(` - UPDATE edi.fileConfig - SET checksum = ? - WHERE name = ?`, - [file.checksum, file.name], options); - } + // Update files checksum + for (const file of updatableFiles) { + console.log(`Updating file ${file.name} checksum...`); + await Self.rawSql(` + UPDATE edi.fileConfig + SET checksum = ? + WHERE name = ?`, + [file.checksum, file.name], options); + } - await tx.commit(); + await tx.commit(); - // Clean files - try { - console.debug(`Cleaning files...`); - await fs.remove(tempPath); - } catch (error) { - if (error.code !== 'ENOENT') - throw e; - } + // Clean files + try { + console.debug(`Cleaning files...`); + await fs.remove(tempPath); + } catch (error) { + if (error.code !== 'ENOENT') + throw e; + } - return true; - } catch (error) { - await tx.rollback(); - throw error; - } - }; + return true; + } catch (error) { + await tx.rollback(); + throw error; + } + }; - let ftpClient; - async function getFtpClient() { - if (!ftpClient) { - const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig'); - console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`); + let ftpClient; + async function getFtpClient() { + if (!ftpClient) { + const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig'); + console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`); - const FtpClient = require('ftps'); + const FtpClient = require('ftps'); - ftpClient = new FtpClient({ - host: ftpConfig.host, - username: ftpConfig.user, - password: ftpConfig.password, - procotol: 'ftp' - }); - } + ftpClient = new FtpClient({ + host: ftpConfig.host, + username: ftpConfig.user, + password: ftpConfig.password, + procotol: 'ftp', + additionalLftpCommands: 'set ssl:verify-certificate no' + }); + } - return ftpClient; - } + return ftpClient; + } - async function getChecksum(file) { - const ftpClient = await getFtpClient(); - console.debug(`Checking checksum for file ${file.name}...`); + async function getChecksum(file) { + const ftpClient = await getFtpClient(); + console.debug(`Checking checksum for file ${file.name}...`); - ftpClient.cat(`codes/${file.name}.txt`); + ftpClient.cat(`codes/${file.name}.TXT`); - const response = await new Promise((resolve, reject) => { - ftpClient.exec((err, response) => { - if (err || response.error) { - console.debug(`Error downloading checksum file... ${response.error}`); - return reject(err); - } + const response = await new Promise((resolve, reject) => { + ftpClient.exec((err, response) => { + if (err || response.error) { + console.debug(`Error downloading checksum file... ${response.error}`); + return reject(err); + } - resolve(response); - }); - }); + resolve(response); + }); + }); - if (response && response.data) { - const fileContents = response.data; - const rows = fileContents.split('\n'); - const row = rows[4]; - const columns = row.split(/\s+/); + if (response && response.data) { + const fileContents = response.data; + const rows = fileContents.split('\n'); + const row = rows[4]; + const columns = row.split(/\s+/); - let fileChecksum; - if (file.keyValue) - fileChecksum = columns[1]; + let fileChecksum; + if (file.keyValue) + fileChecksum = columns[1]; - if (!file.keyValue) - fileChecksum = columns[0]; + if (!file.keyValue) + fileChecksum = columns[0]; - return fileChecksum; - } - } + return fileChecksum; + } + } - async function downloadFile(remoteFile, tempFile) { - const ftpClient = await getFtpClient(); + async function downloadFile(remoteFile, tempFile) { + const ftpClient = await getFtpClient(); - ftpClient.get(remoteFile, tempFile); + ftpClient.get(remoteFile, tempFile); - return new Promise((resolve, reject) => { - ftpClient.exec((err, response) => { - if (err || response.error) { - console.debug(`Error downloading file... ${response.error}`); - return reject(err); - } + return new Promise((resolve, reject) => { + ftpClient.exec((err, response) => { + if (err || response.error) { + console.debug(`Error downloading file... ${response.error}`); + return reject(err); + } - resolve(response); - }); - }); - } + resolve(response); + }); + }); + } - async function extractFile(fileName, tempFile, tempDir) { - const JSZip = require('jszip'); + async function extractFile(fileName, tempFile, tempDir) { + const JSZip = require('jszip'); - try { - await fs.mkdir(tempDir); - console.debug(`Extracting file ${fileName}...`); - } catch (error) { - if (error.code !== 'EEXIST') - throw e; - } + try { + await fs.mkdir(tempDir); + console.debug(`Extracting file ${fileName}...`); + } catch (error) { + if (error.code !== 'EEXIST') + throw e; + } - const fileStream = await fs.readFile(tempFile); - if (fileStream) { - const zip = new JSZip(); - const zipContents = await zip.loadAsync(fileStream); + const fileStream = await fs.readFile(tempFile); + if (fileStream) { + const zip = new JSZip(); + const zipContents = await zip.loadAsync(fileStream); - if (!zipContents) return; + if (!zipContents) return; - const fileNames = Object.keys(zipContents.files); + const fileNames = Object.keys(zipContents.files); - for (const fileName of fileNames) { - const fileContent = await zip.file(fileName).async('nodebuffer'); - const dest = path.join(tempDir, fileName); - await fs.writeFile(dest, fileContent); - } - } - } + for (const fileName of fileNames) { + const fileContent = await zip.file(fileName).async('nodebuffer'); + const dest = path.join(tempDir, fileName); + await fs.writeFile(dest, fileContent); + } + } + } - async function dumpData(tempDir, table, options) { - const toTable = table.toTable; - const baseName = table.fileName; + async function dumpData(tempDir, table, options) { + const toTable = table.toTable; + const baseName = table.fileName; - console.log(`Emptying table ${toTable}...`); - const tableName = `edi.${toTable}`; - await Self.rawSql(`DELETE FROM ??`, [tableName]); + console.log(`Emptying table ${toTable}...`); + const tableName = `edi.${toTable}`; + await Self.rawSql(`DELETE FROM ??`, [tableName]); - const dirFiles = await fs.readdir(tempDir); - const files = dirFiles.filter(file => file.startsWith(baseName)); + const dirFiles = await fs.readdir(tempDir); + const files = dirFiles.filter(file => file.startsWith(baseName)); - for (const file of files) { - console.log(`Dumping data from file ${file}...`); + for (const file of files) { + console.log(`Dumping data from file ${file}...`); - const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); - const sqlTemplate = await fs.readFile(templatePath, 'utf8'); - const filePath = path.join(tempDir, file); + const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); + const sqlTemplate = await fs.readFile(templatePath, 'utf8'); + const filePath = path.join(tempDir, file); - await Self.rawSql(sqlTemplate, [filePath], options); - await Self.rawSql(` - UPDATE edi.tableConfig - SET updated = ? - WHERE fileName = ? - `, [Date.vnNew(), baseName], options); - } + await Self.rawSql(sqlTemplate, [filePath], options); + await Self.rawSql(` + UPDATE edi.tableConfig + SET updated = ? + WHERE fileName = ? + `, [Date.vnNew(), baseName], options); + } - console.log(`Updated table ${toTable}\n`); - } + console.log(`Updated table ${toTable}\n`); + } };