verdnatura/salix
verdnatura
/
salix
3
4
Fork 0
Code Issues Pull Requests 57 Releases Wiki Activity

Merge pull request 'refs #5684 edi/updateData fix' (!1585) from 5684-ediUpdateData into master
gitea/salix/pipeline/head This commit looks good Details 

Reviewed-on: #1585
Reviewed-by: Alex Moreno <alexm@verdnatura.es>
This commit is contained in:
Guillermo Bonet 2023-06-02 06:29:50 +00:00
parent 8d0ae30610 5b0a1d10bb
commit 20e574aef4
2 changed files with 188 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
back/methods/edi/sql/item.sql
View File

@ -1,9 +1,9 @@
LOAD DATA LOCAL INFILE ? LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`item` INTO TABLE `edi`.`item`
CHARACTER SET ascii
FIELDS TERMINATED BY ';' FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12) LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12)
SET SET id = @col2,
id = @col2,
product_name = @col4, product_name = @col4,
name = @col5, name = @col5,
plant_id = @col7, plant_id = @col7,
@ -11,3 +11,4 @@ LOAD DATA LOCAL INFILE ?
entry_date = STR_TO_DATE(@col10, '%Y%m%d'), entry_date = STR_TO_DATE(@col10, '%Y%m%d'),
expiry_date = IFNULL(NULL,STR_TO_DATE(@col11, '%Y%m%d')), expiry_date = IFNULL(NULL,STR_TO_DATE(@col11, '%Y%m%d')),
change_date_time = STR_TO_DATE(@col12, '%Y%m%d%H%i') change_date_time = STR_TO_DATE(@col12, '%Y%m%d%H%i')

369
back/methods/edi/updateData.js
View File

@ -3,236 +3,237 @@ const path = require('path');
const fs = require('fs-extra'); const fs = require('fs-extra');
module.exports = Self => { module.exports = Self => {
Self.remoteMethodCtx('updateData', { Self.remoteMethodCtx('updateData', {
description: 'Updates schema data from external provider', description: 'Updates schema data from external provider',
accessType: 'WRITE', accessType: 'WRITE',
returns: { returns: {
type: 'object', type: 'object',
root: true root: true
}, },
http: { http: {
path: `/updateData`, path: `/updateData`,
verb: 'POST' verb: 'POST'
} }
}); });
Self.updateData = async() => { Self.updateData = async() => {
const models = Self.app.models; const models = Self.app.models;
// Get files checksum // Get files checksum
const tx = await Self.beginTransaction({}); const tx = await Self.beginTransaction({});
try { try {
const options = {transaction: tx}; const options = {transaction: tx};
const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options); const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options);
const updatableFiles = []; const updatableFiles = [];
for (const file of files) { for (const file of files) {
const fileChecksum = await getChecksum(file); const fileChecksum = await getChecksum(file);
if (file.checksum != fileChecksum) { if (file.checksum != fileChecksum) {
updatableFiles.push({ updatableFiles.push({
name: file.name, name: file.name,
checksum: fileChecksum checksum: fileChecksum
}); });
} else } else
console.debug(`File already updated, skipping...`); console.debug(`File already updated, skipping...`);
} }
if (updatableFiles.length === 0) if (updatableFiles.length === 0)
return false; return false;
// Download files // Download files
const container = await models.TempContainer.container('edi'); const container = await models.TempContainer.container('edi');
const tempPath = path.join(container.client.root, container.name); const tempPath = path.join(container.client.root, container.name);
let remoteFile; let remoteFile;
let tempDir; let tempDir;
let tempFile; let tempFile;
const fileNames = updatableFiles.map(file => file.name); const fileNames = updatableFiles.map(file => file.name);
const tables = await Self.rawSql(` const tables = await Self.rawSql(`
SELECT fileName, toTable, file SELECT fileName, toTable, file
FROM edi.tableConfig FROM edi.tableConfig
WHERE file IN (?)`, [fileNames], options); WHERE file IN (?)`, [fileNames], options);
for (const table of tables) { for (const table of tables) {
const fileName = table.file; const fileName = table.file;
remoteFile = `codes/${fileName}.ZIP`; remoteFile = `codes/${fileName}.ZIP`;
tempDir = `${tempPath}/${fileName}`; tempDir = `${tempPath}/${fileName}`;
tempFile = `${tempPath}/${fileName}.zip`; tempFile = `${tempPath}/${fileName}.zip`;
try { try {
await fs.readFile(tempFile); await fs.readFile(tempFile);
} catch (error) { } catch (error) {
if (error.code === 'ENOENT') { if (error.code === 'ENOENT') {
console.debug(`Downloading file ${fileName}...`); console.debug(`Downloading file ${fileName}...`);
const downloadOutput = await downloadFile(remoteFile, tempFile); const downloadOutput = await downloadFile(remoteFile, tempFile);
if (downloadOutput.error) if (downloadOutput.error)
continue; continue;
} }
} }
await extractFile(fileName, tempFile, tempDir); await extractFile(fileName, tempFile, tempDir);
console.debug(`Updating table ${table.toTable}...`); console.debug(`Updating table ${table.toTable}...`);
await dumpData(tempDir, table, options); await dumpData(tempDir, table, options);
} }
// Update files checksum // Update files checksum
for (const file of updatableFiles) { for (const file of updatableFiles) {
console.log(`Updating file ${file.name} checksum...`); console.log(`Updating file ${file.name} checksum...`);
await Self.rawSql(` await Self.rawSql(`
UPDATE edi.fileConfig UPDATE edi.fileConfig
SET checksum = ? SET checksum = ?
WHERE name = ?`, WHERE name = ?`,
[file.checksum, file.name], options); [file.checksum, file.name], options);
} }
await tx.commit(); await tx.commit();
// Clean files // Clean files
try { try {
console.debug(`Cleaning files...`); console.debug(`Cleaning files...`);
await fs.remove(tempPath); await fs.remove(tempPath);
} catch (error) { } catch (error) {
if (error.code !== 'ENOENT') if (error.code !== 'ENOENT')
throw e; throw e;
} }
return true; return true;
} catch (error) { } catch (error) {
await tx.rollback(); await tx.rollback();
throw error; throw error;
} }
}; };
let ftpClient; let ftpClient;
async function getFtpClient() { async function getFtpClient() {
if (!ftpClient) { if (!ftpClient) {
const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig'); const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig');
console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`); console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`);
const FtpClient = require('ftps'); const FtpClient = require('ftps');
ftpClient = new FtpClient({ ftpClient = new FtpClient({
host: ftpConfig.host, host: ftpConfig.host,
username: ftpConfig.user, username: ftpConfig.user,
password: ftpConfig.password, password: ftpConfig.password,
procotol: 'ftp' procotol: 'ftp',
}); additionalLftpCommands: 'set ssl:verify-certificate no'
} });
}
return ftpClient; return ftpClient;
} }
async function getChecksum(file) { async function getChecksum(file) {
const ftpClient = await getFtpClient(); const ftpClient = await getFtpClient();
console.debug(`Checking checksum for file ${file.name}...`); console.debug(`Checking checksum for file ${file.name}...`);
ftpClient.cat(`codes/${file.name}.txt`); ftpClient.cat(`codes/${file.name}.TXT`);
const response = await new Promise((resolve, reject) => { const response = await new Promise((resolve, reject) => {
ftpClient.exec((err, response) => { ftpClient.exec((err, response) => {
if (err || response.error) { if (err || response.error) {
console.debug(`Error downloading checksum file... ${response.error}`); console.debug(`Error downloading checksum file... ${response.error}`);
return reject(err); return reject(err);
} }
resolve(response); resolve(response);
}); });
}); });
if (response && response.data) { if (response && response.data) {
const fileContents = response.data; const fileContents = response.data;
const rows = fileContents.split('\n'); const rows = fileContents.split('\n');
const row = rows[4]; const row = rows[4];
const columns = row.split(/\s+/); const columns = row.split(/\s+/);
let fileChecksum; let fileChecksum;
if (file.keyValue) if (file.keyValue)
fileChecksum = columns[1]; fileChecksum = columns[1];
if (!file.keyValue) if (!file.keyValue)
fileChecksum = columns[0]; fileChecksum = columns[0];
return fileChecksum; return fileChecksum;
} }
} }
async function downloadFile(remoteFile, tempFile) { async function downloadFile(remoteFile, tempFile) {
const ftpClient = await getFtpClient(); const ftpClient = await getFtpClient();
ftpClient.get(remoteFile, tempFile); ftpClient.get(remoteFile, tempFile);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
ftpClient.exec((err, response) => { ftpClient.exec((err, response) => {
if (err || response.error) { if (err || response.error) {
console.debug(`Error downloading file... ${response.error}`); console.debug(`Error downloading file... ${response.error}`);
return reject(err); return reject(err);
} }
resolve(response); resolve(response);
}); });
}); });
} }
async function extractFile(fileName, tempFile, tempDir) { async function extractFile(fileName, tempFile, tempDir) {
const JSZip = require('jszip'); const JSZip = require('jszip');
try { try {
await fs.mkdir(tempDir); await fs.mkdir(tempDir);
console.debug(`Extracting file ${fileName}...`); console.debug(`Extracting file ${fileName}...`);
} catch (error) { } catch (error) {
if (error.code !== 'EEXIST') if (error.code !== 'EEXIST')
throw e; throw e;
} }
const fileStream = await fs.readFile(tempFile); const fileStream = await fs.readFile(tempFile);
if (fileStream) { if (fileStream) {
const zip = new JSZip(); const zip = new JSZip();
const zipContents = await zip.loadAsync(fileStream); const zipContents = await zip.loadAsync(fileStream);
if (!zipContents) return; if (!zipContents) return;
const fileNames = Object.keys(zipContents.files); const fileNames = Object.keys(zipContents.files);
for (const fileName of fileNames) { for (const fileName of fileNames) {
const fileContent = await zip.file(fileName).async('nodebuffer'); const fileContent = await zip.file(fileName).async('nodebuffer');
const dest = path.join(tempDir, fileName); const dest = path.join(tempDir, fileName);
await fs.writeFile(dest, fileContent); await fs.writeFile(dest, fileContent);
} }
} }
} }
async function dumpData(tempDir, table, options) { async function dumpData(tempDir, table, options) {
const toTable = table.toTable; const toTable = table.toTable;
const baseName = table.fileName; const baseName = table.fileName;
console.log(`Emptying table ${toTable}...`); console.log(`Emptying table ${toTable}...`);
const tableName = `edi.${toTable}`; const tableName = `edi.${toTable}`;
await Self.rawSql(`DELETE FROM ??`, [tableName]); await Self.rawSql(`DELETE FROM ??`, [tableName]);
const dirFiles = await fs.readdir(tempDir); const dirFiles = await fs.readdir(tempDir);
const files = dirFiles.filter(file => file.startsWith(baseName)); const files = dirFiles.filter(file => file.startsWith(baseName));
for (const file of files) { for (const file of files) {
console.log(`Dumping data from file ${file}...`); console.log(`Dumping data from file ${file}...`);
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
const sqlTemplate = await fs.readFile(templatePath, 'utf8'); const sqlTemplate = await fs.readFile(templatePath, 'utf8');
const filePath = path.join(tempDir, file); const filePath = path.join(tempDir, file);
await Self.rawSql(sqlTemplate, [filePath], options); await Self.rawSql(sqlTemplate, [filePath], options);
await Self.rawSql(` await Self.rawSql(`
UPDATE edi.tableConfig UPDATE edi.tableConfig
SET updated = ? SET updated = ?
WHERE fileName = ? WHERE fileName = ?
`, [Date.vnNew(), baseName], options); `, [Date.vnNew(), baseName], options);
} }
console.log(`Updated table ${toTable}\n`); console.log(`Updated table ${toTable}\n`);
} }
}; };