Merge pull request 'refs #5684 edi/updateData fix' (!1585) from 5684-ediUpdateData into master
gitea/salix/pipeline/head This commit looks good
Details
gitea/salix/pipeline/head This commit looks good
Details
Reviewed-on: #1585 Reviewed-by: Alex Moreno <alexm@verdnatura.es>
This commit is contained in:
commit
20e574aef4
|
|
@ -1,9 +1,9 @@
|
|||
LOAD DATA LOCAL INFILE ?
|
||||
INTO TABLE `edi`.`item`
|
||||
CHARACTER SET ascii
|
||||
FIELDS TERMINATED BY ';'
|
||||
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12)
|
||||
SET
|
||||
id = @col2,
|
||||
SET id = @col2,
|
||||
product_name = @col4,
|
||||
name = @col5,
|
||||
plant_id = @col7,
|
||||
|
|
@ -11,3 +11,4 @@ LOAD DATA LOCAL INFILE ?
|
|||
entry_date = STR_TO_DATE(@col10, '%Y%m%d'),
|
||||
expiry_date = IFNULL(NULL,STR_TO_DATE(@col11, '%Y%m%d')),
|
||||
change_date_time = STR_TO_DATE(@col12, '%Y%m%d%H%i')
|
||||
|
||||
|
|
@ -3,236 +3,237 @@ const path = require('path');
|
|||
const fs = require('fs-extra');
|
||||
|
||||
module.exports = Self => {
|
||||
Self.remoteMethodCtx('updateData', {
|
||||
description: 'Updates schema data from external provider',
|
||||
accessType: 'WRITE',
|
||||
returns: {
|
||||
type: 'object',
|
||||
root: true
|
||||
},
|
||||
http: {
|
||||
path: `/updateData`,
|
||||
verb: 'POST'
|
||||
}
|
||||
});
|
||||
Self.remoteMethodCtx('updateData', {
|
||||
description: 'Updates schema data from external provider',
|
||||
accessType: 'WRITE',
|
||||
returns: {
|
||||
type: 'object',
|
||||
root: true
|
||||
},
|
||||
http: {
|
||||
path: `/updateData`,
|
||||
verb: 'POST'
|
||||
}
|
||||
});
|
||||
|
||||
Self.updateData = async() => {
|
||||
const models = Self.app.models;
|
||||
Self.updateData = async() => {
|
||||
const models = Self.app.models;
|
||||
|
||||
// Get files checksum
|
||||
const tx = await Self.beginTransaction({});
|
||||
// Get files checksum
|
||||
const tx = await Self.beginTransaction({});
|
||||
|
||||
try {
|
||||
const options = {transaction: tx};
|
||||
const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options);
|
||||
try {
|
||||
const options = {transaction: tx};
|
||||
const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options);
|
||||
|
||||
const updatableFiles = [];
|
||||
for (const file of files) {
|
||||
const fileChecksum = await getChecksum(file);
|
||||
const updatableFiles = [];
|
||||
for (const file of files) {
|
||||
const fileChecksum = await getChecksum(file);
|
||||
|
||||
if (file.checksum != fileChecksum) {
|
||||
updatableFiles.push({
|
||||
name: file.name,
|
||||
checksum: fileChecksum
|
||||
});
|
||||
} else
|
||||
console.debug(`File already updated, skipping...`);
|
||||
}
|
||||
if (file.checksum != fileChecksum) {
|
||||
updatableFiles.push({
|
||||
name: file.name,
|
||||
checksum: fileChecksum
|
||||
});
|
||||
} else
|
||||
console.debug(`File already updated, skipping...`);
|
||||
}
|
||||
|
||||
if (updatableFiles.length === 0)
|
||||
return false;
|
||||
if (updatableFiles.length === 0)
|
||||
return false;
|
||||
|
||||
// Download files
|
||||
const container = await models.TempContainer.container('edi');
|
||||
const tempPath = path.join(container.client.root, container.name);
|
||||
// Download files
|
||||
const container = await models.TempContainer.container('edi');
|
||||
const tempPath = path.join(container.client.root, container.name);
|
||||
|
||||
let remoteFile;
|
||||
let tempDir;
|
||||
let tempFile;
|
||||
let remoteFile;
|
||||
let tempDir;
|
||||
let tempFile;
|
||||
|
||||
const fileNames = updatableFiles.map(file => file.name);
|
||||
const fileNames = updatableFiles.map(file => file.name);
|
||||
|
||||
const tables = await Self.rawSql(`
|
||||
SELECT fileName, toTable, file
|
||||
FROM edi.tableConfig
|
||||
WHERE file IN (?)`, [fileNames], options);
|
||||
const tables = await Self.rawSql(`
|
||||
SELECT fileName, toTable, file
|
||||
FROM edi.tableConfig
|
||||
WHERE file IN (?)`, [fileNames], options);
|
||||
|
||||
for (const table of tables) {
|
||||
const fileName = table.file;
|
||||
for (const table of tables) {
|
||||
const fileName = table.file;
|
||||
|
||||
remoteFile = `codes/${fileName}.ZIP`;
|
||||
tempDir = `${tempPath}/${fileName}`;
|
||||
tempFile = `${tempPath}/${fileName}.zip`;
|
||||
remoteFile = `codes/${fileName}.ZIP`;
|
||||
tempDir = `${tempPath}/${fileName}`;
|
||||
tempFile = `${tempPath}/${fileName}.zip`;
|
||||
|
||||
try {
|
||||
await fs.readFile(tempFile);
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
console.debug(`Downloading file ${fileName}...`);
|
||||
const downloadOutput = await downloadFile(remoteFile, tempFile);
|
||||
if (downloadOutput.error)
|
||||
continue;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await fs.readFile(tempFile);
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
console.debug(`Downloading file ${fileName}...`);
|
||||
const downloadOutput = await downloadFile(remoteFile, tempFile);
|
||||
if (downloadOutput.error)
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
await extractFile(fileName, tempFile, tempDir);
|
||||
await extractFile(fileName, tempFile, tempDir);
|
||||
|
||||
console.debug(`Updating table ${table.toTable}...`);
|
||||
await dumpData(tempDir, table, options);
|
||||
}
|
||||
console.debug(`Updating table ${table.toTable}...`);
|
||||
await dumpData(tempDir, table, options);
|
||||
}
|
||||
|
||||
// Update files checksum
|
||||
for (const file of updatableFiles) {
|
||||
console.log(`Updating file ${file.name} checksum...`);
|
||||
await Self.rawSql(`
|
||||
UPDATE edi.fileConfig
|
||||
SET checksum = ?
|
||||
WHERE name = ?`,
|
||||
[file.checksum, file.name], options);
|
||||
}
|
||||
// Update files checksum
|
||||
for (const file of updatableFiles) {
|
||||
console.log(`Updating file ${file.name} checksum...`);
|
||||
await Self.rawSql(`
|
||||
UPDATE edi.fileConfig
|
||||
SET checksum = ?
|
||||
WHERE name = ?`,
|
||||
[file.checksum, file.name], options);
|
||||
}
|
||||
|
||||
await tx.commit();
|
||||
await tx.commit();
|
||||
|
||||
// Clean files
|
||||
try {
|
||||
console.debug(`Cleaning files...`);
|
||||
await fs.remove(tempPath);
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT')
|
||||
throw e;
|
||||
}
|
||||
// Clean files
|
||||
try {
|
||||
console.debug(`Cleaning files...`);
|
||||
await fs.remove(tempPath);
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT')
|
||||
throw e;
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
await tx.rollback();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
return true;
|
||||
} catch (error) {
|
||||
await tx.rollback();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
let ftpClient;
|
||||
async function getFtpClient() {
|
||||
if (!ftpClient) {
|
||||
const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig');
|
||||
console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`);
|
||||
let ftpClient;
|
||||
async function getFtpClient() {
|
||||
if (!ftpClient) {
|
||||
const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig');
|
||||
console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`);
|
||||
|
||||
const FtpClient = require('ftps');
|
||||
const FtpClient = require('ftps');
|
||||
|
||||
ftpClient = new FtpClient({
|
||||
host: ftpConfig.host,
|
||||
username: ftpConfig.user,
|
||||
password: ftpConfig.password,
|
||||
procotol: 'ftp'
|
||||
});
|
||||
}
|
||||
ftpClient = new FtpClient({
|
||||
host: ftpConfig.host,
|
||||
username: ftpConfig.user,
|
||||
password: ftpConfig.password,
|
||||
procotol: 'ftp',
|
||||
additionalLftpCommands: 'set ssl:verify-certificate no'
|
||||
});
|
||||
}
|
||||
|
||||
return ftpClient;
|
||||
}
|
||||
return ftpClient;
|
||||
}
|
||||
|
||||
async function getChecksum(file) {
|
||||
const ftpClient = await getFtpClient();
|
||||
console.debug(`Checking checksum for file ${file.name}...`);
|
||||
async function getChecksum(file) {
|
||||
const ftpClient = await getFtpClient();
|
||||
console.debug(`Checking checksum for file ${file.name}...`);
|
||||
|
||||
ftpClient.cat(`codes/${file.name}.txt`);
|
||||
ftpClient.cat(`codes/${file.name}.TXT`);
|
||||
|
||||
const response = await new Promise((resolve, reject) => {
|
||||
ftpClient.exec((err, response) => {
|
||||
if (err || response.error) {
|
||||
console.debug(`Error downloading checksum file... ${response.error}`);
|
||||
return reject(err);
|
||||
}
|
||||
const response = await new Promise((resolve, reject) => {
|
||||
ftpClient.exec((err, response) => {
|
||||
if (err || response.error) {
|
||||
console.debug(`Error downloading checksum file... ${response.error}`);
|
||||
return reject(err);
|
||||
}
|
||||
|
||||
resolve(response);
|
||||
});
|
||||
});
|
||||
resolve(response);
|
||||
});
|
||||
});
|
||||
|
||||
if (response && response.data) {
|
||||
const fileContents = response.data;
|
||||
const rows = fileContents.split('\n');
|
||||
const row = rows[4];
|
||||
const columns = row.split(/\s+/);
|
||||
if (response && response.data) {
|
||||
const fileContents = response.data;
|
||||
const rows = fileContents.split('\n');
|
||||
const row = rows[4];
|
||||
const columns = row.split(/\s+/);
|
||||
|
||||
let fileChecksum;
|
||||
if (file.keyValue)
|
||||
fileChecksum = columns[1];
|
||||
let fileChecksum;
|
||||
if (file.keyValue)
|
||||
fileChecksum = columns[1];
|
||||
|
||||
if (!file.keyValue)
|
||||
fileChecksum = columns[0];
|
||||
if (!file.keyValue)
|
||||
fileChecksum = columns[0];
|
||||
|
||||
return fileChecksum;
|
||||
}
|
||||
}
|
||||
return fileChecksum;
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadFile(remoteFile, tempFile) {
|
||||
const ftpClient = await getFtpClient();
|
||||
async function downloadFile(remoteFile, tempFile) {
|
||||
const ftpClient = await getFtpClient();
|
||||
|
||||
ftpClient.get(remoteFile, tempFile);
|
||||
ftpClient.get(remoteFile, tempFile);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
ftpClient.exec((err, response) => {
|
||||
if (err || response.error) {
|
||||
console.debug(`Error downloading file... ${response.error}`);
|
||||
return reject(err);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
ftpClient.exec((err, response) => {
|
||||
if (err || response.error) {
|
||||
console.debug(`Error downloading file... ${response.error}`);
|
||||
return reject(err);
|
||||
}
|
||||
|
||||
resolve(response);
|
||||
});
|
||||
});
|
||||
}
|
||||
resolve(response);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function extractFile(fileName, tempFile, tempDir) {
|
||||
const JSZip = require('jszip');
|
||||
async function extractFile(fileName, tempFile, tempDir) {
|
||||
const JSZip = require('jszip');
|
||||
|
||||
try {
|
||||
await fs.mkdir(tempDir);
|
||||
console.debug(`Extracting file ${fileName}...`);
|
||||
} catch (error) {
|
||||
if (error.code !== 'EEXIST')
|
||||
throw e;
|
||||
}
|
||||
try {
|
||||
await fs.mkdir(tempDir);
|
||||
console.debug(`Extracting file ${fileName}...`);
|
||||
} catch (error) {
|
||||
if (error.code !== 'EEXIST')
|
||||
throw e;
|
||||
}
|
||||
|
||||
const fileStream = await fs.readFile(tempFile);
|
||||
if (fileStream) {
|
||||
const zip = new JSZip();
|
||||
const zipContents = await zip.loadAsync(fileStream);
|
||||
const fileStream = await fs.readFile(tempFile);
|
||||
if (fileStream) {
|
||||
const zip = new JSZip();
|
||||
const zipContents = await zip.loadAsync(fileStream);
|
||||
|
||||
if (!zipContents) return;
|
||||
if (!zipContents) return;
|
||||
|
||||
const fileNames = Object.keys(zipContents.files);
|
||||
const fileNames = Object.keys(zipContents.files);
|
||||
|
||||
for (const fileName of fileNames) {
|
||||
const fileContent = await zip.file(fileName).async('nodebuffer');
|
||||
const dest = path.join(tempDir, fileName);
|
||||
await fs.writeFile(dest, fileContent);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const fileName of fileNames) {
|
||||
const fileContent = await zip.file(fileName).async('nodebuffer');
|
||||
const dest = path.join(tempDir, fileName);
|
||||
await fs.writeFile(dest, fileContent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function dumpData(tempDir, table, options) {
|
||||
const toTable = table.toTable;
|
||||
const baseName = table.fileName;
|
||||
async function dumpData(tempDir, table, options) {
|
||||
const toTable = table.toTable;
|
||||
const baseName = table.fileName;
|
||||
|
||||
console.log(`Emptying table ${toTable}...`);
|
||||
const tableName = `edi.${toTable}`;
|
||||
await Self.rawSql(`DELETE FROM ??`, [tableName]);
|
||||
console.log(`Emptying table ${toTable}...`);
|
||||
const tableName = `edi.${toTable}`;
|
||||
await Self.rawSql(`DELETE FROM ??`, [tableName]);
|
||||
|
||||
const dirFiles = await fs.readdir(tempDir);
|
||||
const files = dirFiles.filter(file => file.startsWith(baseName));
|
||||
const dirFiles = await fs.readdir(tempDir);
|
||||
const files = dirFiles.filter(file => file.startsWith(baseName));
|
||||
|
||||
for (const file of files) {
|
||||
console.log(`Dumping data from file ${file}...`);
|
||||
for (const file of files) {
|
||||
console.log(`Dumping data from file ${file}...`);
|
||||
|
||||
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
|
||||
const sqlTemplate = await fs.readFile(templatePath, 'utf8');
|
||||
const filePath = path.join(tempDir, file);
|
||||
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
|
||||
const sqlTemplate = await fs.readFile(templatePath, 'utf8');
|
||||
const filePath = path.join(tempDir, file);
|
||||
|
||||
await Self.rawSql(sqlTemplate, [filePath], options);
|
||||
await Self.rawSql(`
|
||||
UPDATE edi.tableConfig
|
||||
SET updated = ?
|
||||
WHERE fileName = ?
|
||||
`, [Date.vnNew(), baseName], options);
|
||||
}
|
||||
await Self.rawSql(sqlTemplate, [filePath], options);
|
||||
await Self.rawSql(`
|
||||
UPDATE edi.tableConfig
|
||||
SET updated = ?
|
||||
WHERE fileName = ?
|
||||
`, [Date.vnNew(), baseName], options);
|
||||
}
|
||||
|
||||
console.log(`Updated table ${toTable}\n`);
|
||||
}
|
||||
console.log(`Updated table ${toTable}\n`);
|
||||
}
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue