verdnatura/salix
verdnatura
/
salix
3
4
Fork 0
Code Issues Pull Requests 67 Releases Wiki Activity

Merge branch 'test' of https://gitea.verdnatura.es/verdnatura/salix
gitea/salix/pipeline/head This commit looks good Details

This commit is contained in:
Joan Sanchez 2022-09-08 11:21:52 +02:00
parent 44b63af0a7 4251b0baeb
commit da92f771ac
1 changed files with 95 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

213
back/methods/edi/updateData.js
View File

@ -20,80 +20,90 @@ module.exports = Self => {
const models = Self.app.models;
// Get files checksum
const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig');
const tx = await Self.beginTransaction({});
const updatableFiles = [];
for (const file of files) {
const fileChecksum = await getChecksum(file);
try {
const options = {transaction: tx};
const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options);
if (file.checksum != fileChecksum) {
updatableFiles.push({
name: file.name,
checksum: fileChecksum
});
} else
console.debug(`File already updated, skipping...`);
}
const updatableFiles = [];
for (const file of files) {
const fileChecksum = await getChecksum(file);
if (updatableFiles.length === 0)
return false;
// Download files
const container = await models.TempContainer.container('edi');
const tempPath = path.join(container.client.root, container.name);
let remoteFile;
let tempDir;
let tempFile;
const fileNames = updatableFiles.map(file => file.name);
const tables = await Self.rawSql(`
SELECT fileName, toTable, file
FROM edi.tableConfig
WHERE file IN (?)`, [fileNames]);
for (const table of tables) {
const fileName = table.file;
console.debug(`Downloading file ${fileName}...`);
remoteFile = `codes/${fileName}.ZIP`;
tempDir = `${tempPath}/${fileName}`;
tempFile = `${tempPath}/${fileName}.zip`;
try {
await fs.readFile(tempFile);
} catch (error) {
if (error.code === 'ENOENT') {
const downloadOutput = await downloadFile(remoteFile, tempFile);
if (downloadOutput.error)
continue;
}
if (file.checksum != fileChecksum) {
updatableFiles.push({
name: file.name,
checksum: fileChecksum
});
} else
console.debug(`File already updated, skipping...`);
}
console.debug(`Extracting file ${fileName}...`);
await extractFile(tempFile, tempDir);
if (updatableFiles.length === 0)
return false;
console.debug(`Updating table ${table.toTable}...`);
await dumpData(tempDir, table);
}
// Download files
const container = await models.TempContainer.container('edi');
const tempPath = path.join(container.client.root, container.name);
// Update files checksum
for (const file of updatableFiles) {
await Self.rawSql(`
UPDATE edi.fileConfig
SET checksum = ?
WHERE name = ?`,
[file.checksum, file.name]);
}
let remoteFile;
let tempDir;
let tempFile;
// Clean files
try {
await fs.remove(tempPath);
const fileNames = updatableFiles.map(file => file.name);
const tables = await Self.rawSql(`
SELECT fileName, toTable, file
FROM edi.tableConfig
WHERE file IN (?)`, [fileNames], options);
for (const table of tables) {
const fileName = table.file;
remoteFile = `codes/${fileName}.ZIP`;
tempDir = `${tempPath}/${fileName}`;
tempFile = `${tempPath}/${fileName}.zip`;
try {
await fs.readFile(tempFile);
} catch (error) {
if (error.code === 'ENOENT') {
console.debug(`Downloading file ${fileName}...`);
const downloadOutput = await downloadFile(remoteFile, tempFile);
if (downloadOutput.error)
continue;
}
}
await extractFile(fileName, tempFile, tempDir);
console.debug(`Updating table ${table.toTable}...`);
await dumpData(tempDir, table, options);
}
// Update files checksum
for (const file of updatableFiles) {
console.log(`Updating file ${file.name} checksum...`);
await Self.rawSql(`
UPDATE edi.fileConfig
SET checksum = ?
WHERE name = ?`,
[file.checksum, file.name], options);
}
await tx.commit();
// Clean files
try {
console.debug(`Cleaning files...`);
await fs.remove(tempPath);
} catch (error) {
if (error.code !== 'ENOENT')
throw e;
}
} catch (error) {
if (error.code !== 'ENOENT')
throw e;
await tx.rollback();
throw error;
}
return true;
@ -126,9 +136,9 @@ module.exports = Self => {
const response = await new Promise((resolve, reject) => {
ftpClient.exec((err, response) => {
if (response.error) {
if (err || response.error) {
console.debug(`Error downloading checksum file... ${response.error}`);
reject(err);
return reject(err);
}
resolve(response);
@ -159,9 +169,9 @@ module.exports = Self => {
return new Promise((resolve, reject) => {
ftpClient.exec((err, response) => {
if (response.error) {
if (err || response.error) {
console.debug(`Error downloading file... ${response.error}`);
reject(err);
return reject(err);
}
resolve(response);
@ -169,11 +179,12 @@ module.exports = Self => {
});
}
async function extractFile(tempFile, tempDir) {
async function extractFile(fileName, tempFile, tempDir) {
const JSZip = require('jszip');
try {
await fs.mkdir(tempDir);
console.debug(`Extracting file ${fileName}...`);
} catch (error) {
if (error.code !== 'EEXIST')
throw e;
@ -196,66 +207,32 @@ module.exports = Self => {
}
}
async function dumpData(tempDir, table) {
async function dumpData(tempDir, table, options) {
const toTable = table.toTable;
const baseName = table.fileName;
const firstEntry = entries[0];
const entryName = firstEntry.entryName;
const startIndex = (entryName.length - 10);
const endIndex = (entryName.length - 4);
const dateString = entryName.substring(startIndex, endIndex);
console.log(`Emptying table ${toTable}...`);
const tableName = `edi.${toTable}`;
await Self.rawSql(`DELETE FROM ??`, [tableName]);
const lastUpdated = new Date();
const dirFiles = await fs.readdir(tempDir);
const files = dirFiles.filter(file => file.startsWith(baseName));
let updated = null;
if (file.updated) {
updated = new Date(file.updated);
updated.setHours(0, 0, 0, 0);
}
for (const file of files) {
console.log(`Dumping data from file ${file}...`);
// Format string date to a date object
lastUpdated.setFullYear(`20${dateString.substring(4, 6)}`);
lastUpdated.setMonth(parseInt(dateString.substring(2, 4)) - 1);
lastUpdated.setDate(dateString.substring(0, 2));
lastUpdated.setHours(0, 0, 0, 0);
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
const sqlTemplate = await fs.readFile(templatePath, 'utf8');
const filePath = path.join(tempDir, file);
if (updated && lastUpdated <= updated) {
console.debug(`Table ${toTable} already updated, skipping...`);
return;
}
const tx = await Self.beginTransaction({});
try {
const options = {transaction: tx};
const tableName = `edi.${toTable}`;
await Self.rawSql(`DELETE FROM ??`, [tableName], options);
const dirFiles = await fs.readdir(tempDir);
const files = dirFiles.filter(file => file.startsWith(baseName));
for (const file of files) {
console.log(`Dumping data from file ${file}...`);
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
const sqlTemplate = await fs.readFile(templatePath, 'utf8');
const filePath = path.join(tempDir, file);
await Self.rawSql(sqlTemplate, [filePath], options);
await Self.rawSql(`
await Self.rawSql(sqlTemplate, [filePath], options);
await Self.rawSql(`
UPDATE edi.tableConfig
SET updated = ?
WHERE fileName = ?
`, [new Date(), baseName], options);
}
tx.commit();
} catch (error) {
tx.rollback();
throw error;
}
console.log(`Updated table ${toTable}\n`);
}
};