verdnatura/salix
verdnatura
/
salix
3
4
Fork 0
Code Issues Pull Requests 57 Releases Wiki Activity

Updated transaction
gitea/salix/pipeline/head This commit looks good Details

This commit is contained in:
Joan Sanchez 2022-09-08 09:21:05 +02:00
parent ab86ba0652
commit 151e1b52ed
1 changed files with 100 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
back/methods/edi/updateData.js
View File

@ -20,80 +20,89 @@ module.exports = Self => {
const models = Self.app.models; const models = Self.app.models;
// Get files checksum // Get files checksum
const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig'); const tx = await Self.beginTransaction({});
const updatableFiles = []; try {
for (const file of files) { const options = {transaction: tx};
const fileChecksum = await getChecksum(file); const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options);
if (file.checksum != fileChecksum) { const updatableFiles = [];
updatableFiles.push({ for (const file of files) {
name: file.name, const fileChecksum = await getChecksum(file);
checksum: fileChecksum
});
} else
console.debug(`File already updated, skipping...`);
}
if (updatableFiles.length === 0) if (file.checksum != fileChecksum) {
return false; updatableFiles.push({
name: file.name,
// Download files checksum: fileChecksum
const container = await models.TempContainer.container('edi'); });
const tempPath = path.join(container.client.root, container.name); } else
console.debug(`File already updated, skipping...`);
let remoteFile;
let tempDir;
let tempFile;
const fileNames = updatableFiles.map(file => file.name);
const tables = await Self.rawSql(`
SELECT fileName, toTable, file
FROM edi.tableConfig
WHERE file IN (?)`, [fileNames]);
for (const table of tables) {
const fileName = table.file;
remoteFile = `codes/${fileName}.ZIP`;
tempDir = `${tempPath}/${fileName}`;
tempFile = `${tempPath}/${fileName}.zip`;
try {
await fs.readFile(tempFile);
} catch (error) {
if (error.code === 'ENOENT') {
console.debug(`Downloading file ${fileName}...`);
const downloadOutput = await downloadFile(remoteFile, tempFile);
if (downloadOutput.error)
continue;
}
} }
await extractFile(fileName, tempFile, tempDir); if (updatableFiles.length === 0)
return false;
console.debug(`Updating table ${table.toTable}...`); // Download files
await dumpData(tempDir, table); const container = await models.TempContainer.container('edi');
} const tempPath = path.join(container.client.root, container.name);
// Update files checksum let remoteFile;
for (const file of updatableFiles) { let tempDir;
console.log(`Updating file ${file.name} checksum...`); let tempFile;
await Self.rawSql(`
UPDATE edi.fileConfig
SET checksum = ?
WHERE name = ?`,
[file.checksum, file.name]);
}
// Clean files const fileNames = updatableFiles.map(file => file.name);
try {
console.debug(`Cleaning files...`); const tables = await Self.rawSql(`
await fs.remove(tempPath); SELECT fileName, toTable, file
FROM edi.tableConfig
WHERE file IN (?)`, [fileNames], options);
for (const table of tables) {
const fileName = table.file;
remoteFile = `codes/${fileName}.ZIP`;
tempDir = `${tempPath}/${fileName}`;
tempFile = `${tempPath}/${fileName}.zip`;
try {
await fs.readFile(tempFile);
} catch (error) {
if (error.code === 'ENOENT') {
console.debug(`Downloading file ${fileName}...`);
const downloadOutput = await downloadFile(remoteFile, tempFile);
if (downloadOutput.error)
continue;
}
}
await extractFile(fileName, tempFile, tempDir);
console.debug(`Updating table ${table.toTable}...`);
await dumpData(tempDir, table, options);
}
// Update files checksum
for (const file of updatableFiles) {
console.log(`Updating file ${file.name} checksum...`);
await Self.rawSql(`
UPDATE edi.fileConfig
SET checksum = ?
WHERE name = ?`,
[file.checksum, file.name], options);
}
// Clean files
try {
console.debug(`Cleaning files...`);
await fs.remove(tempPath);
} catch (error) {
if (error.code !== 'ENOENT')
throw e;
}
} catch (error) { } catch (error) {
if (error.code !== 'ENOENT') await tx.rollback();
throw e; console.log(error);
throw error;
} }
return true; return true;
@ -197,48 +206,48 @@ module.exports = Self => {
} }
} }
async function dumpData(tempDir, table) { async function dumpData(tempDir, table, options) {
const toTable = table.toTable; const toTable = table.toTable;
const baseName = table.fileName; const baseName = table.fileName;
console.log(`Starting transaction...`); console.log(`Starting transaction...`);
// const tx = await Self.beginTransaction({}); // const tx = await Self.beginTransaction({});
console.log(`Started transaction`); console.log(`Started transaction`);
try { // try {
// const options = {transaction: tx}; // const options = {transaction: tx};
console.log(`Emptying table ${toTable}...`); console.log(`Emptying table ${toTable}...`);
const tableName = `edi.${toTable}`; const tableName = `edi.${toTable}`;
await Self.rawSql(`DELETE FROM ??`, [tableName]); await Self.rawSql(`DELETE FROM ??`, [tableName]);
console.log(`Reading files...`); console.log(`Reading files...`);
const dirFiles = await fs.readdir(tempDir); const dirFiles = await fs.readdir(tempDir);
const files = dirFiles.filter(file => file.startsWith(baseName)); const files = dirFiles.filter(file => file.startsWith(baseName));
console.log('Files to import: ' + files.join(', ')); console.log('Files to import: ' + files.join(', '));
for (const file of files) { for (const file of files) {
console.log(`Dumping data from file ${file}...`); console.log(`Dumping data from file ${file}...`);
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
const sqlTemplate = await fs.readFile(templatePath, 'utf8'); const sqlTemplate = await fs.readFile(templatePath, 'utf8');
const filePath = path.join(tempDir, file); const filePath = path.join(tempDir, file);
await Self.rawSql(sqlTemplate, [filePath]); await Self.rawSql(sqlTemplate, [filePath], options);
await Self.rawSql(` await Self.rawSql(`
UPDATE edi.tableConfig UPDATE edi.tableConfig
SET updated = ? SET updated = ?
WHERE fileName = ? WHERE fileName = ?
`, [new Date(), baseName]); `, [new Date(), baseName], options);
}
// await tx.commit();
console.log(`Closed transaction`);
} catch (error) {
// await tx.rollback();
console.log(error);
throw error;
} }
// await tx.commit();
// console.log(`Closed transaction`);
// } catch (error) {
// await tx.rollback();
// console.log(error);
// throw error;
// }
console.log(`Updated table ${toTable}\n`); console.log(`Updated table ${toTable}\n`);
} }
}; };