Updated transaction
gitea/salix/pipeline/head This commit looks good
Details
gitea/salix/pipeline/head This commit looks good
Details
This commit is contained in:
parent
ab86ba0652
commit
151e1b52ed
|
|
@ -20,80 +20,89 @@ module.exports = Self => {
|
|||
const models = Self.app.models;
|
||||
|
||||
// Get files checksum
|
||||
const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig');
|
||||
const tx = await Self.beginTransaction({});
|
||||
|
||||
const updatableFiles = [];
|
||||
for (const file of files) {
|
||||
const fileChecksum = await getChecksum(file);
|
||||
try {
|
||||
const options = {transaction: tx};
|
||||
const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options);
|
||||
|
||||
if (file.checksum != fileChecksum) {
|
||||
updatableFiles.push({
|
||||
name: file.name,
|
||||
checksum: fileChecksum
|
||||
});
|
||||
} else
|
||||
console.debug(`File already updated, skipping...`);
|
||||
}
|
||||
const updatableFiles = [];
|
||||
for (const file of files) {
|
||||
const fileChecksum = await getChecksum(file);
|
||||
|
||||
if (updatableFiles.length === 0)
|
||||
return false;
|
||||
|
||||
// Download files
|
||||
const container = await models.TempContainer.container('edi');
|
||||
const tempPath = path.join(container.client.root, container.name);
|
||||
|
||||
let remoteFile;
|
||||
let tempDir;
|
||||
let tempFile;
|
||||
|
||||
const fileNames = updatableFiles.map(file => file.name);
|
||||
|
||||
const tables = await Self.rawSql(`
|
||||
SELECT fileName, toTable, file
|
||||
FROM edi.tableConfig
|
||||
WHERE file IN (?)`, [fileNames]);
|
||||
|
||||
for (const table of tables) {
|
||||
const fileName = table.file;
|
||||
|
||||
remoteFile = `codes/${fileName}.ZIP`;
|
||||
tempDir = `${tempPath}/${fileName}`;
|
||||
tempFile = `${tempPath}/${fileName}.zip`;
|
||||
|
||||
try {
|
||||
await fs.readFile(tempFile);
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
console.debug(`Downloading file ${fileName}...`);
|
||||
const downloadOutput = await downloadFile(remoteFile, tempFile);
|
||||
if (downloadOutput.error)
|
||||
continue;
|
||||
}
|
||||
if (file.checksum != fileChecksum) {
|
||||
updatableFiles.push({
|
||||
name: file.name,
|
||||
checksum: fileChecksum
|
||||
});
|
||||
} else
|
||||
console.debug(`File already updated, skipping...`);
|
||||
}
|
||||
|
||||
await extractFile(fileName, tempFile, tempDir);
|
||||
if (updatableFiles.length === 0)
|
||||
return false;
|
||||
|
||||
console.debug(`Updating table ${table.toTable}...`);
|
||||
await dumpData(tempDir, table);
|
||||
}
|
||||
// Download files
|
||||
const container = await models.TempContainer.container('edi');
|
||||
const tempPath = path.join(container.client.root, container.name);
|
||||
|
||||
// Update files checksum
|
||||
for (const file of updatableFiles) {
|
||||
console.log(`Updating file ${file.name} checksum...`);
|
||||
await Self.rawSql(`
|
||||
UPDATE edi.fileConfig
|
||||
SET checksum = ?
|
||||
WHERE name = ?`,
|
||||
[file.checksum, file.name]);
|
||||
}
|
||||
let remoteFile;
|
||||
let tempDir;
|
||||
let tempFile;
|
||||
|
||||
// Clean files
|
||||
try {
|
||||
console.debug(`Cleaning files...`);
|
||||
await fs.remove(tempPath);
|
||||
const fileNames = updatableFiles.map(file => file.name);
|
||||
|
||||
const tables = await Self.rawSql(`
|
||||
SELECT fileName, toTable, file
|
||||
FROM edi.tableConfig
|
||||
WHERE file IN (?)`, [fileNames], options);
|
||||
|
||||
for (const table of tables) {
|
||||
const fileName = table.file;
|
||||
|
||||
remoteFile = `codes/${fileName}.ZIP`;
|
||||
tempDir = `${tempPath}/${fileName}`;
|
||||
tempFile = `${tempPath}/${fileName}.zip`;
|
||||
|
||||
try {
|
||||
await fs.readFile(tempFile);
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
console.debug(`Downloading file ${fileName}...`);
|
||||
const downloadOutput = await downloadFile(remoteFile, tempFile);
|
||||
if (downloadOutput.error)
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
await extractFile(fileName, tempFile, tempDir);
|
||||
|
||||
console.debug(`Updating table ${table.toTable}...`);
|
||||
await dumpData(tempDir, table, options);
|
||||
}
|
||||
|
||||
// Update files checksum
|
||||
for (const file of updatableFiles) {
|
||||
console.log(`Updating file ${file.name} checksum...`);
|
||||
await Self.rawSql(`
|
||||
UPDATE edi.fileConfig
|
||||
SET checksum = ?
|
||||
WHERE name = ?`,
|
||||
[file.checksum, file.name], options);
|
||||
}
|
||||
|
||||
// Clean files
|
||||
try {
|
||||
console.debug(`Cleaning files...`);
|
||||
await fs.remove(tempPath);
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT')
|
||||
throw e;
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT')
|
||||
throw e;
|
||||
await tx.rollback();
|
||||
console.log(error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
return true;
|
||||
|
|
@ -197,48 +206,48 @@ module.exports = Self => {
|
|||
}
|
||||
}
|
||||
|
||||
async function dumpData(tempDir, table) {
|
||||
async function dumpData(tempDir, table, options) {
|
||||
const toTable = table.toTable;
|
||||
const baseName = table.fileName;
|
||||
|
||||
console.log(`Starting transaction...`);
|
||||
// const tx = await Self.beginTransaction({});
|
||||
console.log(`Started transaction`);
|
||||
try {
|
||||
// const options = {transaction: tx};
|
||||
// try {
|
||||
// const options = {transaction: tx};
|
||||
|
||||
console.log(`Emptying table ${toTable}...`);
|
||||
const tableName = `edi.${toTable}`;
|
||||
await Self.rawSql(`DELETE FROM ??`, [tableName]);
|
||||
console.log(`Emptying table ${toTable}...`);
|
||||
const tableName = `edi.${toTable}`;
|
||||
await Self.rawSql(`DELETE FROM ??`, [tableName]);
|
||||
|
||||
console.log(`Reading files...`);
|
||||
const dirFiles = await fs.readdir(tempDir);
|
||||
const files = dirFiles.filter(file => file.startsWith(baseName));
|
||||
console.log(`Reading files...`);
|
||||
const dirFiles = await fs.readdir(tempDir);
|
||||
const files = dirFiles.filter(file => file.startsWith(baseName));
|
||||
|
||||
console.log('Files to import: ' + files.join(', '));
|
||||
console.log('Files to import: ' + files.join(', '));
|
||||
|
||||
for (const file of files) {
|
||||
console.log(`Dumping data from file ${file}...`);
|
||||
for (const file of files) {
|
||||
console.log(`Dumping data from file ${file}...`);
|
||||
|
||||
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
|
||||
const sqlTemplate = await fs.readFile(templatePath, 'utf8');
|
||||
const filePath = path.join(tempDir, file);
|
||||
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
|
||||
const sqlTemplate = await fs.readFile(templatePath, 'utf8');
|
||||
const filePath = path.join(tempDir, file);
|
||||
|
||||
await Self.rawSql(sqlTemplate, [filePath]);
|
||||
await Self.rawSql(`
|
||||
await Self.rawSql(sqlTemplate, [filePath], options);
|
||||
await Self.rawSql(`
|
||||
UPDATE edi.tableConfig
|
||||
SET updated = ?
|
||||
WHERE fileName = ?
|
||||
`, [new Date(), baseName]);
|
||||
}
|
||||
|
||||
// await tx.commit();
|
||||
console.log(`Closed transaction`);
|
||||
} catch (error) {
|
||||
// await tx.rollback();
|
||||
console.log(error);
|
||||
throw error;
|
||||
`, [new Date(), baseName], options);
|
||||
}
|
||||
|
||||
// await tx.commit();
|
||||
// console.log(`Closed transaction`);
|
||||
// } catch (error) {
|
||||
// await tx.rollback();
|
||||
// console.log(error);
|
||||
// throw error;
|
||||
// }
|
||||
console.log(`Updated table ${toTable}\n`);
|
||||
}
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue