verdnatura/salix
verdnatura
/
salix
3
4
Fork 0
Code Issues Pull Requests 67 Releases Wiki Activity

Merge branch 'test' of https://gitea.verdnatura.es/verdnatura/salix
gitea/salix/pipeline/head This commit looks good Details

This commit is contained in:
Joan Sanchez 2022-06-03 14:00:05 +02:00
parent 77f3af7679 5059f72ea5
commit 96f500d161
13 changed files with 62 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
back/methods/edi/sql/bucket.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE bucket
INTO TABLE `edi`.`bucket`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12)
SET

2
back/methods/edi/sql/bucket_type.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE bucket_type
INTO TABLE `edi`.`bucket_type`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6)
SET

2
back/methods/edi/sql/feature.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `feature`
INTO TABLE `edi`.`feature`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7)
SET

2
back/methods/edi/sql/genus.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE genus
INTO TABLE `edi`.`genus`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6)
SET

2
back/methods/edi/sql/item.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE item
INTO TABLE `edi`.`item`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12)
SET

2
back/methods/edi/sql/item_feature.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `item_feature`
INTO TABLE `edi`.`item_feature`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8)
SET

2
back/methods/edi/sql/item_group.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE item_group
INTO TABLE `edi`.`item_group`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6)
SET

2
back/methods/edi/sql/plant.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE plant
INTO TABLE `edi`.`plant`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9)
SET

2
back/methods/edi/sql/specie.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE specie
INTO TABLE `edi`.`specie`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7)
SET

2
back/methods/edi/sql/supplier.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE edi.supplier
INTO TABLE `edi`.`supplier`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12, @col13, @col14, @col15, @col16, @col17, @col18, @col19, @col20)
SET

2
back/methods/edi/sql/type.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `type`
INTO TABLE `edi`.`type`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7)
SET

2
back/methods/edi/sql/value.sql
View File

@ -1,5 +1,5 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `value`
INTO TABLE `edi`.`value`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7)
SET

93
back/methods/edi/updateData.js
View File

@ -22,6 +22,9 @@ module.exports = Self => {
const container = await models.TempContainer.container('edi');
const tempPath = path.join(container.client.root, container.name);
// Temporary file clean
await fs.rmdir(`${tempPath}/*`, {recursive: true});
const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig');
console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`);
@ -48,6 +51,12 @@ module.exports = Self => {
tempDir = `${tempPath}/${fileName}`;
tempFile = `${tempPath}/${fileName}.zip`;
// if (fs.existsSync(tempFile))
// await fs.unlink(tempFile);
// if (fs.existsSync(tempDir))
// await fs.rmdir(tempDir, {recursive: true});
await extractFile({
ftpClient: ftpClient,
file: file,
@ -61,7 +70,6 @@ module.exports = Self => {
if (fs.existsSync(tempFile))
await fs.unlink(tempFile);
await fs.rmdir(tempDir, {recursive: true});
console.error(error);
}
}
@ -86,9 +94,6 @@ module.exports = Self => {
zip.extractAllTo(paths.tempDir, false);
if (fs.existsSync(paths.tempFile))
await fs.unlink(paths.tempFile);
await dumpData({file, entries, paths});
await fs.rmdir(paths.tempDir, {recursive: true});
@ -99,57 +104,59 @@ module.exports = Self => {
const toTable = file.toTable;
const baseName = file.fileName;
for (const zipEntry of entries) {
const entryName = zipEntry.entryName;
console.log(`Reading file ${entryName}...`);
const tx = await Self.beginTransaction({});
const startIndex = (entryName.length - 10);
const endIndex = (entryName.length - 4);
const dateString = entryName.substring(startIndex, endIndex);
const lastUpdated = new Date();
try {
const options = {transaction: tx};
// Format string date to a date object
let updated = null;
if (file.updated) {
updated = new Date(file.updated);
updated.setHours(0, 0, 0, 0);
}
const tableName = `edi.${toTable}`;
await Self.rawSql(`DELETE FROM ??`, [tableName], options);
lastUpdated.setFullYear(`20${dateString.substring(4, 6)}`);
lastUpdated.setMonth(parseInt(dateString.substring(2, 4)) - 1);
lastUpdated.setDate(dateString.substring(0, 2));
lastUpdated.setHours(0, 0, 0, 0);
for (const zipEntry of entries) {
const entryName = zipEntry.entryName;
console.log(`Reading file ${entryName}...`);
if (updated && lastUpdated <= updated) {
console.debug(`Table ${toTable} already updated, skipping...`);
continue;
}
const startIndex = (entryName.length - 10);
const endIndex = (entryName.length - 4);
const dateString = entryName.substring(startIndex, endIndex);
const lastUpdated = new Date();
console.log('Dumping data...');
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
const sqlTemplate = fs.readFileSync(templatePath, 'utf8');
// Format string date to a date object
let updated = null;
if (file.updated) {
updated = new Date(file.updated);
updated.setHours(0, 0, 0, 0);
}
const rawPath = path.join(paths.tempDir, entryName);
lastUpdated.setFullYear(`20${dateString.substring(4, 6)}`);
lastUpdated.setMonth(parseInt(dateString.substring(2, 4)) - 1);
lastUpdated.setDate(dateString.substring(0, 2));
lastUpdated.setHours(0, 0, 0, 0);
try {
const tx = await Self.beginTransaction({});
const options = {transaction: tx};
if (updated && lastUpdated <= updated) {
console.debug(`Table ${toTable} already updated, skipping...`);
continue;
}
console.log('Dumping data...');
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
const sqlTemplate = fs.readFileSync(templatePath, 'utf8');
const rawPath = path.join(paths.tempDir, entryName);
await Self.rawSql(`DELETE FROM edi.${toTable}`, null, options);
await Self.rawSql(sqlTemplate, [rawPath], options);
await Self.rawSql(`
UPDATE edi.fileConfig
SET updated = ?
WHERE fileName = ?
`, [lastUpdated, baseName], options);
tx.commit();
} catch (error) {
tx.rollback();
throw error;
UPDATE edi.fileConfig
SET updated = ?
WHERE fileName = ?
`, [lastUpdated, baseName], options);
}
console.log(`Updated table ${toTable}\n`);
tx.commit();
} catch (error) {
tx.rollback();
throw error;
}
console.log(`Updated table ${toTable}\n`);
}
};