From f3eb36f8ee756ed343a2b866795d43d943ceba26 Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 11:47:43 +0200 Subject: [PATCH 01/11] Edi fixes --- back/methods/edi/updateData.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index df77906da..2b7fbaa31 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -135,7 +135,8 @@ module.exports = Self => { const tx = await Self.beginTransaction({}); const options = {transaction: tx}; - await Self.rawSql(`DELETE FROM edi.${toTable}`, null, options); + const tableName = `edi.${toTable}`; + await Self.rawSql(`DELETE FROM ??`, [tableName], options); await Self.rawSql(sqlTemplate, [rawPath], options); await Self.rawSql(` UPDATE edi.fileConfig From b98ac4f233399a400ea9ee71b57202490bc88036 Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 12:18:06 +0200 Subject: [PATCH 02/11] Debugging logs --- back/methods/edi/updateData.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index 2b7fbaa31..fda4d8443 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -81,6 +81,8 @@ module.exports = Self => { } const AdmZip = require('adm-zip'); + console.log(paths.remoteFile); + console.log(paths.tempFile); const zip = new AdmZip(paths.tempFile); const entries = zip.getEntries(); @@ -131,8 +133,8 @@ module.exports = Self => { const rawPath = path.join(paths.tempDir, entryName); + const tx = await Self.beginTransaction({}); try { - const tx = await Self.beginTransaction({}); const options = {transaction: tx}; const tableName = `edi.${toTable}`; From b1ce77d3621f11ee77df7836745a5d3d3462e81c Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 12:35:45 +0200 Subject: [PATCH 03/11] Debug logs --- back/methods/edi/updateData.js | 1 + 1 file changed, 1 insertion(+) diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index fda4d8443..cef60adf9 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -83,6 +83,7 @@ module.exports = Self => { const AdmZip = require('adm-zip'); console.log(paths.remoteFile); console.log(paths.tempFile); + console.log('Exists temp file: ', fs.existsSync(paths.tempFile)) const zip = new AdmZip(paths.tempFile); const entries = zip.getEntries(); From d0e7ef8f47bf1a214f1c118b720f6c092f9170a0 Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 12:44:39 +0200 Subject: [PATCH 04/11] Fixed SQL templates --- back/methods/edi/sql/bucket.sql | 2 +- back/methods/edi/sql/bucket_type.sql | 2 +- back/methods/edi/sql/feature.sql | 2 +- back/methods/edi/sql/genus.sql | 2 +- back/methods/edi/sql/item.sql | 2 +- back/methods/edi/sql/item_feature.sql | 2 +- back/methods/edi/sql/item_group.sql | 2 +- back/methods/edi/sql/plant.sql | 2 +- back/methods/edi/sql/specie.sql | 2 +- back/methods/edi/sql/supplier.sql | 2 +- back/methods/edi/sql/type.sql | 2 +- back/methods/edi/sql/value.sql | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/back/methods/edi/sql/bucket.sql b/back/methods/edi/sql/bucket.sql index 92121386c..33dca6796 100644 --- a/back/methods/edi/sql/bucket.sql +++ b/back/methods/edi/sql/bucket.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE bucket + INTO TABLE `edi`.`bucket` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12) SET diff --git a/back/methods/edi/sql/bucket_type.sql b/back/methods/edi/sql/bucket_type.sql index 68498f183..f748fac84 100644 --- a/back/methods/edi/sql/bucket_type.sql +++ b/back/methods/edi/sql/bucket_type.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE bucket_type + INTO TABLE `edi`.`bucket_type` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6) SET diff --git a/back/methods/edi/sql/feature.sql b/back/methods/edi/sql/feature.sql index 081cfde7b..7486a5bd3 100644 --- a/back/methods/edi/sql/feature.sql +++ b/back/methods/edi/sql/feature.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE `feature` + INTO TABLE `edi`.`feature` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7) SET diff --git a/back/methods/edi/sql/genus.sql b/back/methods/edi/sql/genus.sql index dd882629c..52b85f020 100644 --- a/back/methods/edi/sql/genus.sql +++ b/back/methods/edi/sql/genus.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE genus + INTO TABLE `edi`.`genus` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6) SET diff --git a/back/methods/edi/sql/item.sql b/back/methods/edi/sql/item.sql index 543c83381..8d794c7cf 100644 --- a/back/methods/edi/sql/item.sql +++ b/back/methods/edi/sql/item.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE item + INTO TABLE `edi`.`item` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12) SET diff --git a/back/methods/edi/sql/item_feature.sql b/back/methods/edi/sql/item_feature.sql index 23b0f3afa..f16da6ed0 100644 --- a/back/methods/edi/sql/item_feature.sql +++ b/back/methods/edi/sql/item_feature.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE `item_feature` + INTO TABLE `edi`.`item_feature` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8) SET diff --git a/back/methods/edi/sql/item_group.sql b/back/methods/edi/sql/item_group.sql index 31da6c57f..1a57d7727 100644 --- a/back/methods/edi/sql/item_group.sql +++ b/back/methods/edi/sql/item_group.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE item_group + INTO TABLE `edi`.`item_group` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6) SET diff --git a/back/methods/edi/sql/plant.sql b/back/methods/edi/sql/plant.sql index 50720fba5..8d6f90b78 100644 --- a/back/methods/edi/sql/plant.sql +++ b/back/methods/edi/sql/plant.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE plant + INTO TABLE `edi`.`plant` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9) SET diff --git a/back/methods/edi/sql/specie.sql b/back/methods/edi/sql/specie.sql index e27478ae8..956ec2b69 100644 --- a/back/methods/edi/sql/specie.sql +++ b/back/methods/edi/sql/specie.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE specie + INTO TABLE `edi`.`specie` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7) SET diff --git a/back/methods/edi/sql/supplier.sql b/back/methods/edi/sql/supplier.sql index a1d3376d6..b57e4409f 100644 --- a/back/methods/edi/sql/supplier.sql +++ b/back/methods/edi/sql/supplier.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE edi.supplier + INTO TABLE `edi`.`.supplier` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12, @col13, @col14, @col15, @col16, @col17, @col18, @col19, @col20) SET diff --git a/back/methods/edi/sql/type.sql b/back/methods/edi/sql/type.sql index 88c4ac026..1ff60fb0b 100644 --- a/back/methods/edi/sql/type.sql +++ b/back/methods/edi/sql/type.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE `type` + INTO TABLE `edi`.`type` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7) SET diff --git a/back/methods/edi/sql/value.sql b/back/methods/edi/sql/value.sql index c8c4deef5..2a469f201 100644 --- a/back/methods/edi/sql/value.sql +++ b/back/methods/edi/sql/value.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE `value` + INTO TABLE `edi`.`value` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7) SET From 9bb264da38a48fac4a1a4e1996926c1f9c974361 Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 12:52:41 +0200 Subject: [PATCH 05/11] Fixes --- back/methods/edi/updateData.js | 80 +++++++++++++++++----------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index cef60adf9..b3b52e060 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -83,7 +83,7 @@ module.exports = Self => { const AdmZip = require('adm-zip'); console.log(paths.remoteFile); console.log(paths.tempFile); - console.log('Exists temp file: ', fs.existsSync(paths.tempFile)) + console.log('Exists temp file: ', fs.existsSync(paths.tempFile)); const zip = new AdmZip(paths.tempFile); const entries = zip.getEntries(); @@ -102,58 +102,58 @@ module.exports = Self => { const toTable = file.toTable; const baseName = file.fileName; - for (const zipEntry of entries) { - const entryName = zipEntry.entryName; - console.log(`Reading file ${entryName}...`); + const tableName = `edi.${toTable}`; + await Self.rawSql(`DELETE FROM ??`, [tableName], options); - const startIndex = (entryName.length - 10); - const endIndex = (entryName.length - 4); - const dateString = entryName.substring(startIndex, endIndex); - const lastUpdated = new Date(); + const tx = await Self.beginTransaction({}); + try { + const options = {transaction: tx}; - // Format string date to a date object - let updated = null; - if (file.updated) { - updated = new Date(file.updated); - updated.setHours(0, 0, 0, 0); - } + for (const zipEntry of entries) { + const entryName = zipEntry.entryName; + console.log(`Reading file ${entryName}...`); - lastUpdated.setFullYear(`20${dateString.substring(4, 6)}`); - lastUpdated.setMonth(parseInt(dateString.substring(2, 4)) - 1); - lastUpdated.setDate(dateString.substring(0, 2)); - lastUpdated.setHours(0, 0, 0, 0); + const startIndex = (entryName.length - 10); + const endIndex = (entryName.length - 4); + const dateString = entryName.substring(startIndex, endIndex); + const lastUpdated = new Date(); - if (updated && lastUpdated <= updated) { - console.debug(`Table ${toTable} already updated, skipping...`); - continue; - } + // Format string date to a date object + let updated = null; + if (file.updated) { + updated = new Date(file.updated); + updated.setHours(0, 0, 0, 0); + } - console.log('Dumping data...'); - const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); - const sqlTemplate = fs.readFileSync(templatePath, 'utf8'); + lastUpdated.setFullYear(`20${dateString.substring(4, 6)}`); + lastUpdated.setMonth(parseInt(dateString.substring(2, 4)) - 1); + lastUpdated.setDate(dateString.substring(0, 2)); + lastUpdated.setHours(0, 0, 0, 0); - const rawPath = path.join(paths.tempDir, entryName); + if (updated && lastUpdated <= updated) { + console.debug(`Table ${toTable} already updated, skipping...`); + continue; + } - const tx = await Self.beginTransaction({}); - try { - const options = {transaction: tx}; + console.log('Dumping data...'); + const templatePath = path.join(__dirname, `./sql/${toTable}.sql`); + const sqlTemplate = fs.readFileSync(templatePath, 'utf8'); + + const rawPath = path.join(paths.tempDir, entryName); - const tableName = `edi.${toTable}`; - await Self.rawSql(`DELETE FROM ??`, [tableName], options); await Self.rawSql(sqlTemplate, [rawPath], options); await Self.rawSql(` - UPDATE edi.fileConfig - SET updated = ? - WHERE fileName = ? - `, [lastUpdated, baseName], options); + UPDATE edi.fileConfig + SET updated = ? + WHERE fileName = ? + `, [lastUpdated, baseName], options); tx.commit(); - } catch (error) { - tx.rollback(); - throw error; } - - console.log(`Updated table ${toTable}\n`); + } catch (error) { + tx.rollback(); + throw error; } + console.log(`Updated table ${toTable}\n`); } }; From b790ee88d941f1c9b77dc8e980e795c12a45d209 Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 12:57:39 +0200 Subject: [PATCH 06/11] Edi fixes --- back/methods/edi/updateData.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index b3b52e060..b40a4f2c5 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -102,13 +102,14 @@ module.exports = Self => { const toTable = file.toTable; const baseName = file.fileName; - const tableName = `edi.${toTable}`; - await Self.rawSql(`DELETE FROM ??`, [tableName], options); - const tx = await Self.beginTransaction({}); + try { const options = {transaction: tx}; + const tableName = `edi.${toTable}`; + await Self.rawSql(`DELETE FROM ??`, [tableName], options); + for (const zipEntry of entries) { const entryName = zipEntry.entryName; console.log(`Reading file ${entryName}...`); From 33049c507ff84fb6693d6bfba19545aad1939e98 Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 13:01:39 +0200 Subject: [PATCH 07/11] Transaction fix --- back/methods/edi/updateData.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index b40a4f2c5..3d1b65e5f 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -148,9 +148,9 @@ module.exports = Self => { SET updated = ? WHERE fileName = ? `, [lastUpdated, baseName], options); - - tx.commit(); } + + tx.commit(); } catch (error) { tx.rollback(); throw error; From 3752ca234719953a1b34fc47710b357fe1a5a9b3 Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 13:22:32 +0200 Subject: [PATCH 08/11] Changed when file is removed --- back/methods/edi/updateData.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index 3d1b65e5f..fadde0032 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -48,6 +48,12 @@ module.exports = Self => { tempDir = `${tempPath}/${fileName}`; tempFile = `${tempPath}/${fileName}.zip`; + if (fs.existsSync(tempFile)) + await fs.unlink(tempFile); + + if (fs.existsSync(tempDir)) + await fs.rmdir(tempDir, {recursive: true}); + await extractFile({ ftpClient: ftpClient, file: file, @@ -61,7 +67,6 @@ module.exports = Self => { if (fs.existsSync(tempFile)) await fs.unlink(tempFile); - await fs.rmdir(tempDir, {recursive: true}); console.error(error); } } @@ -89,9 +94,6 @@ module.exports = Self => { zip.extractAllTo(paths.tempDir, false); - if (fs.existsSync(paths.tempFile)) - await fs.unlink(paths.tempFile); - await dumpData({file, entries, paths}); await fs.rmdir(paths.tempDir, {recursive: true}); From 391960b3905bb992f06521fbd4cef39edd27cb17 Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 13:27:58 +0200 Subject: [PATCH 09/11] Fixed sql table name --- back/methods/edi/sql/supplier.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/back/methods/edi/sql/supplier.sql b/back/methods/edi/sql/supplier.sql index b57e4409f..aa2ddc4d8 100644 --- a/back/methods/edi/sql/supplier.sql +++ b/back/methods/edi/sql/supplier.sql @@ -1,5 +1,5 @@ LOAD DATA LOCAL INFILE ? - INTO TABLE `edi`.`.supplier` + INTO TABLE `edi`.`supplier` FIELDS TERMINATED BY ';' LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12, @col13, @col14, @col15, @col16, @col17, @col18, @col19, @col20) SET From bdb581db9418941bd322266421e3ed07f31a908b Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 13:47:40 +0200 Subject: [PATCH 10/11] Disable file delete --- back/methods/edi/updateData.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index fadde0032..0693ac93b 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -19,6 +19,8 @@ module.exports = Self => { Self.updateData = async() => { const models = Self.app.models; + await models.TempContainer.destroyContainer('edi'); + const container = await models.TempContainer.container('edi'); const tempPath = path.join(container.client.root, container.name); @@ -48,11 +50,11 @@ module.exports = Self => { tempDir = `${tempPath}/${fileName}`; tempFile = `${tempPath}/${fileName}.zip`; - if (fs.existsSync(tempFile)) - await fs.unlink(tempFile); + // if (fs.existsSync(tempFile)) + // await fs.unlink(tempFile); - if (fs.existsSync(tempDir)) - await fs.rmdir(tempDir, {recursive: true}); + // if (fs.existsSync(tempDir)) + // await fs.rmdir(tempDir, {recursive: true}); await extractFile({ ftpClient: ftpClient, @@ -86,9 +88,6 @@ module.exports = Self => { } const AdmZip = require('adm-zip'); - console.log(paths.remoteFile); - console.log(paths.tempFile); - console.log('Exists temp file: ', fs.existsSync(paths.tempFile)); const zip = new AdmZip(paths.tempFile); const entries = zip.getEntries(); From 5059f72ea5bfc8e146036caadb780145ac3a2550 Mon Sep 17 00:00:00 2001 From: joan Date: Fri, 3 Jun 2022 13:54:58 +0200 Subject: [PATCH 11/11] Clean directory --- back/methods/edi/updateData.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/back/methods/edi/updateData.js b/back/methods/edi/updateData.js index 0693ac93b..33ad76e6d 100644 --- a/back/methods/edi/updateData.js +++ b/back/methods/edi/updateData.js @@ -19,11 +19,12 @@ module.exports = Self => { Self.updateData = async() => { const models = Self.app.models; - await models.TempContainer.destroyContainer('edi'); - const container = await models.TempContainer.container('edi'); const tempPath = path.join(container.client.root, container.name); + // Temporary file clean + await fs.rmdir(`${tempPath}/*`, {recursive: true}); + const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig'); console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`);