diff --git a/back/methods/edi/syncData.js b/back/methods/edi/syncData.js index 7f601df573..090a6d8180 100644 --- a/back/methods/edi/syncData.js +++ b/back/methods/edi/syncData.js @@ -1,10 +1,12 @@ const UserError = require('vn-loopback/util/user-error'); -const fs = require("fs"); +const fs = require('fs-extra'); const fastCsv = require("fast-csv"); const axios = require('axios'); +const path = require('path'); +const { pipeline } = require('stream/promises'); module.exports = Self => { - Self.remoteMethod('syncData', { + Self.remoteMethodCtx('syncData', { description: 'Sync schema data from external provider', accessType: 'WRITE', returns: { @@ -17,9 +19,9 @@ module.exports = Self => { } }); - Self.syncData = async () => { + Self.syncData = async ctx => { const models = Self.app.models; - const tx = await Self.beginTransaction({}); + let tx; try { const tables = await models.TableMultiConfig.find(); if (!tables?.length) throw new UserError(`No tables to sync`); @@ -32,25 +34,22 @@ module.exports = Self => { const data = await getData(floricodeConfig.url, table.method, token); if (!data) continue; - await Self.rawSql(`TRUNCATE edi.??`, [table.toTable]); + tx = await Self.beginTransaction({}); + const options = {transaction: tx, userId: ctx.req.accessToken.userId}; - const ws = fs.createWriteStream("data.csv"); + await Self.rawSql(`DELETE FROM edi.??`, [table.toTable]); - // Falta implementar la escritura de los datos en el archivo CSV - await fastCsv - .write(JSON.parse(data[0]), { headers: true, delimiter: ";" }) - .pipe(ws); - - const templatePath = path.join(__dirname, `./sql/${table.toTable}.sql`); + const ws = fs.createWriteStream(path.join(__dirname, `/${table.toTable}.csv`)); + await pipeline(fastCsv.write(data, { delimiter: ';' }), ws); + const templatePath = path.join(__dirname, `./syncSql/${table.toTable}.sql`); const sqlTemplate = await fs.readFile(templatePath, 'utf8'); - const filePath = path.join(tempDir, file); - await Self.rawSql(sqlTemplate, [filePath], options); - await fs.remove(ws); - await table.updateAttribute('udpated', Date.vnNew()); + await Self.rawSql(sqlTemplate, [ws.path], options); + await fs.remove(ws.path); + await table.updateAttribute('updated', Date.vnNew(), options); + await tx.commit(); } - await tx.commit(); } catch (e) { - await tx.rollback(); + if (tx) await tx.rollback(); throw e; } }; @@ -72,13 +71,13 @@ module.exports = Self => { let data = []; let count = 0; const maxCount = await getCount(url, method, token); - while (count < maxCount) { + while (count < maxCount) { const request = await axios.get(`${url}/v2/${method}?$skip=${count}`, { headers: { 'Authorization': `Bearer ${token}` } }); - data.push(request.data.value); + data.push(...request.data.value); count += request.data.value.length; } return data; diff --git a/back/methods/edi/syncSql/bucket.sql b/back/methods/edi/syncSql/bucket.sql new file mode 100644 index 0000000000..0effaaf98a --- /dev/null +++ b/back/methods/edi/syncSql/bucket.sql @@ -0,0 +1,14 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`bucket` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11) + SET bucket_id = @col3, + bucket_type_id = @col5, + description = @col6, + x_size = @col7, + y_size = @col8, + z_size = @col9, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col11, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/bucket_type.sql b/back/methods/edi/syncSql/bucket_type.sql new file mode 100644 index 0000000000..570f26ae29 --- /dev/null +++ b/back/methods/edi/syncSql/bucket_type.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`bucket_type` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5) + SET bucket_type_id = @col3, + description = @col4, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col5, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/feature.sql b/back/methods/edi/syncSql/feature.sql new file mode 100644 index 0000000000..3cfb98a096 --- /dev/null +++ b/back/methods/edi/syncSql/feature.sql @@ -0,0 +1,11 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`feature` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6) + SET item_id = @col3, + feature_type_id = @col4, + feature_value = @col5, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/genus.sql b/back/methods/edi/syncSql/genus.sql new file mode 100644 index 0000000000..9e4fddb4d5 --- /dev/null +++ b/back/methods/edi/syncSql/genus.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`genus` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5) + SET genus_id = @col3, + latin_genus_name = @col4, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col5, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/item.sql b/back/methods/edi/syncSql/item.sql new file mode 100644 index 0000000000..8f605d2ffa --- /dev/null +++ b/back/methods/edi/syncSql/item.sql @@ -0,0 +1,14 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`item` + CHARACTER SET ascii + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11) + SET id = @col3, + product_name = @col5, + name = @col6, + plant_id = @col8, + group_id = @col10, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col11, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/item_feature.sql b/back/methods/edi/syncSql/item_feature.sql new file mode 100644 index 0000000000..fd4926ee5f --- /dev/null +++ b/back/methods/edi/syncSql/item_feature.sql @@ -0,0 +1,12 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`item_feature` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7) + SET item_id = @col3, + feature = @col4, + regulation_type = @col5, + presentation_order = @col6, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col7, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/item_group.sql b/back/methods/edi/syncSql/item_group.sql new file mode 100644 index 0000000000..a330969d7d --- /dev/null +++ b/back/methods/edi/syncSql/item_group.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`item_group` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5) + SET group_code = @col3, + dutch_group_description = @col4, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col5, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/plant.sql b/back/methods/edi/syncSql/plant.sql new file mode 100644 index 0000000000..9ece3d245e --- /dev/null +++ b/back/methods/edi/syncSql/plant.sql @@ -0,0 +1,11 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`plant` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8) + SET plant_id = @col4, + genus_id = @col5, + specie_id = @col6, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col8, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/specie.sql b/back/methods/edi/syncSql/specie.sql new file mode 100644 index 0000000000..9fce2a810d --- /dev/null +++ b/back/methods/edi/syncSql/specie.sql @@ -0,0 +1,11 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`specie` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6) + SET specie_id = @col3, + genus_id = @col4, + latin_species_name = @col5, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/supplier.sql b/back/methods/edi/syncSql/supplier.sql new file mode 100644 index 0000000000..0a83d360eb --- /dev/null +++ b/back/methods/edi/syncSql/supplier.sql @@ -0,0 +1,11 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`supplier` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12, @col13, @col14, @col15, @col16, @col17, @col18, @col19, @col20, @col21) + SET GLNAddressCode = @col3, + supplier_id = @col9, + company_name = @col4, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col17, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/type.sql b/back/methods/edi/syncSql/type.sql new file mode 100644 index 0000000000..eec5b0dcc3 --- /dev/null +++ b/back/methods/edi/syncSql/type.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`type` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6) + SET type_id = @col3, + type_group_id = @col4, + description = @col5, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/value.sql b/back/methods/edi/syncSql/value.sql new file mode 100644 index 0000000000..ea49eb3e48 --- /dev/null +++ b/back/methods/edi/syncSql/value.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`value` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6) + SET type_id = @col3, + type_value = @col4, + type_description = @col5, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/db/versions/11442-blackBirch/01-firstScript.sql b/db/versions/11442-blackBirch/01-firstScript.sql index e1bb974383..821b513066 100644 --- a/db/versions/11442-blackBirch/01-firstScript.sql +++ b/db/versions/11442-blackBirch/01-firstScript.sql @@ -32,7 +32,7 @@ UPDATE edi.tableMultiConfig AND fileName='FB'; UPDATE edi.tableMultiConfig - SET id = 3, method = 'VBN/RegulatoryFeatureType' + SET id = 3, method = 'VBN/ProductFeature' WHERE id IS NULL AND fileName='FF'; @@ -47,7 +47,7 @@ UPDATE edi.tableMultiConfig AND fileName='FP'; UPDATE edi.tableMultiConfig - SET id = 6, method = 'VBN/ProductFeature' + SET id = 6, method = 'VBN/RegulatoryFeatureType' WHERE id IS NULL AND fileName='FY'; @@ -72,7 +72,7 @@ UPDATE edi.tableMultiConfig AND fileName='CC'; UPDATE edi.tableMultiConfig - SET id = 11, method = 'FEC/FeatureType' + SET id = 11, method = 'VBN/FeatureType' WHERE id IS NULL AND fileName='FE';