From 1b0da71c9d64861b6e226a6b3ae70183c0fb1460 Mon Sep 17 00:00:00 2001 From: guillermo Date: Mon, 17 Feb 2025 15:03:23 +0100 Subject: [PATCH 01/10] feat: refs #8244 Floricode api --- back/methods/edi/syncData.js | 95 +++++++++++++++++++ back/model-config.json | 6 ++ back/models/edi.js | 1 + back/models/floricode-config.json | 28 ++++++ back/models/table-multi-config.json | 24 +++++ .../11442-blackBirch/00-firstScript.sql | 11 +++ .../11442-blackBirch/01-firstScript.sql | 87 +++++++++++++++++ package.json | 1 + pnpm-lock.yaml | 59 +++++++++++- 9 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 back/methods/edi/syncData.js create mode 100644 back/models/floricode-config.json create mode 100644 back/models/table-multi-config.json create mode 100644 db/versions/11442-blackBirch/00-firstScript.sql create mode 100644 db/versions/11442-blackBirch/01-firstScript.sql diff --git a/back/methods/edi/syncData.js b/back/methods/edi/syncData.js new file mode 100644 index 0000000000..7f601df573 --- /dev/null +++ b/back/methods/edi/syncData.js @@ -0,0 +1,95 @@ +const UserError = require('vn-loopback/util/user-error'); +const fs = require("fs"); +const fastCsv = require("fast-csv"); +const axios = require('axios'); + +module.exports = Self => { + Self.remoteMethod('syncData', { + description: 'Sync schema data from external provider', + accessType: 'WRITE', + returns: { + type: 'object', + root: true + }, + http: { + path: `/syncData`, + verb: 'POST' + } + }); + + Self.syncData = async () => { + const models = Self.app.models; + const tx = await Self.beginTransaction({}); + try { + const tables = await models.TableMultiConfig.find(); + if (!tables?.length) throw new UserError(`No tables to sync`); + + const floricodeConfig = await models.FloricodeConfig.findOne(); + if (!floricodeConfig) throw new UserError(`Floricode service is not configured`); + + const token = await getToken(floricodeConfig); + for (const table of tables) { + const data = await getData(floricodeConfig.url, table.method, token); + if (!data) continue; + + await Self.rawSql(`TRUNCATE edi.??`, [table.toTable]); + + const ws = fs.createWriteStream("data.csv"); + + // Falta implementar la escritura de los datos en el archivo CSV + await fastCsv + .write(JSON.parse(data[0]), { headers: true, delimiter: ";" }) + .pipe(ws); + + const templatePath = path.join(__dirname, `./sql/${table.toTable}.sql`); + const sqlTemplate = await fs.readFile(templatePath, 'utf8'); + const filePath = path.join(tempDir, file); + await Self.rawSql(sqlTemplate, [filePath], options); + await fs.remove(ws); + await table.updateAttribute('udpated', Date.vnNew()); + } + await tx.commit(); + } catch (e) { + await tx.rollback(); + throw e; + } + }; + + async function getToken(floricodeConfig) { + const response = await axios.post(`${floricodeConfig.url}/oauth/token`, { + grant_type: 'client_credentials', + client_id: floricodeConfig.user, + client_secret: floricodeConfig.password + }, { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded' + } + }); + return response.data.access_token; + } + + async function getData(url, method, token) { + let data = []; + let count = 0; + const maxCount = await getCount(url, method, token); + while (count < maxCount) { + const request = await axios.get(`${url}/v2/${method}?$skip=${count}`, { + headers: { + 'Authorization': `Bearer ${token}` + } + }); + data.push(request.data.value); + count += request.data.value.length; + } + return data; + } + + async function getCount(url, method, token) { + const request = await axios.get(`${url}/v2/${method}?$count=true`, { + headers: { + 'Authorization': `Bearer ${token}` + } + }); + return request.data["@odata.count"]; + } +}; diff --git a/back/model-config.json b/back/model-config.json index 2ced867f7a..ee98809b62 100644 --- a/back/model-config.json +++ b/back/model-config.json @@ -70,6 +70,9 @@ "Expedition_PrintOut": { "dataSource": "vn" }, + "FloricodeConfig": { + "dataSource": "vn" + }, "Image": { "dataSource": "vn" }, @@ -154,6 +157,9 @@ "SaySimpleConfig": { "dataSource": "vn" }, + "TableMultiConfig": { + "dataSource": "vn" + }, "TempContainer": { "dataSource": "tempStorage" }, diff --git a/back/models/edi.js b/back/models/edi.js index bfddf2746e..b639938ab1 100644 --- a/back/models/edi.js +++ b/back/models/edi.js @@ -1,3 +1,4 @@ module.exports = Self => { require('../methods/edi/updateData')(Self); + require('../methods/edi/syncData')(Self); }; diff --git a/back/models/floricode-config.json b/back/models/floricode-config.json new file mode 100644 index 0000000000..9e6976d2b2 --- /dev/null +++ b/back/models/floricode-config.json @@ -0,0 +1,28 @@ +{ + "name": "FloricodeConfig", + "base": "VnModel", + "options": { + "mysql": { + "table": "edi.floricodeConfig" + } + }, + "properties": { + "id": { + "type": "number", + "id": true, + "required": true + }, + "url": { + "type": "string", + "required": true + }, + "user": { + "type": "string", + "required": false + }, + "password": { + "type": "string", + "required": false + } + } +} diff --git a/back/models/table-multi-config.json b/back/models/table-multi-config.json new file mode 100644 index 0000000000..b973ba7a54 --- /dev/null +++ b/back/models/table-multi-config.json @@ -0,0 +1,24 @@ +{ + "name": "TableMultiConfig", + "base": "VnModel", + "options": { + "mysql": { + "table": "edi.tableMultiConfig" + } + }, + "properties": { + "id": { + "type": "number", + "required": true + }, + "toTable": { + "type": "string" + }, + "method": { + "type": "string" + }, + "updated": { + "type": "date" + } + } +} diff --git a/db/versions/11442-blackBirch/00-firstScript.sql b/db/versions/11442-blackBirch/00-firstScript.sql new file mode 100644 index 0000000000..2379b13535 --- /dev/null +++ b/db/versions/11442-blackBirch/00-firstScript.sql @@ -0,0 +1,11 @@ +CREATE TABLE `edi`.`floricodeConfig` ( + `id` int(10) unsigned NOT NULL, + `url` varchar(100) NOT NULL, + `user` varchar(50) NOT NULL, + `password` varchar(100) NOT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `floricodeConfig_check` CHECK (`id` = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; + +INSERT INTO salix.ACL (model,property,accessType,permission,principalType,principalId) + VALUES ('Edi','syncData','WRITE','ALLOW','ROLE','system'); diff --git a/db/versions/11442-blackBirch/01-firstScript.sql b/db/versions/11442-blackBirch/01-firstScript.sql new file mode 100644 index 0000000000..e1bb974383 --- /dev/null +++ b/db/versions/11442-blackBirch/01-firstScript.sql @@ -0,0 +1,87 @@ +-- Temporal data to dev +INSERT INTO salix.ACL (model,property,accessType,permission,principalType,principalId) + VALUES ('Edi','syncData','WRITE','ALLOW','ROLE','employee'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('CC', 'supplier', 'FEC010104', '2025-02-15'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('CK', 'bucket', 'VBN020101', '2025-02-14'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FB', 'bucket_type', 'VBN020101', '2025-02-14'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FE', 'type', 'florecompc2', '2025-02-15'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FF', 'feature', 'florecompc2', '2025-02-15'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FG', 'genus', 'florecompc2', '2025-02-15'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FO', 'item_group', 'florecompc2', '2025-02-15'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FP', 'item', 'florecompc2', '2025-02-15'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FS', 'specie', 'florecompc2', '2025-02-15'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FT', 'plant', 'florecompc2', '2025-02-15'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FV', 'value', 'florecompc2', '2025-02-15'); +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FY', 'item_feature', 'florecompc2', '2025-02-15'); +-- --------- + +ALTER TABLE edi.tableMultiConfig + ADD `method` varchar(100) DEFAULT NULL NULL AFTER file, + DROP PRIMARY KEY, + DROP KEY to_table, + ADD id int(10) unsigned NULL FIRST; + +UPDATE edi.tableMultiConfig + SET id = 1, method = 'VBN/Packaging' + WHERE id IS NULL + AND fileName='CK'; + +UPDATE edi.tableMultiConfig + SET id = 2, method = 'VBN/PackagingType' + WHERE id IS NULL + AND fileName='FB'; + +UPDATE edi.tableMultiConfig + SET id = 3, method = 'VBN/RegulatoryFeatureType' + WHERE id IS NULL + AND fileName='FF'; + +UPDATE edi.tableMultiConfig + SET id = 4, method = 'VBN/Genus' + WHERE id IS NULL + AND fileName='FG'; + +UPDATE edi.tableMultiConfig + SET id = 5, method = 'VBN/Product' + WHERE id IS NULL + AND fileName='FP'; + +UPDATE edi.tableMultiConfig + SET id = 6, method = 'VBN/ProductFeature' + WHERE id IS NULL + AND fileName='FY'; + +UPDATE edi.tableMultiConfig + SET id = 7, method = 'VBN/ProductGroup' + WHERE id IS NULL + AND fileName='FO'; + +UPDATE edi.tableMultiConfig + SET id = 8, method = 'VBN/Plant' + WHERE id IS NULL + AND fileName='FT'; + +UPDATE edi.tableMultiConfig + SET id = 9, method = 'VBN/Species' + WHERE id IS NULL + AND fileName='FS'; + +UPDATE edi.tableMultiConfig + SET id = 10, method = 'FEC/Company' + WHERE id IS NULL + AND fileName='CC'; + +UPDATE edi.tableMultiConfig + SET id = 11, method = 'FEC/FeatureType' + WHERE id IS NULL + AND fileName='FE'; + +UPDATE edi.tableMultiConfig + SET id = 12, method = 'VBN/FeatureValue' + WHERE id IS NULL + AND fileName='FV'; + +ALTER TABLE edi.tableMultiConfig + ADD CONSTRAINT tableMultiConfig_pk PRIMARY KEY (id), + MODIFY COLUMN id int(10) unsigned auto_increment NOT NULL, + ADD CONSTRAINT tableMultiConfig_unique UNIQUE KEY (toTable); diff --git a/package.json b/package.json index 7f2025552a..6d3cdf5200 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "body-parser": "^1.19.2", "compression": "^1.7.3", "ejs": "2.3.1", + "fast-csv": "^5.0.2", "form-data": "^4.0.0", "fs-extra": "^5.0.0", "ftps": "^1.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 042b91fe02..caf4d9dd44 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ dependencies: ejs: specifier: 2.3.1 version: 2.3.1 + fast-csv: + specifier: ^5.0.2 + version: 5.0.2 form-data: specifier: ^4.0.0 version: 4.0.0 @@ -1708,6 +1711,27 @@ packages: - supports-color dev: true + /@fast-csv/format@5.0.2: + resolution: {integrity: sha512-fRYcWvI8vs0Zxa/8fXd/QlmQYWWkJqKZPAXM+vksnplb3owQFKTPPh9JqOtD0L3flQw/AZjjXdPkD7Kp/uHm8g==} + dependencies: + lodash.escaperegexp: 4.1.2 + lodash.isboolean: 3.0.3 + lodash.isequal: 4.5.0 + lodash.isfunction: 3.0.9 + lodash.isnil: 4.0.0 + dev: false + + /@fast-csv/parse@5.0.2: + resolution: {integrity: sha512-gMu1Btmm99TP+wc0tZnlH30E/F1Gw1Tah3oMDBHNPe9W8S68ixVHjt89Wg5lh7d9RuQMtwN+sGl5kxR891+fzw==} + dependencies: + lodash.escaperegexp: 4.1.2 + lodash.groupby: 4.6.0 + lodash.isfunction: 3.0.9 + lodash.isnil: 4.0.0 + lodash.isundefined: 3.0.1 + lodash.uniq: 4.5.0 + dev: false + /@fastify/busboy@2.1.0: resolution: {integrity: sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==} engines: {node: '>=14'} @@ -6114,6 +6138,14 @@ packages: time-stamp: 1.1.0 dev: true + /fast-csv@5.0.2: + resolution: {integrity: sha512-CnB2zYAzzeh5Ta0UhSf32NexLy2SsEsSMY+fMWPV40k1OgaLEbm9Hf5dms3z/9fASZHBjB6i834079gVeksEqQ==} + engines: {node: '>=10.0.0'} + dependencies: + '@fast-csv/format': 5.0.2 + '@fast-csv/parse': 5.0.2 + dev: false + /fast-deep-equal@2.0.1: resolution: {integrity: sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==} dev: false @@ -9383,9 +9415,12 @@ packages: lodash._root: 3.0.1 dev: true + /lodash.escaperegexp@4.1.2: + resolution: {integrity: sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==} + dev: false + /lodash.groupby@4.6.0: resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==} - dev: true /lodash.isarguments@3.1.0: resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} @@ -9395,10 +9430,31 @@ packages: resolution: {integrity: sha512-JwObCrNJuT0Nnbuecmqr5DgtuBppuCvGD9lxjFpAzwnVtdGoDQ1zig+5W8k5/6Gcn0gZ3936HDAlGd28i7sOGQ==} dev: true + /lodash.isboolean@3.0.3: + resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} + dev: false + + /lodash.isequal@4.5.0: + resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==} + deprecated: This package is deprecated. Use require('node:util').isDeepStrictEqual instead. + dev: false + + /lodash.isfunction@3.0.9: + resolution: {integrity: sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==} + dev: false + + /lodash.isnil@4.0.0: + resolution: {integrity: sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==} + dev: false + /lodash.isplainobject@4.0.6: resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} dev: true + /lodash.isundefined@3.0.1: + resolution: {integrity: sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==} + dev: false + /lodash.kebabcase@4.1.1: resolution: {integrity: sha512-N8XRTIMMqqDgSy4VLKPnJ/+hpGZN+PHQiJnSenYqPaVV/NCqEogTnAdZLQiGKhxX+JCs8waWq2t1XHWKOmlY8g==} dev: true @@ -9458,7 +9514,6 @@ packages: /lodash.uniq@4.5.0: resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==} - dev: true /lodash.upperfirst@4.3.1: resolution: {integrity: sha512-sReKOYJIJf74dhJONhU4e0/shzi1trVbSWDOhKYE5XV2O+H7Sb2Dihwuc7xWxVl+DgFPyTqIN3zMfT9cq5iWDg==} From bf1770dcc556230769c5b3eeb136f0025798488d Mon Sep 17 00:00:00 2001 From: guillermo Date: Tue, 18 Feb 2025 13:05:57 +0100 Subject: [PATCH 02/10] feat: refs #8244 Floricode api --- back/methods/edi/syncData.js | 39 +++++++++---------- back/methods/edi/syncSql/bucket.sql | 14 +++++++ back/methods/edi/syncSql/bucket_type.sql | 10 +++++ back/methods/edi/syncSql/feature.sql | 11 ++++++ back/methods/edi/syncSql/genus.sql | 10 +++++ back/methods/edi/syncSql/item.sql | 14 +++++++ back/methods/edi/syncSql/item_feature.sql | 12 ++++++ back/methods/edi/syncSql/item_group.sql | 10 +++++ back/methods/edi/syncSql/plant.sql | 11 ++++++ back/methods/edi/syncSql/specie.sql | 11 ++++++ back/methods/edi/syncSql/supplier.sql | 11 ++++++ back/methods/edi/syncSql/type.sql | 10 +++++ back/methods/edi/syncSql/value.sql | 10 +++++ .../11442-blackBirch/01-firstScript.sql | 6 +-- 14 files changed, 156 insertions(+), 23 deletions(-) create mode 100644 back/methods/edi/syncSql/bucket.sql create mode 100644 back/methods/edi/syncSql/bucket_type.sql create mode 100644 back/methods/edi/syncSql/feature.sql create mode 100644 back/methods/edi/syncSql/genus.sql create mode 100644 back/methods/edi/syncSql/item.sql create mode 100644 back/methods/edi/syncSql/item_feature.sql create mode 100644 back/methods/edi/syncSql/item_group.sql create mode 100644 back/methods/edi/syncSql/plant.sql create mode 100644 back/methods/edi/syncSql/specie.sql create mode 100644 back/methods/edi/syncSql/supplier.sql create mode 100644 back/methods/edi/syncSql/type.sql create mode 100644 back/methods/edi/syncSql/value.sql diff --git a/back/methods/edi/syncData.js b/back/methods/edi/syncData.js index 7f601df573..090a6d8180 100644 --- a/back/methods/edi/syncData.js +++ b/back/methods/edi/syncData.js @@ -1,10 +1,12 @@ const UserError = require('vn-loopback/util/user-error'); -const fs = require("fs"); +const fs = require('fs-extra'); const fastCsv = require("fast-csv"); const axios = require('axios'); +const path = require('path'); +const { pipeline } = require('stream/promises'); module.exports = Self => { - Self.remoteMethod('syncData', { + Self.remoteMethodCtx('syncData', { description: 'Sync schema data from external provider', accessType: 'WRITE', returns: { @@ -17,9 +19,9 @@ module.exports = Self => { } }); - Self.syncData = async () => { + Self.syncData = async ctx => { const models = Self.app.models; - const tx = await Self.beginTransaction({}); + let tx; try { const tables = await models.TableMultiConfig.find(); if (!tables?.length) throw new UserError(`No tables to sync`); @@ -32,25 +34,22 @@ module.exports = Self => { const data = await getData(floricodeConfig.url, table.method, token); if (!data) continue; - await Self.rawSql(`TRUNCATE edi.??`, [table.toTable]); + tx = await Self.beginTransaction({}); + const options = {transaction: tx, userId: ctx.req.accessToken.userId}; - const ws = fs.createWriteStream("data.csv"); + await Self.rawSql(`DELETE FROM edi.??`, [table.toTable]); - // Falta implementar la escritura de los datos en el archivo CSV - await fastCsv - .write(JSON.parse(data[0]), { headers: true, delimiter: ";" }) - .pipe(ws); - - const templatePath = path.join(__dirname, `./sql/${table.toTable}.sql`); + const ws = fs.createWriteStream(path.join(__dirname, `/${table.toTable}.csv`)); + await pipeline(fastCsv.write(data, { delimiter: ';' }), ws); + const templatePath = path.join(__dirname, `./syncSql/${table.toTable}.sql`); const sqlTemplate = await fs.readFile(templatePath, 'utf8'); - const filePath = path.join(tempDir, file); - await Self.rawSql(sqlTemplate, [filePath], options); - await fs.remove(ws); - await table.updateAttribute('udpated', Date.vnNew()); + await Self.rawSql(sqlTemplate, [ws.path], options); + await fs.remove(ws.path); + await table.updateAttribute('updated', Date.vnNew(), options); + await tx.commit(); } - await tx.commit(); } catch (e) { - await tx.rollback(); + if (tx) await tx.rollback(); throw e; } }; @@ -72,13 +71,13 @@ module.exports = Self => { let data = []; let count = 0; const maxCount = await getCount(url, method, token); - while (count < maxCount) { + while (count < maxCount) { const request = await axios.get(`${url}/v2/${method}?$skip=${count}`, { headers: { 'Authorization': `Bearer ${token}` } }); - data.push(request.data.value); + data.push(...request.data.value); count += request.data.value.length; } return data; diff --git a/back/methods/edi/syncSql/bucket.sql b/back/methods/edi/syncSql/bucket.sql new file mode 100644 index 0000000000..0effaaf98a --- /dev/null +++ b/back/methods/edi/syncSql/bucket.sql @@ -0,0 +1,14 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`bucket` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11) + SET bucket_id = @col3, + bucket_type_id = @col5, + description = @col6, + x_size = @col7, + y_size = @col8, + z_size = @col9, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col11, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/bucket_type.sql b/back/methods/edi/syncSql/bucket_type.sql new file mode 100644 index 0000000000..570f26ae29 --- /dev/null +++ b/back/methods/edi/syncSql/bucket_type.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`bucket_type` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5) + SET bucket_type_id = @col3, + description = @col4, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col5, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/feature.sql b/back/methods/edi/syncSql/feature.sql new file mode 100644 index 0000000000..3cfb98a096 --- /dev/null +++ b/back/methods/edi/syncSql/feature.sql @@ -0,0 +1,11 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`feature` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6) + SET item_id = @col3, + feature_type_id = @col4, + feature_value = @col5, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/genus.sql b/back/methods/edi/syncSql/genus.sql new file mode 100644 index 0000000000..9e4fddb4d5 --- /dev/null +++ b/back/methods/edi/syncSql/genus.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`genus` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5) + SET genus_id = @col3, + latin_genus_name = @col4, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col5, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/item.sql b/back/methods/edi/syncSql/item.sql new file mode 100644 index 0000000000..8f605d2ffa --- /dev/null +++ b/back/methods/edi/syncSql/item.sql @@ -0,0 +1,14 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`item` + CHARACTER SET ascii + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11) + SET id = @col3, + product_name = @col5, + name = @col6, + plant_id = @col8, + group_id = @col10, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col11, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/item_feature.sql b/back/methods/edi/syncSql/item_feature.sql new file mode 100644 index 0000000000..fd4926ee5f --- /dev/null +++ b/back/methods/edi/syncSql/item_feature.sql @@ -0,0 +1,12 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`item_feature` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7) + SET item_id = @col3, + feature = @col4, + regulation_type = @col5, + presentation_order = @col6, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col7, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/item_group.sql b/back/methods/edi/syncSql/item_group.sql new file mode 100644 index 0000000000..a330969d7d --- /dev/null +++ b/back/methods/edi/syncSql/item_group.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`item_group` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5) + SET group_code = @col3, + dutch_group_description = @col4, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col5, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/plant.sql b/back/methods/edi/syncSql/plant.sql new file mode 100644 index 0000000000..9ece3d245e --- /dev/null +++ b/back/methods/edi/syncSql/plant.sql @@ -0,0 +1,11 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`plant` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8) + SET plant_id = @col4, + genus_id = @col5, + specie_id = @col6, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col8, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/specie.sql b/back/methods/edi/syncSql/specie.sql new file mode 100644 index 0000000000..9fce2a810d --- /dev/null +++ b/back/methods/edi/syncSql/specie.sql @@ -0,0 +1,11 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`specie` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6) + SET specie_id = @col3, + genus_id = @col4, + latin_species_name = @col5, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/supplier.sql b/back/methods/edi/syncSql/supplier.sql new file mode 100644 index 0000000000..0a83d360eb --- /dev/null +++ b/back/methods/edi/syncSql/supplier.sql @@ -0,0 +1,11 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`supplier` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' + (@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12, @col13, @col14, @col15, @col16, @col17, @col18, @col19, @col20, @col21) + SET GLNAddressCode = @col3, + supplier_id = @col9, + company_name = @col4, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col17, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/type.sql b/back/methods/edi/syncSql/type.sql new file mode 100644 index 0000000000..eec5b0dcc3 --- /dev/null +++ b/back/methods/edi/syncSql/type.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`type` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6) + SET type_id = @col3, + type_group_id = @col4, + description = @col5, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/back/methods/edi/syncSql/value.sql b/back/methods/edi/syncSql/value.sql new file mode 100644 index 0000000000..ea49eb3e48 --- /dev/null +++ b/back/methods/edi/syncSql/value.sql @@ -0,0 +1,10 @@ +LOAD DATA LOCAL INFILE ? + INTO TABLE `edi`.`value` + FIELDS TERMINATED BY ';' + LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6) + SET type_id = @col3, + type_value = @col4, + type_description = @col5, + entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'), + expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'), + change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s') diff --git a/db/versions/11442-blackBirch/01-firstScript.sql b/db/versions/11442-blackBirch/01-firstScript.sql index e1bb974383..821b513066 100644 --- a/db/versions/11442-blackBirch/01-firstScript.sql +++ b/db/versions/11442-blackBirch/01-firstScript.sql @@ -32,7 +32,7 @@ UPDATE edi.tableMultiConfig AND fileName='FB'; UPDATE edi.tableMultiConfig - SET id = 3, method = 'VBN/RegulatoryFeatureType' + SET id = 3, method = 'VBN/ProductFeature' WHERE id IS NULL AND fileName='FF'; @@ -47,7 +47,7 @@ UPDATE edi.tableMultiConfig AND fileName='FP'; UPDATE edi.tableMultiConfig - SET id = 6, method = 'VBN/ProductFeature' + SET id = 6, method = 'VBN/RegulatoryFeatureType' WHERE id IS NULL AND fileName='FY'; @@ -72,7 +72,7 @@ UPDATE edi.tableMultiConfig AND fileName='CC'; UPDATE edi.tableMultiConfig - SET id = 11, method = 'FEC/FeatureType' + SET id = 11, method = 'VBN/FeatureType' WHERE id IS NULL AND fileName='FE'; From 96cba1f24fcabeff688affac9ddff86334d13cd6 Mon Sep 17 00:00:00 2001 From: guillermo Date: Tue, 18 Feb 2025 13:21:29 +0100 Subject: [PATCH 03/10] feat: refs #8244 Last changes --- .../11442-blackBirch/01-firstScript.sql | 22 ------------------- .../11442-blackBirch/02-firstScript.sql | 4 ++++ 2 files changed, 4 insertions(+), 22 deletions(-) create mode 100644 db/versions/11442-blackBirch/02-firstScript.sql diff --git a/db/versions/11442-blackBirch/01-firstScript.sql b/db/versions/11442-blackBirch/01-firstScript.sql index 821b513066..18aee4ec2f 100644 --- a/db/versions/11442-blackBirch/01-firstScript.sql +++ b/db/versions/11442-blackBirch/01-firstScript.sql @@ -1,20 +1,3 @@ --- Temporal data to dev -INSERT INTO salix.ACL (model,property,accessType,permission,principalType,principalId) - VALUES ('Edi','syncData','WRITE','ALLOW','ROLE','employee'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('CC', 'supplier', 'FEC010104', '2025-02-15'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('CK', 'bucket', 'VBN020101', '2025-02-14'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FB', 'bucket_type', 'VBN020101', '2025-02-14'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FE', 'type', 'florecompc2', '2025-02-15'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FF', 'feature', 'florecompc2', '2025-02-15'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FG', 'genus', 'florecompc2', '2025-02-15'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FO', 'item_group', 'florecompc2', '2025-02-15'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FP', 'item', 'florecompc2', '2025-02-15'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FS', 'specie', 'florecompc2', '2025-02-15'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FT', 'plant', 'florecompc2', '2025-02-15'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FV', 'value', 'florecompc2', '2025-02-15'); -INSERT INTO edi.tableMultiConfig (fileName, toTable, file, updated) VALUES('FY', 'item_feature', 'florecompc2', '2025-02-15'); --- --------- - ALTER TABLE edi.tableMultiConfig ADD `method` varchar(100) DEFAULT NULL NULL AFTER file, DROP PRIMARY KEY, @@ -80,8 +63,3 @@ UPDATE edi.tableMultiConfig SET id = 12, method = 'VBN/FeatureValue' WHERE id IS NULL AND fileName='FV'; - -ALTER TABLE edi.tableMultiConfig - ADD CONSTRAINT tableMultiConfig_pk PRIMARY KEY (id), - MODIFY COLUMN id int(10) unsigned auto_increment NOT NULL, - ADD CONSTRAINT tableMultiConfig_unique UNIQUE KEY (toTable); diff --git a/db/versions/11442-blackBirch/02-firstScript.sql b/db/versions/11442-blackBirch/02-firstScript.sql new file mode 100644 index 0000000000..609940cf1d --- /dev/null +++ b/db/versions/11442-blackBirch/02-firstScript.sql @@ -0,0 +1,4 @@ +ALTER TABLE edi.tableMultiConfig + ADD CONSTRAINT tableMultiConfig_pk PRIMARY KEY (id), + MODIFY COLUMN id int(10) unsigned auto_increment NOT NULL, + ADD CONSTRAINT tableMultiConfig_unique UNIQUE KEY (toTable); From cba381ce2fdbe6b23fb9c2281d7469c59ed4abe1 Mon Sep 17 00:00:00 2001 From: guillermo Date: Tue, 18 Feb 2025 13:52:27 +0100 Subject: [PATCH 04/10] feat: refs #8244 Refactor --- back/methods/edi/syncData.js | 51 ++++++++++++------------------------ 1 file changed, 17 insertions(+), 34 deletions(-) diff --git a/back/methods/edi/syncData.js b/back/methods/edi/syncData.js index 090a6d8180..1ba98f32fb 100644 --- a/back/methods/edi/syncData.js +++ b/back/methods/edi/syncData.js @@ -18,17 +18,17 @@ module.exports = Self => { verb: 'POST' } }); - + Self.syncData = async ctx => { const models = Self.app.models; let tx; try { - const tables = await models.TableMultiConfig.find(); - if (!tables?.length) throw new UserError(`No tables to sync`); - const floricodeConfig = await models.FloricodeConfig.findOne(); if (!floricodeConfig) throw new UserError(`Floricode service is not configured`); + const tables = await models.TableMultiConfig.find(); + if (!tables?.length) throw new UserError(`No tables to sync`); + const token = await getToken(floricodeConfig); for (const table of tables) { const data = await getData(floricodeConfig.url, table.method, token); @@ -38,13 +38,13 @@ module.exports = Self => { const options = {transaction: tx, userId: ctx.req.accessToken.userId}; await Self.rawSql(`DELETE FROM edi.??`, [table.toTable]); - const ws = fs.createWriteStream(path.join(__dirname, `/${table.toTable}.csv`)); await pipeline(fastCsv.write(data, { delimiter: ';' }), ws); const templatePath = path.join(__dirname, `./syncSql/${table.toTable}.sql`); const sqlTemplate = await fs.readFile(templatePath, 'utf8'); await Self.rawSql(sqlTemplate, [ws.path], options); - await fs.remove(ws.path); + + fs.remove(ws.path); await table.updateAttribute('updated', Date.vnNew(), options); await tx.commit(); } @@ -54,41 +54,24 @@ module.exports = Self => { } }; - async function getToken(floricodeConfig) { - const response = await axios.post(`${floricodeConfig.url}/oauth/token`, { - grant_type: 'client_credentials', - client_id: floricodeConfig.user, - client_secret: floricodeConfig.password - }, { - headers: { - 'Content-Type': 'application/x-www-form-urlencoded' - } - }); - return response.data.access_token; - } + const getToken = async ({ url, user, password }) => + (await axios.post(`${url}/oauth/token`, + { grant_type: 'client_credentials', client_id: user, client_secret: password }, + { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } } + )).data.access_token; async function getData(url, method, token) { let data = []; let count = 0; - const maxCount = await getCount(url, method, token); + const maxCount = (await get(`${url}/v2/${method}?$count=true`, token))["@odata.count"]; while (count < maxCount) { - const request = await axios.get(`${url}/v2/${method}?$skip=${count}`, { - headers: { - 'Authorization': `Bearer ${token}` - } - }); - data.push(...request.data.value); - count += request.data.value.length; + const response = await get(`${url}/v2/${method}?$skip=${count}`, token) + data.push(...response.value); + count += response.value.length; } return data; } - async function getCount(url, method, token) { - const request = await axios.get(`${url}/v2/${method}?$count=true`, { - headers: { - 'Authorization': `Bearer ${token}` - } - }); - return request.data["@odata.count"]; - } + const get = async (url, token) => + (await axios.get(url, { headers: { Authorization: `Bearer ${token}` } })).data; }; From 8185343d128a142229d53195f6d2a886b9120a9f Mon Sep 17 00:00:00 2001 From: guillermo Date: Wed, 19 Feb 2025 09:51:04 +0100 Subject: [PATCH 05/10] feat: refs #8244 Added tests --- back/methods/edi/specs/syncData.spec.js | 55 +++++++++++++++++++++++++ back/methods/edi/syncData.js | 39 +++++++++++------- db/dump/fixtures.before.sql | 3 ++ 3 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 back/methods/edi/specs/syncData.spec.js diff --git a/back/methods/edi/specs/syncData.spec.js b/back/methods/edi/specs/syncData.spec.js new file mode 100644 index 0000000000..81adc92b4a --- /dev/null +++ b/back/methods/edi/specs/syncData.spec.js @@ -0,0 +1,55 @@ +const models = require('vn-loopback/server/server').models; + +describe('edi syncData()', function() { + const ediModel = models.Edi; + + it('should be insert into the table', async() => { + const tx = await ediModel.beginTransaction({}); + const ctx = {req: {accessToken: {userId: 1}}}; + const options = {transaction: tx}; + let error; + try { + await models.FloricodeConfig.create({ + id: 1, + url: 'http://sample.com', + user: 'sample', + password: 'sample' + }, options); + + spyOn(ediModel, 'getToken').and.returnValue(Promise.resolve('sampleToken')); + spyOn(ediModel, 'getData').and.returnValue(Promise.resolve([ + { + expiry_date: '2001-01-01', + entry_date: '2001-01-01', + genus_id: 1, + latin_genus_name: 'Oasis', + change_date_time: '2001-03-15T10:30:15+01:00', + }, { + expiry_date: null, + entry_date: '2001-01-02', + genus_id: 2, + latin_genus_name: 'Ibiza', + change_date_time: '2001-02-03T18:20:42+00:00', + } + ])); + + await ediModel.syncData(ctx, options); + await tx.rollback(); + } catch (e) { + error = e; + await tx.rollback(); + } + + expect(error).toBeUndefined(); + }); + + it('should throw an error if floricode service is not configured', async function() { + let error; + try { + await ediModel.syncData(); + } catch (e) { + error = e; + } + expect(error).toBeDefined(); + }); +}); diff --git a/back/methods/edi/syncData.js b/back/methods/edi/syncData.js index 1ba98f32fb..1cba9cd4f2 100644 --- a/back/methods/edi/syncData.js +++ b/back/methods/edi/syncData.js @@ -19,19 +19,22 @@ module.exports = Self => { } }); - Self.syncData = async ctx => { + Self.syncData = async (ctx, options) => { const models = Self.app.models; + const myOptions = {}; + if (typeof options == 'object') + Object.assign(myOptions, options); let tx; try { - const floricodeConfig = await models.FloricodeConfig.findOne(); + const floricodeConfig = await models.FloricodeConfig.findOne({}, myOptions); if (!floricodeConfig) throw new UserError(`Floricode service is not configured`); - const tables = await models.TableMultiConfig.find(); + const tables = await models.TableMultiConfig.find({}, myOptions); if (!tables?.length) throw new UserError(`No tables to sync`); - const token = await getToken(floricodeConfig); + const token = await Self.getToken(floricodeConfig); for (const table of tables) { - const data = await getData(floricodeConfig.url, table.method, token); + const data = await Self.getData(floricodeConfig.url, table.method, token); if (!data) continue; tx = await Self.beginTransaction({}); @@ -54,24 +57,30 @@ module.exports = Self => { } }; - const getToken = async ({ url, user, password }) => - (await axios.post(`${url}/oauth/token`, - { grant_type: 'client_credentials', client_id: user, client_secret: password }, - { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } } + Self.getToken = async function ({ url, user, password}) { + return (await axios.post(`${url}/oauth/token`, { + grant_type: 'client_credentials', + client_id: user, + client_secret: password + }, { + headers: { 'Content-Type': 'application/x-www-form-urlencoded' } + } )).data.access_token; + }; - async function getData(url, method, token) { + Self.getData = async function (url, method, token) { let data = []; let count = 0; - const maxCount = (await get(`${url}/v2/${method}?$count=true`, token))["@odata.count"]; + const maxCount = (await Self.get(`${url}/v2/${method}?$count=true`, token))["@odata.count"]; while (count < maxCount) { - const response = await get(`${url}/v2/${method}?$skip=${count}`, token) + const response = await Self.get(`${url}/v2/${method}?$skip=${count}`, token) data.push(...response.value); count += response.value.length; } return data; - } + }; - const get = async (url, token) => - (await axios.get(url, { headers: { Authorization: `Bearer ${token}` } })).data; + Self.get = async function get(url, token) { + return (await axios.get(url, { headers: { Authorization: `Bearer ${token}` } })).data; + }; }; diff --git a/db/dump/fixtures.before.sql b/db/dump/fixtures.before.sql index 8dc363590c..7e3facd591 100644 --- a/db/dump/fixtures.before.sql +++ b/db/dump/fixtures.before.sql @@ -4142,3 +4142,6 @@ INSERT IGNORE INTO vn.vehicleType (id, name) (2, 'furgoneta'), (3, 'cabeza tractora'), (4, 'remolque'); + +INSERT INTO edi.tableMultiConfig (fileName, toTable, file, `method`, updated) + VALUES ('FG', 'genus', 'florecompc2', 'VBN/Genus', '2001-01-01'); From 2b248e3f6c6cbbb83322739798df7a57b155277e Mon Sep 17 00:00:00 2001 From: guillermo Date: Wed, 19 Feb 2025 12:36:12 +0100 Subject: [PATCH 06/10] feat: refs #8244 Requested changes --- back/methods/edi/specs/syncData.spec.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/back/methods/edi/specs/syncData.spec.js b/back/methods/edi/specs/syncData.spec.js index 81adc92b4a..14b3b515df 100644 --- a/back/methods/edi/specs/syncData.spec.js +++ b/back/methods/edi/specs/syncData.spec.js @@ -41,6 +41,8 @@ describe('edi syncData()', function() { } expect(error).toBeUndefined(); + const data = await ediModel.rawSql('SELECT * FROM edi.genus', [], options); + expect(data.length).toEqual(2); }); it('should throw an error if floricode service is not configured', async function() { From 0019c37f8379ada9fab2976a33576998aca0e6c1 Mon Sep 17 00:00:00 2001 From: guillermo Date: Wed, 19 Feb 2025 13:45:06 +0100 Subject: [PATCH 07/10] feat: refs #8244 Requested changes --- back/methods/edi/specs/syncData.spec.js | 10 ++++++---- back/methods/edi/syncData.js | 21 +++++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/back/methods/edi/specs/syncData.spec.js b/back/methods/edi/specs/syncData.spec.js index 14b3b515df..7fe24cca9c 100644 --- a/back/methods/edi/specs/syncData.spec.js +++ b/back/methods/edi/specs/syncData.spec.js @@ -5,7 +5,6 @@ describe('edi syncData()', function() { it('should be insert into the table', async() => { const tx = await ediModel.beginTransaction({}); - const ctx = {req: {accessToken: {userId: 1}}}; const options = {transaction: tx}; let error; try { @@ -33,7 +32,12 @@ describe('edi syncData()', function() { } ])); - await ediModel.syncData(ctx, options); + await ediModel.syncData(options); + + const data = await ediModel.rawSql('SELECT * FROM edi.genus', [], options); + // The table is deleted within the method itself; it will always be 2 + expect(data.length).toEqual(2); + await tx.rollback(); } catch (e) { error = e; @@ -41,8 +45,6 @@ describe('edi syncData()', function() { } expect(error).toBeUndefined(); - const data = await ediModel.rawSql('SELECT * FROM edi.genus', [], options); - expect(data.length).toEqual(2); }); it('should throw an error if floricode service is not configured', async function() { diff --git a/back/methods/edi/syncData.js b/back/methods/edi/syncData.js index 1cba9cd4f2..01789148de 100644 --- a/back/methods/edi/syncData.js +++ b/back/methods/edi/syncData.js @@ -6,7 +6,7 @@ const path = require('path'); const { pipeline } = require('stream/promises'); module.exports = Self => { - Self.remoteMethodCtx('syncData', { + Self.remoteMethod('syncData', { description: 'Sync schema data from external provider', accessType: 'WRITE', returns: { @@ -19,7 +19,7 @@ module.exports = Self => { } }); - Self.syncData = async (ctx, options) => { + Self.syncData = async options => { const models = Self.app.models; const myOptions = {}; if (typeof options == 'object') @@ -37,19 +37,24 @@ module.exports = Self => { const data = await Self.getData(floricodeConfig.url, table.method, token); if (!data) continue; - tx = await Self.beginTransaction({}); - const options = {transaction: tx, userId: ctx.req.accessToken.userId}; + if (!myOptions.transaction) { + tx = await Self.beginTransaction({}); + myOptions.transaction = tx; + } - await Self.rawSql(`DELETE FROM edi.??`, [table.toTable]); + await Self.rawSql(`DELETE FROM edi.??`, [table.toTable], myOptions); const ws = fs.createWriteStream(path.join(__dirname, `/${table.toTable}.csv`)); await pipeline(fastCsv.write(data, { delimiter: ';' }), ws); const templatePath = path.join(__dirname, `./syncSql/${table.toTable}.sql`); const sqlTemplate = await fs.readFile(templatePath, 'utf8'); - await Self.rawSql(sqlTemplate, [ws.path], options); + await Self.rawSql(sqlTemplate, [ws.path], myOptions); fs.remove(ws.path); - await table.updateAttribute('updated', Date.vnNew(), options); - await tx.commit(); + await table.updateAttribute('updated', Date.vnNew(), myOptions); + if (tx) { + await tx.commit(); + delete myOptions.transaction; + } } } catch (e) { if (tx) await tx.rollback(); From ea5b78394d8b1f0d68d0820a2247559a258c717f Mon Sep 17 00:00:00 2001 From: guillermo Date: Fri, 21 Feb 2025 12:08:34 +0100 Subject: [PATCH 08/10] fix: refs #8244 refs #7759 Requested changes --- back/methods/edi/syncData.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/back/methods/edi/syncData.js b/back/methods/edi/syncData.js index 01789148de..802e4911e4 100644 --- a/back/methods/edi/syncData.js +++ b/back/methods/edi/syncData.js @@ -24,7 +24,7 @@ module.exports = Self => { const myOptions = {}; if (typeof options == 'object') Object.assign(myOptions, options); - let tx; + let tx, ws; try { const floricodeConfig = await models.FloricodeConfig.findOne({}, myOptions); if (!floricodeConfig) throw new UserError(`Floricode service is not configured`); @@ -43,13 +43,13 @@ module.exports = Self => { } await Self.rawSql(`DELETE FROM edi.??`, [table.toTable], myOptions); - const ws = fs.createWriteStream(path.join(__dirname, `/${table.toTable}.csv`)); + ws = fs.createWriteStream(path.join(__dirname, `/${table.toTable}.csv`)); await pipeline(fastCsv.write(data, { delimiter: ';' }), ws); const templatePath = path.join(__dirname, `./syncSql/${table.toTable}.sql`); const sqlTemplate = await fs.readFile(templatePath, 'utf8'); await Self.rawSql(sqlTemplate, [ws.path], myOptions); - fs.remove(ws.path); + await fs.remove(ws.path); await table.updateAttribute('updated', Date.vnNew(), myOptions); if (tx) { await tx.commit(); @@ -58,6 +58,8 @@ module.exports = Self => { } } catch (e) { if (tx) await tx.rollback(); + if (await fs.pathExists(ws.path)) + await fs.remove(ws.path); throw e; } }; From 76751ad584469e48c2db08b294ddd23dbe25ec67 Mon Sep 17 00:00:00 2001 From: guillermo Date: Fri, 21 Feb 2025 12:11:42 +0100 Subject: [PATCH 09/10] fix: refs #8244 refs #7759 Requested changes --- back/methods/edi/syncData.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/back/methods/edi/syncData.js b/back/methods/edi/syncData.js index 802e4911e4..36c527aaa2 100644 --- a/back/methods/edi/syncData.js +++ b/back/methods/edi/syncData.js @@ -58,8 +58,8 @@ module.exports = Self => { } } catch (e) { if (tx) await tx.rollback(); - if (await fs.pathExists(ws.path)) - await fs.remove(ws.path); + if (await fs.pathExists(ws?.path)) + await fs.remove(ws?.path); throw e; } }; From 87b3abd4e2be763574d69f6f0353530988276f72 Mon Sep 17 00:00:00 2001 From: guillermo Date: Thu, 27 Feb 2025 09:04:08 +0100 Subject: [PATCH 10/10] fix: refs #8244 Path test --- .../ticket/back/methods/ticket/specs/getTicketProblems.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ticket/back/methods/ticket/specs/getTicketProblems.spec.js b/modules/ticket/back/methods/ticket/specs/getTicketProblems.spec.js index f46c27e4dc..40d4b6b1f3 100644 --- a/modules/ticket/back/methods/ticket/specs/getTicketProblems.spec.js +++ b/modules/ticket/back/methods/ticket/specs/getTicketProblems.spec.js @@ -10,7 +10,7 @@ describe('ticket getTicketProblems()', () => { const problems = await models.Ticket.getTicketProblems(ctx, 11, options); - expect(problems[7].totalProblems).toEqual(3); + expect(problems[7].totalProblems).toEqual(2); await tx.rollback(); } catch (e) {