Merge branch 'dev' into 8645-addSupplierRefToAgencyTermFilter
gitea/salix/pipeline/pr-dev This commit looks good Details

This commit is contained in:
Jose Antonio Tubau 2025-02-27 11:00:22 +00:00
commit 44dc7bbb58
25 changed files with 487 additions and 3 deletions

View File

@ -0,0 +1,59 @@
const models = require('vn-loopback/server/server').models;
describe('edi syncData()', function() {
const ediModel = models.Edi;
it('should be insert into the table', async() => {
const tx = await ediModel.beginTransaction({});
const options = {transaction: tx};
let error;
try {
await models.FloricodeConfig.create({
id: 1,
url: 'http://sample.com',
user: 'sample',
password: 'sample'
}, options);
spyOn(ediModel, 'getToken').and.returnValue(Promise.resolve('sampleToken'));
spyOn(ediModel, 'getData').and.returnValue(Promise.resolve([
{
expiry_date: '2001-01-01',
entry_date: '2001-01-01',
genus_id: 1,
latin_genus_name: 'Oasis',
change_date_time: '2001-03-15T10:30:15+01:00',
}, {
expiry_date: null,
entry_date: '2001-01-02',
genus_id: 2,
latin_genus_name: 'Ibiza',
change_date_time: '2001-02-03T18:20:42+00:00',
}
]));
await ediModel.syncData(options);
const data = await ediModel.rawSql('SELECT * FROM edi.genus', [], options);
// The table is deleted within the method itself; it will always be 2
expect(data.length).toEqual(2);
await tx.rollback();
} catch (e) {
error = e;
await tx.rollback();
}
expect(error).toBeUndefined();
});
it('should throw an error if floricode service is not configured', async function() {
let error;
try {
await ediModel.syncData();
} catch (e) {
error = e;
}
expect(error).toBeDefined();
});
});

View File

@ -0,0 +1,93 @@
const UserError = require('vn-loopback/util/user-error');
const fs = require('fs-extra');
const fastCsv = require("fast-csv");
const axios = require('axios');
const path = require('path');
const { pipeline } = require('stream/promises');
module.exports = Self => {
Self.remoteMethod('syncData', {
description: 'Sync schema data from external provider',
accessType: 'WRITE',
returns: {
type: 'object',
root: true
},
http: {
path: `/syncData`,
verb: 'POST'
}
});
Self.syncData = async options => {
const models = Self.app.models;
const myOptions = {};
if (typeof options == 'object')
Object.assign(myOptions, options);
let tx, ws;
try {
const floricodeConfig = await models.FloricodeConfig.findOne({}, myOptions);
if (!floricodeConfig) throw new UserError(`Floricode service is not configured`);
const tables = await models.TableMultiConfig.find({}, myOptions);
if (!tables?.length) throw new UserError(`No tables to sync`);
const token = await Self.getToken(floricodeConfig);
for (const table of tables) {
const data = await Self.getData(floricodeConfig.url, table.method, token);
if (!data) continue;
if (!myOptions.transaction) {
tx = await Self.beginTransaction({});
myOptions.transaction = tx;
}
await Self.rawSql(`DELETE FROM edi.??`, [table.toTable], myOptions);
ws = fs.createWriteStream(path.join(__dirname, `/${table.toTable}.csv`));
await pipeline(fastCsv.write(data, { delimiter: ';' }), ws);
const templatePath = path.join(__dirname, `./syncSql/${table.toTable}.sql`);
const sqlTemplate = await fs.readFile(templatePath, 'utf8');
await Self.rawSql(sqlTemplate, [ws.path], myOptions);
await fs.remove(ws.path);
await table.updateAttribute('updated', Date.vnNew(), myOptions);
if (tx) {
await tx.commit();
delete myOptions.transaction;
}
}
} catch (e) {
if (tx) await tx.rollback();
if (await fs.pathExists(ws?.path))
await fs.remove(ws?.path);
throw e;
}
};
Self.getToken = async function ({ url, user, password}) {
return (await axios.post(`${url}/oauth/token`, {
grant_type: 'client_credentials',
client_id: user,
client_secret: password
}, {
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
}
)).data.access_token;
};
Self.getData = async function (url, method, token) {
let data = [];
let count = 0;
const maxCount = (await Self.get(`${url}/v2/${method}?$count=true`, token))["@odata.count"];
while (count < maxCount) {
const response = await Self.get(`${url}/v2/${method}?$skip=${count}`, token)
data.push(...response.value);
count += response.value.length;
}
return data;
};
Self.get = async function get(url, token) {
return (await axios.get(url, { headers: { Authorization: `Bearer ${token}` } })).data;
};
};

View File

@ -0,0 +1,14 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`bucket`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11)
SET bucket_id = @col3,
bucket_type_id = @col5,
description = @col6,
x_size = @col7,
y_size = @col8,
z_size = @col9,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col11, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,10 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`bucket_type`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5)
SET bucket_type_id = @col3,
description = @col4,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col5, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,11 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`feature`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5, @col6)
SET item_id = @col3,
feature_type_id = @col4,
feature_value = @col5,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,10 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`genus`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5)
SET genus_id = @col3,
latin_genus_name = @col4,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col5, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,14 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`item`
CHARACTER SET ascii
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11)
SET id = @col3,
product_name = @col5,
name = @col6,
plant_id = @col8,
group_id = @col10,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col11, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,12 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`item_feature`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5, @col6, @col7)
SET item_id = @col3,
feature = @col4,
regulation_type = @col5,
presentation_order = @col6,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col7, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,10 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`item_group`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5)
SET group_code = @col3,
dutch_group_description = @col4,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col5, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,11 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`plant`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8)
SET plant_id = @col4,
genus_id = @col5,
specie_id = @col6,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col8, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,11 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`specie`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5, @col6)
SET specie_id = @col3,
genus_id = @col4,
latin_species_name = @col5,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,11 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`supplier`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n'
(@col1, @col2, @col3, @col4, @col5, @col6, @col7, @col8, @col9, @col10, @col11, @col12, @col13, @col14, @col15, @col16, @col17, @col18, @col19, @col20, @col21)
SET GLNAddressCode = @col3,
supplier_id = @col9,
company_name = @col4,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col17, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,10 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`type`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6)
SET type_id = @col3,
type_group_id = @col4,
description = @col5,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -0,0 +1,10 @@
LOAD DATA LOCAL INFILE ?
INTO TABLE `edi`.`value`
FIELDS TERMINATED BY ';'
LINES TERMINATED BY '\n' (@col1, @col2, @col3, @col4, @col5, @col6)
SET type_id = @col3,
type_value = @col4,
type_description = @col5,
entry_date = STR_TO_DATE(@col2, '%Y-%m-%d'),
expiry_date = STR_TO_DATE(@col1, '%Y-%m-%d'),
change_date_time = STR_TO_DATE(REGEXP_REPLACE(@col6, '\\+.*$', ''), '%Y-%m-%dT%H:%i:%s')

View File

@ -70,6 +70,9 @@
"Expedition_PrintOut": {
"dataSource": "vn"
},
"FloricodeConfig": {
"dataSource": "vn"
},
"Image": {
"dataSource": "vn"
},
@ -154,6 +157,9 @@
"SaySimpleConfig": {
"dataSource": "vn"
},
"TableMultiConfig": {
"dataSource": "vn"
},
"TempContainer": {
"dataSource": "tempStorage"
},

View File

@ -1,3 +1,4 @@
module.exports = Self => {
require('../methods/edi/updateData')(Self);
require('../methods/edi/syncData')(Self);
};

View File

@ -0,0 +1,28 @@
{
"name": "FloricodeConfig",
"base": "VnModel",
"options": {
"mysql": {
"table": "edi.floricodeConfig"
}
},
"properties": {
"id": {
"type": "number",
"id": true,
"required": true
},
"url": {
"type": "string",
"required": true
},
"user": {
"type": "string",
"required": false
},
"password": {
"type": "string",
"required": false
}
}
}

View File

@ -0,0 +1,24 @@
{
"name": "TableMultiConfig",
"base": "VnModel",
"options": {
"mysql": {
"table": "edi.tableMultiConfig"
}
},
"properties": {
"id": {
"type": "number",
"required": true
},
"toTable": {
"type": "string"
},
"method": {
"type": "string"
},
"updated": {
"type": "date"
}
}
}

View File

@ -4154,5 +4154,8 @@ INSERT IGNORE INTO vn.vehicleType (id, name)
(3, 'cabeza tractora'),
(4, 'remolque');
INSERT INTO edi.tableMultiConfig (fileName, toTable, file, `method`, updated)
VALUES ('FG', 'genus', 'florecompc2', 'VBN/Genus', '2001-01-01');
INSERT INTO vn.addressWaste (addressFk, type)
VALUES (11, 'fault');

View File

@ -0,0 +1,11 @@
CREATE TABLE `edi`.`floricodeConfig` (
`id` int(10) unsigned NOT NULL,
`url` varchar(100) NOT NULL,
`user` varchar(50) NOT NULL,
`password` varchar(100) NOT NULL,
PRIMARY KEY (`id`),
CONSTRAINT `floricodeConfig_check` CHECK (`id` = 1)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
INSERT INTO salix.ACL (model,property,accessType,permission,principalType,principalId)
VALUES ('Edi','syncData','WRITE','ALLOW','ROLE','system');

View File

@ -0,0 +1,65 @@
ALTER TABLE edi.tableMultiConfig
ADD `method` varchar(100) DEFAULT NULL NULL AFTER file,
DROP PRIMARY KEY,
DROP KEY to_table,
ADD id int(10) unsigned NULL FIRST;
UPDATE edi.tableMultiConfig
SET id = 1, method = 'VBN/Packaging'
WHERE id IS NULL
AND fileName='CK';
UPDATE edi.tableMultiConfig
SET id = 2, method = 'VBN/PackagingType'
WHERE id IS NULL
AND fileName='FB';
UPDATE edi.tableMultiConfig
SET id = 3, method = 'VBN/ProductFeature'
WHERE id IS NULL
AND fileName='FF';
UPDATE edi.tableMultiConfig
SET id = 4, method = 'VBN/Genus'
WHERE id IS NULL
AND fileName='FG';
UPDATE edi.tableMultiConfig
SET id = 5, method = 'VBN/Product'
WHERE id IS NULL
AND fileName='FP';
UPDATE edi.tableMultiConfig
SET id = 6, method = 'VBN/RegulatoryFeatureType'
WHERE id IS NULL
AND fileName='FY';
UPDATE edi.tableMultiConfig
SET id = 7, method = 'VBN/ProductGroup'
WHERE id IS NULL
AND fileName='FO';
UPDATE edi.tableMultiConfig
SET id = 8, method = 'VBN/Plant'
WHERE id IS NULL
AND fileName='FT';
UPDATE edi.tableMultiConfig
SET id = 9, method = 'VBN/Species'
WHERE id IS NULL
AND fileName='FS';
UPDATE edi.tableMultiConfig
SET id = 10, method = 'FEC/Company'
WHERE id IS NULL
AND fileName='CC';
UPDATE edi.tableMultiConfig
SET id = 11, method = 'VBN/FeatureType'
WHERE id IS NULL
AND fileName='FE';
UPDATE edi.tableMultiConfig
SET id = 12, method = 'VBN/FeatureValue'
WHERE id IS NULL
AND fileName='FV';

View File

@ -0,0 +1,4 @@
ALTER TABLE edi.tableMultiConfig
ADD CONSTRAINT tableMultiConfig_pk PRIMARY KEY (id),
MODIFY COLUMN id int(10) unsigned auto_increment NOT NULL,
ADD CONSTRAINT tableMultiConfig_unique UNIQUE KEY (toTable);

View File

@ -10,7 +10,7 @@ describe('ticket getTicketProblems()', () => {
const problems = await models.Ticket.getTicketProblems(ctx, 11, options);
expect(problems[7].totalProblems).toEqual(3);
expect(problems[7].totalProblems).toEqual(2);
await tx.rollback();
} catch (e) {

View File

@ -18,6 +18,7 @@
"body-parser": "^1.19.2",
"compression": "^1.7.3",
"ejs": "2.3.1",
"fast-csv": "^5.0.2",
"form-data": "^4.0.0",
"fs-extra": "^5.0.0",
"ftps": "^1.2.0",

View File

@ -20,6 +20,9 @@ dependencies:
ejs:
specifier: 2.3.1
version: 2.3.1
fast-csv:
specifier: ^5.0.2
version: 5.0.2
form-data:
specifier: ^4.0.0
version: 4.0.0
@ -1708,6 +1711,27 @@ packages:
- supports-color
dev: true
/@fast-csv/format@5.0.2:
resolution: {integrity: sha512-fRYcWvI8vs0Zxa/8fXd/QlmQYWWkJqKZPAXM+vksnplb3owQFKTPPh9JqOtD0L3flQw/AZjjXdPkD7Kp/uHm8g==}
dependencies:
lodash.escaperegexp: 4.1.2
lodash.isboolean: 3.0.3
lodash.isequal: 4.5.0
lodash.isfunction: 3.0.9
lodash.isnil: 4.0.0
dev: false
/@fast-csv/parse@5.0.2:
resolution: {integrity: sha512-gMu1Btmm99TP+wc0tZnlH30E/F1Gw1Tah3oMDBHNPe9W8S68ixVHjt89Wg5lh7d9RuQMtwN+sGl5kxR891+fzw==}
dependencies:
lodash.escaperegexp: 4.1.2
lodash.groupby: 4.6.0
lodash.isfunction: 3.0.9
lodash.isnil: 4.0.0
lodash.isundefined: 3.0.1
lodash.uniq: 4.5.0
dev: false
/@fastify/busboy@2.1.0:
resolution: {integrity: sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==}
engines: {node: '>=14'}
@ -6114,6 +6138,14 @@ packages:
time-stamp: 1.1.0
dev: true
/fast-csv@5.0.2:
resolution: {integrity: sha512-CnB2zYAzzeh5Ta0UhSf32NexLy2SsEsSMY+fMWPV40k1OgaLEbm9Hf5dms3z/9fASZHBjB6i834079gVeksEqQ==}
engines: {node: '>=10.0.0'}
dependencies:
'@fast-csv/format': 5.0.2
'@fast-csv/parse': 5.0.2
dev: false
/fast-deep-equal@2.0.1:
resolution: {integrity: sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==}
dev: false
@ -9383,9 +9415,12 @@ packages:
lodash._root: 3.0.1
dev: true
/lodash.escaperegexp@4.1.2:
resolution: {integrity: sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==}
dev: false
/lodash.groupby@4.6.0:
resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==}
dev: true
/lodash.isarguments@3.1.0:
resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
@ -9395,10 +9430,31 @@ packages:
resolution: {integrity: sha512-JwObCrNJuT0Nnbuecmqr5DgtuBppuCvGD9lxjFpAzwnVtdGoDQ1zig+5W8k5/6Gcn0gZ3936HDAlGd28i7sOGQ==}
dev: true
/lodash.isboolean@3.0.3:
resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==}
dev: false
/lodash.isequal@4.5.0:
resolution: {integrity: sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==}
deprecated: This package is deprecated. Use require('node:util').isDeepStrictEqual instead.
dev: false
/lodash.isfunction@3.0.9:
resolution: {integrity: sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==}
dev: false
/lodash.isnil@4.0.0:
resolution: {integrity: sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==}
dev: false
/lodash.isplainobject@4.0.6:
resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==}
dev: true
/lodash.isundefined@3.0.1:
resolution: {integrity: sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==}
dev: false
/lodash.kebabcase@4.1.1:
resolution: {integrity: sha512-N8XRTIMMqqDgSy4VLKPnJ/+hpGZN+PHQiJnSenYqPaVV/NCqEogTnAdZLQiGKhxX+JCs8waWq2t1XHWKOmlY8g==}
dev: true
@ -9458,7 +9514,6 @@ packages:
/lodash.uniq@4.5.0:
resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==}
dev: true
/lodash.upperfirst@4.3.1:
resolution: {integrity: sha512-sReKOYJIJf74dhJONhU4e0/shzi1trVbSWDOhKYE5XV2O+H7Sb2Dihwuc7xWxVl+DgFPyTqIN3zMfT9cq5iWDg==}