From fa8eab61395e3058abb37e6aece3accca896ded2 Mon Sep 17 00:00:00 2001 From: guillermo Date: Wed, 17 May 2023 10:24:10 +0200 Subject: [PATCH] refs #4823 Supplier to organization and more --- .env_template | 3 +- floriday.js | 10 +- main.js | 3 +- models/conf/sequenceNumber.js | 34 ---- models/{conf => config}/clientConfig.js | 0 .../organization.js} | 12 +- .../{supplier => organization}/warehouses.js | 4 +- models/sequelize.js | 60 +++---- models/supplier/connections.js | 21 --- models/supplyLine/supplyLine.js | 2 +- models/tradeItem/tradeItem.js | 2 +- utils.js | 160 +++++------------- 12 files changed, 95 insertions(+), 216 deletions(-) delete mode 100644 models/conf/sequenceNumber.js rename models/{conf => config}/clientConfig.js (100%) rename models/{supplier/supplier.js => organization/organization.js} (87%) rename models/{supplier => organization}/warehouses.js (95%) delete mode 100644 models/supplier/connections.js diff --git a/.env_template b/.env_template index fd7f3d1..464a07a 100644 --- a/.env_template +++ b/.env_template @@ -21,8 +21,7 @@ MS_PRODUCTION_SCHEDULE = 300000 MS_TEST_SCHEDULE = 100000 SECRETS = true FORCE_SYNC = true -SYNC_SEQUENCE = true -SYNC_SUPPLIER = true +SYNC_ORGANIZATION = true SYNC_WAREHOUSE = true SYNC_CONN = true SYNC_TRADEITEM = true diff --git a/floriday.js b/floriday.js index 15fb275..bafe06b 100644 --- a/floriday.js +++ b/floriday.js @@ -11,10 +11,9 @@ class Floriday { try { await utils.checkConfig(); await utils.requestToken(); - if (JSON.parse(env.SYNC_SEQUENCE)) await utils.syncSequence(); - if (JSON.parse(env.SYNC_SUPPLIER)) await utils.syncSuppliers(); - if (JSON.parse(env.SYNC_WAREHOUSE)) await utils.syncWarehouses(); + if (JSON.parse(env.SYNC_ORGANIZATION)) await utils.syncSuppliers(); if (JSON.parse(env.SYNC_CONN)) await utils.syncConnections(); + if (JSON.parse(env.SYNC_WAREHOUSE)) await utils.syncWarehouses(); if (JSON.parse(env.SYNC_TRADEITEM)) await utils.syncTradeItems(); } catch (err) { utils.criticalError(err); @@ -55,12 +54,15 @@ class Floriday { async trunk() { try{ + if (JSON.parse(env.SYNC_CONN)) await utils.syncConnections(); await utils.syncSupplyLines(); // Continuar con todo lo que haga falta realizar en la rutina } catch (err) { - throw err; + if (err.name === 'SequelizeConnectionRefusedError') + throw err; + utils.criticalError(err); } } diff --git a/main.js b/main.js index 0234789..8903a06 100644 --- a/main.js +++ b/main.js @@ -2,12 +2,11 @@ import Floriday from './floriday.js'; async function main() { const floriday = new Floriday(); - await floriday.start(); - process.on('SIGINT', async () => { await floriday.stop(); process.exit(); }); + await floriday.start(); try { await floriday.schedule() diff --git a/models/conf/sequenceNumber.js b/models/conf/sequenceNumber.js deleted file mode 100644 index d68e811..0000000 --- a/models/conf/sequenceNumber.js +++ /dev/null @@ -1,34 +0,0 @@ -import { Sequelize } from 'sequelize'; - -const sequenceNumber = { - id: { - type: Sequelize.INTEGER, - primaryKey: true, - autoIncrement: true, - }, - sequenceNumber: { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: 0, - }, - maximumSequenceNumber: { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: 0, - }, - model: { - type: Sequelize.STRING, - }, -}; - -export default (sequelize) => { - const sequenceNumbers = sequelize.define( - 'sequenceNumber', - sequenceNumber, - { - timestamps: false, - freezeTableName: true, - } - ); - return sequenceNumbers; -}; diff --git a/models/conf/clientConfig.js b/models/config/clientConfig.js similarity index 100% rename from models/conf/clientConfig.js rename to models/config/clientConfig.js diff --git a/models/supplier/supplier.js b/models/organization/organization.js similarity index 87% rename from models/supplier/supplier.js rename to models/organization/organization.js index cfcb745..bbaaa38 100644 --- a/models/supplier/supplier.js +++ b/models/organization/organization.js @@ -1,7 +1,7 @@ import { Sequelize } from 'sequelize'; -const suppliers = { - supplierOrganizationId: { +const organizations = { + organizationId: { type: Sequelize.STRING, allowNull: false, primaryKey: true, @@ -60,12 +60,12 @@ const suppliers = { }; export default (sequelize) => { - const Suppliers = sequelize.define( - 'supplier', - suppliers, { + const Organizations = sequelize.define( + 'organization', + organizations, { timestamps: false, freezeTableName: true, } ); - return Suppliers; + return Organizations; }; \ No newline at end of file diff --git a/models/supplier/warehouses.js b/models/organization/warehouses.js similarity index 95% rename from models/supplier/warehouses.js rename to models/organization/warehouses.js index bc23320..31f6fa2 100644 --- a/models/supplier/warehouses.js +++ b/models/organization/warehouses.js @@ -32,7 +32,7 @@ const warehouses = { sequenceNumber: { type: Sequelize.INTEGER, }, - supplierOrganizationId: { + organizationId: { type: Sequelize.STRING, }, lastSync: { @@ -49,7 +49,7 @@ const warehouses = { export default (sequelize) => { const Warehouses = sequelize.define( - 'supplier_warehouses', + 'organization_warehouses', warehouses, { timestamps: false, freezeTableName: true, diff --git a/models/sequelize.js b/models/sequelize.js index 0bd122d..47a4519 100644 --- a/models/sequelize.js +++ b/models/sequelize.js @@ -34,13 +34,11 @@ import supplyLine from './supplyLine/supplyLine.js'; import volumePrices from './supplyLine/volumePrices.js'; // Conf Models -import clientConfig from './conf/clientConfig.js'; -import sequenceNumber from './conf/sequenceNumber.js'; +import clientConfig from './config/clientConfig.js'; -// Supplier Models -import supplier from './supplier/supplier.js'; -import connections from './supplier/connections.js'; -import warehouses from './supplier/warehouses.js'; +// Organization Models +import organization from './organization/organization.js'; +import warehouses from './organization/warehouses.js'; // TradeItem Models import tradeItem from './tradeItem/tradeItem.js'; @@ -74,31 +72,29 @@ import characteristics from './tradeItem/characteristics.js'; */ let models = { sequelize: sequelize, + clientConfig: clientConfig(sequelize), + organization: organization(sequelize), + warehouses: warehouses(sequelize), tradeItem: tradeItem(sequelize), - packingConfiguration: packingConfigurations(sequelize), - photo: photos(sequelize), + botanicalName: botanicalNames(sequelize), characteristic: characteristics(sequelize), countryOfOriginIsoCode: countryOfOriginIsoCodes(sequelize), - package: packageModel(sequelize), + packingConfiguration: packingConfigurations(sequelize), + photo: photos(sequelize), seasonalPeriod: seasonalPeriod(sequelize), - clientConfig: clientConfig(sequelize), - botanicalName: botanicalNames(sequelize), supplyLine: supplyLine(sequelize), volumePrices: volumePrices(sequelize), - supplier: supplier(sequelize), - sequenceNumber: sequenceNumber(sequelize), - connection: connections(sequelize), - warehouses: warehouses(sequelize), + package: packageModel(sequelize), }; try { - /* Remove ID atribute from models */ + // Remove ID atribute from models models.characteristic.removeAttribute('id'); models.seasonalPeriod.removeAttribute('id'); models.package.removeAttribute('id'); models.botanicalName.removeAttribute('id'); models.countryOfOriginIsoCode.removeAttribute('id'); - /* ------------------------------ */ + // ------------------------------- models.characteristic.belongsTo(models.tradeItem, { foreignKey: 'tradeItemId', @@ -160,22 +156,28 @@ try { targetKey: 'tradeItemId', }); - models.tradeItem.belongsTo(models.supplier, { - foreignKey: 'supplierOrganizationId', - as: 'supplierOrganization_Id', - targetKey: 'supplierOrganizationId', + models.supplyLine.belongsTo(models.warehouses, { + foreignKey: 'warehouseId', + as: 'warehouse_Fk', + targetKey: 'warehouseId', }); - models.supplyLine.belongsTo(models.supplier, { - foreignKey: 'supplierOrganizationId', - as: 'supplierOrganization_Id', - targetKey: 'supplierOrganizationId', + models.tradeItem.belongsTo(models.organization, { + foreignKey: 'organizationId', + as: 'Organization_Fk', + targetKey: 'organizationId', }); - models.warehouses.belongsTo(models.supplier, { - foreignKey: 'supplierOrganizationId', - as: 'supplierOrganization_Id', - targetKey: 'supplierOrganizationId', + models.supplyLine.belongsTo(models.organization, { + foreignKey: 'organizationId', + as: 'Organization_Fk', + targetKey: 'organizationId', + }); + + models.warehouses.belongsTo(models.organization, { + foreignKey: 'organizationId', + as: 'Organization_Fk', + targetKey: 'organizationId', }); } catch (err) { criticalError(err); diff --git a/models/supplier/connections.js b/models/supplier/connections.js deleted file mode 100644 index 9d45e34..0000000 --- a/models/supplier/connections.js +++ /dev/null @@ -1,21 +0,0 @@ -import { Sequelize } from 'sequelize'; - -const connections = { - supplierOrganizationId: { - type: Sequelize.STRING, - primaryKey: true, - allowNull: false, - }, -}; - -export default (sequelize) => { - const connection = sequelize.define( - 'supplier_connections', - connections, - { - timestamps: false, - freezeTableName: true, - } - ); - return connection; -}; diff --git a/models/supplyLine/supplyLine.js b/models/supplyLine/supplyLine.js index 5c32a7a..0b51b2f 100644 --- a/models/supplyLine/supplyLine.js +++ b/models/supplyLine/supplyLine.js @@ -59,7 +59,7 @@ const supplyLine = { tradeItemId : { type: Sequelize.STRING, }, - supplierOrganizationId : { + organizationId : { type: Sequelize.STRING, }, lastSync: { diff --git a/models/tradeItem/tradeItem.js b/models/tradeItem/tradeItem.js index 4e34e0e..ed2999a 100644 --- a/models/tradeItem/tradeItem.js +++ b/models/tradeItem/tradeItem.js @@ -32,7 +32,7 @@ const tradeItem = { isHiddenInCatalog: { type: Sequelize.BOOLEAN, }, - supplierOrganizationId: { + organizationId: { type: Sequelize.STRING }, lastSync: { diff --git a/utils.js b/utils.js index 0917f52..f0f36a6 100644 --- a/utils.js +++ b/utils.js @@ -184,74 +184,6 @@ export async function asyncQueue(fnArray, concurrency = 1) { return results; } -// 1. Create an array of functions that will be executed in a queue -// 2. Create a function that will execute the functions in the queue -// 3. Create an array of promises that will execute the run function -// 4. Execute the run function while the concurrency is greater than 0 -// 5. Return the results - -/** - * Syncs the sequence number for the given model - * if no params are given it will reset all the sequence number to 0 - * - * @param {Number} current - current sequence number - * @param {String} model - model name - * @param {Number} maximumSequenceNumber - maximum sequence number - * @returns - */ -export async function syncSequence(current = 0, model = null , maximumSequenceNumber = 0){ - if (model == null && current == 0){ - try { - const spinner = ora(`Syncing sequence...`).start(); - let mockModels = [ - 'supplier', - 'tradeItems', - 'supplyLines', - ]; - let i = 1; - for (let mockModel in mockModels) { - spinner.text = `Syncing ${i++} sequences...` - const element = mockModels[mockModel]; - await syncSequence(0, element); - } - spinner.succeed(); - } catch (err) { - spinner.fail(); - throw new Error(err); - } - } else if (current) { - let tx = await models.sequelize.transaction(); - try { - let sequence = await models.sequenceNumber.findOrCreate({ - where: { - model: model - }, - defaults: { - model: model, - sequenceNumber: current, - maximumSequenceNumber: maximumSequenceNumber - }, - transaction: tx - }); - - if (sequence[1] == false){ - await models.sequenceNumber.update({ - sequenceNumber: current, - maximumSequenceNumber: maximumSequenceNumber - }, { - where: { - model: model - }, - transaction: tx - }); - } - await tx.commit(); - } catch (error) { - await tx.rollback(); - console.log(`Error while syncing sequence number for: ${model}: ${error}`); - } - } -} /** * Sync the suppliers @@ -278,14 +210,12 @@ export async function syncSuppliers(){ spinner.text = `Syncing suppliers, ${maxSequenceNumber - curSequenceNumber} are missing` if (timeFinish) spinner.text = spinner.text + ` (${timeLeft})` - await models.supplier.upsert({ + await models.organization.upsert({ ...supplier, - supplierOrganizationId: supplier.organizationId, isConnected: JSON.parse(env.SUPPLIERS_ALWAYS_CONN), lastSync: moment(), }); }; - await syncSequence(curSequenceNumber, 'supplier', maxSequenceNumber); timeFinish = new moment(); timeToGoSec = (timeFinish.diff(timeStart, 'seconds') * (maxSequenceNumber - curSequenceNumber) / 1000) timeToGoMin = Math.trunc(timeToGoSec / 60) @@ -310,7 +240,10 @@ export async function syncConnections(){ await deleteConnections(); const spinner = ora(`Creating connections...`).start(); try { - let connections = await models.connection.findAll(); + let connections = await models.organization.findAll({ + attributes: ['organizationId'], + where: { isConnected: true } + }); let headers = { 'Content-Type': 'application/json', @@ -323,21 +256,10 @@ export async function syncConnections(){ let i = 1; for (let connection of connections) { spinner.text = `Creating ${i++} of ${connections.length} connections...` - let remoteConnection = remoteConnections.find(remoteConnection => remoteConnection == connection.supplierOrganizationId); + let remoteConnection = remoteConnections.find(remoteConnection => remoteConnection == connection.organizationId); if (!remoteConnection) - await vnRequest('PUT', `${env.API_URL}/connections/${connection.supplierOrganizationId}`, null, headers); - - await models.connection.update({isConnected: true }, { - where: { - supplierOrganizationId: connection.supplierOrganizationId - } - }); - await models.supplier.update({ isConnected: true }, { - where: { - supplierOrganizationId: connection.supplierOrganizationId - } - }); + await vnRequest('PUT', `${env.API_URL}/connections/${connection.organizationId}`, null, headers); } spinner.succeed(); } catch (err) { @@ -347,14 +269,15 @@ export async function syncConnections(){ } /** - * Sync the connections in Floriday + * Sync the trade items for organizations that are connected */ export async function syncTradeItems(){ const spinner = ora(`Syncing trade items...`).start(); - const suppliers = await models.supplier.findAll({ + const suppliers = await models.organization.findAll({ + attributes: ['organizationId'], where: { isConnected: true } }); - let i = 0, x = 1; + let i = 1, x = 1; for (let supplier of suppliers) { let headers = { 'Content-Type': 'application/json', @@ -362,7 +285,10 @@ export async function syncTradeItems(){ 'X-Api-Key': process.env.API_KEY }; try { - let tradeItems = (await vnRequest('GET', `${env.API_URL}/trade-items?supplierOrganizationId=${supplier.supplierOrganizationId}`, null, headers)).data + const params = new URLSearchParams({ + supplierOrganizationId: supplier.organizationId, + }).toString(); + let tradeItems = (await vnRequest('GET', `${env.API_URL}/trade-items?${params}`, null, headers)).data spinner.text = `Syncing ${i} trade items of [${x++}|${suppliers.length}] suppliers...` if (!tradeItems.length) continue; @@ -380,10 +306,10 @@ export async function syncTradeItems(){ } /** - * Sync the supply lines for suppliers that are connected + * Sync the supply lines for organizations that are connected */ export async function syncSupplyLines() { - const spinner = ora(`(NEW) Syncing supply lines...`).start(); + const spinner = ora(`Syncing supply lines...`).start(); try { let headers = { 'Content-Type': 'application/json', @@ -392,24 +318,24 @@ export async function syncSupplyLines() { }; let suppliersWithTradeItem = await models.tradeItem.findAll({ - attributes: ['supplierOrganizationId'], - group: ['supplierOrganizationId'] + attributes: ['organizationId'], + group: ['organizationId'] }); - let connectedSuppliers = await models.supplier.findAll({ - attributes: ['supplierOrganizationId'], + let connectedSuppliers = await models.organization.findAll({ + attributes: ['organizationId'], where: { isConnected: true } }); let suppliers = suppliersWithTradeItem.filter(supplier => { return connectedSuppliers.some(connectedSupplier => { - return connectedSupplier.supplierOrganizationId === supplier.supplierOrganizationId; + return connectedSupplier.organizationId === supplier.organizationId; }); - }).map(supplier => supplier.supplierOrganizationId); + }).map(supplier => supplier.organizationId); let i = 0, x = 1; for (let supplier of suppliers) { - spinner.text = `(NEW) Syncing ${i} supply lines of [${x++}|${suppliers.length}] suppliers...` + spinner.text = `Syncing ${i} supply lines of [${x++}|${suppliers.length}] suppliers...` const params = new URLSearchParams({ supplierOrganizationId: supplier, }).toString(); @@ -427,9 +353,10 @@ export async function syncSupplyLines() { insertItem(tradeItem[0]) } - spinner.text = `(NEW) Syncing ${i++} supply lines of [${x}|${suppliers.length}] suppliers...` + spinner.text = `Syncing ${i++} supply lines of [${x}|${suppliers.length}] suppliers...` await models.supplyLine.upsert({ ...supplyLine, + organizationId: supplyLine.supplierOrganizationId, pricePerPiece_currency: supplyLine.pricePerPiece ? supplyLine.pricePerPiece.currency : null, pricePerPiece_value: supplyLine.pricePerPiece ? supplyLine.pricePerPiece.value : null, deliveryPeriod_startDateTime: supplyLine.deliveryPeriod ? supplyLine.deliveryPeriod.startDateTime : null, @@ -466,14 +393,10 @@ export async function insertItem(tradeItem) { try { tx = await models.sequelize.transaction(); - // Upsert supplier connection - await models.connection.upsert({ - supplierOrganizationId: tradeItem.supplierOrganizationId, - }, { transaction: tx }); - // Upsert trade item await models.tradeItem.upsert({ ...tradeItem, + organizationId: tradeItem.supplierOrganizationId, lastSync: moment(), }, { transaction: tx }); @@ -554,7 +477,7 @@ export async function insertItem(tradeItem) { } /** - * Sync the warehouses + * Sync the warehouses for organizations that are connected **/ export async function syncWarehouses(){ let spinner = ora('Syncing warehouses...').start(); @@ -565,16 +488,18 @@ export async function syncWarehouses(){ 'X-Api-Key': process.env.API_KEY, }; - const suppliers = await models.supplier.findAll(); + const suppliers = await models.organization.findAll({ + where: { isConnected: true } + }); + let x = 0, i = 1; for (let supplier of suppliers) { spinner.text = `Syncing ${i} warehouses of [${x++}|${suppliers.length}]...` - const warehouses = (await vnRequest('GET', `${env.API_URL}/organizations/supplier/${supplier.supplierOrganizationId}/warehouses`, null, headers)).data; + const warehouses = (await vnRequest('GET', `${env.API_URL}/organizations/supplier/${supplier.organizationId}/warehouses`, null, headers)).data; for (let warehouse of warehouses) { - spinner.text = `Syncing ${i++} warehouses of [${x}|${suppliers.length}]...` + spinner.text = `Syncing ${i++} warehouses of [${x}|${suppliers.length}] suppliers...` await models.warehouses.upsert({ ...warehouse, - supplierOrganizationId: warehouse.organizationId, location_gln: warehouse.location.gln, location_address_addressLine: warehouse.location.address.addressLine, location_address_city: warehouse.location.address.city, @@ -585,6 +510,7 @@ export async function syncWarehouses(){ }); } } + spinner.succeed(); } catch (err) { spinner.fail(); @@ -596,7 +522,7 @@ export async function syncWarehouses(){ * Removes Floriday connections that we don't have in the database **/ export async function deleteConnections() { - const spinner = ora(`Deleting connections that aren't in the db...`).start(); + let spinner; try { let i = 1; const headers = { @@ -606,12 +532,15 @@ export async function deleteConnections() { }; const connectionsInFloriday = (await vnRequest('GET', `${env.API_URL}/connections`, null, headers)).data; - const connectionsInDb = await models.connection.findAll(); + const connectionsInDb = await models.organization.findAll({ + attributes: ['organizationId'], + where: { isConnected: true } + }); let isExists = false, ghostConnections = []; for (let connectionInFloriday of connectionsInFloriday) { for (let connectionInDb of connectionsInDb) - if (connectionInFloriday == connectionInDb.supplierOrganizationId) { + if (connectionInFloriday == connectionInDb.organizationId) { isExists = true; break; } @@ -619,13 +548,16 @@ export async function deleteConnections() { isExists = false; } + if (ghostConnections.length) + spinner = ora(`Deleting connections that aren't in the db...`).start(); + for (let connection of ghostConnections) { await vnRequest('DELETE', `${env.API_URL}/connections/${connection}`, null, headers); spinner.text = `Deleting ${i++} of ${ghostConnections.length} that aren't in the db...` } - spinner.succeed(); + if (spinner) spinner.succeed(); } catch (err) { - spinner.fail(); + if (spinner) spinner.fail(); criticalError(err); } }