From 771153b556ea620c8b10661d3d6a50477ad1c763 Mon Sep 17 00:00:00 2001 From: Pau Navarro Date: Wed, 8 Feb 2023 12:31:54 +0100 Subject: [PATCH] refactor syncTradeItem --- index.js | 2 +- models/index.js | 64 +++---- models/tradeItem/package.js | 5 - models/tradeItem/packingConfigurations.js | 6 +- models/tradeItem/photos.js | 6 +- models/tradeItem/tradeItem.js | 10 +- utils.js | 193 ++++++++++++++++------ 7 files changed, 179 insertions(+), 107 deletions(-) diff --git a/index.js b/index.js index 1fdd4ca..4e5cc86 100644 --- a/index.js +++ b/index.js @@ -17,7 +17,7 @@ let tokenExpirationDate = await vnUtils.getClientToken(models); process.env.SYNC_SEQUENCE ? await vnUtils.syncSequence() : null; process.env.SYNC_SUPPLIER ? await vnUtils.syncSuppliers() : null; -//process.env.SYNC_TRADEITEM ? await vnUtils.syncTradeItems() : null; +process.env.SYNC_TRADEITEM ? await vnUtils.syncTradeItems() : null; try { // eslint-disable-next-line no-constant-condition diff --git a/models/index.js b/models/index.js index 32ce29b..fd4f5a3 100644 --- a/models/index.js +++ b/models/index.js @@ -52,22 +52,30 @@ import characteristics from './tradeItem/characteristics.js'; let models = { sequelize: sequelize, tradeItem: tradeItem(sequelize), - packingConfigurations: packingConfigurations(sequelize), - photos: photos(sequelize), - characteristics: characteristics(sequelize), - countryOfOriginIsoCodes: countryOfOriginIsoCodes(sequelize), + packingConfiguration: packingConfigurations(sequelize), + photo: photos(sequelize), + characteristic: characteristics(sequelize), + countryOfOriginIsoCode: countryOfOriginIsoCodes(sequelize), package: packageModel(sequelize), seasonalPeriod: seasonalPeriod(sequelize), clientConfig: clientConfig(sequelize), - botanicalNames: botanicalNames(sequelize), - supplyLines: supplyLine(sequelize), - volumePrices: volumePrices(sequelize), - suppliers: suppliers(sequelize), + botanicalName: botanicalNames(sequelize), + supplyLine: supplyLine(sequelize), + volumePrice: volumePrices(sequelize), + supplier: suppliers(sequelize), sequenceNumber: sequenceNumber(sequelize), - connections: connections(sequelize), + connection: connections(sequelize), }; -models.characteristics.belongsTo(models.tradeItem, { +/* Remove ID atribute from models */ +models.characteristic.removeAttribute('id'); +models.seasonalPeriod.removeAttribute('id'); +models.package.removeAttribute('id'); +models.botanicalName.removeAttribute('id'); +models.countryOfOriginIsoCode.removeAttribute('id'); +/* ------------------------------ */ + +models.characteristic.belongsTo(models.tradeItem, { foreignKey: 'tradeItemFk', as: 'tradeItem_Fk', targetKey: 'tradeItemId', @@ -79,68 +87,62 @@ models.seasonalPeriod.belongsTo(models.tradeItem, { targetKey: 'tradeItemId', }); -models.photos.belongsTo(models.tradeItem, { +models.photo.belongsTo(models.tradeItem, { foreignKey: 'tradeItemFk', as: 'tradeItem_Fk', targetKey: 'tradeItemId', }); -models.photos.hasMany(models.seasonalPeriod, { - foreignKey: 'photoFk', - as: 'seasonalPeriod_Fk', - targetKey: 'photoId', -}); -models.seasonalPeriod.belongsTo(models.photos, { - foreignKey: 'photoFk', - as: 'photo_Fk', - targetKey: 'photoId', -}); - - -models.packingConfigurations.belongsTo(models.tradeItem, { +models.packingConfiguration.belongsTo(models.tradeItem, { foreignKey: 'tradeItemFk', as: 'tradeItem_Fk', targetKey: 'tradeItemId', }); -models.packingConfigurations.hasMany(models.package, { +models.packingConfiguration.hasMany(models.package, { foreignKey: 'packingConfigurationFk', as: 'package_Fk', targetKey: 'packingConfigurationId', }); -models.package.belongsTo(models.packingConfigurations, { +models.package.belongsTo(models.packingConfiguration, { foreignKey: 'packingConfigurationFk', as: 'packingConfiguration_Fk', targetKey: 'packingConfigurationId', }); -models.botanicalNames.belongsTo(models.tradeItem, { +models.botanicalName.belongsTo(models.tradeItem, { foreignKey: 'tradeItemFk', as: 'tradeItem_Fk', targetKey: 'tradeItemId', }); -models.countryOfOriginIsoCodes.belongsTo(models.tradeItem, { +models.countryOfOriginIsoCode.belongsTo(models.tradeItem, { foreignKey: 'tradeItemFk', as: 'tradeItem_Fk', targetKey: 'tradeItemId', }); -models.volumePrices.belongsTo(models.supplyLines, { +models.volumePrice.belongsTo(models.supplyLine, { foreignKey: 'supplyLineFk', as: 'supplyLine_Fk', targetKey: 'supplyLineId', }); -models.supplyLines.belongsTo(models.tradeItem, { +models.supplyLine.belongsTo(models.tradeItem, { foreignKey: 'tradeItemFk', as: 'tradeItem_Fk', targetKey: 'tradeItemId', }); -if (process.env.FORCE_SYNC == true) { +models.tradeItem.belongsTo(models.supplier, { + foreignKey: 'supplierOrganizationId', + as: 'supplierOrganization_Id', + targetKey: 'organizationId', +}); + +if (process.env.FORCE_SYNC === 'true') { console.log('Forcing the models...'); await sequelize.sync({ force: true }); } else { diff --git a/models/tradeItem/package.js b/models/tradeItem/package.js index 1e45d52..b629a4c 100644 --- a/models/tradeItem/package.js +++ b/models/tradeItem/package.js @@ -1,11 +1,6 @@ import { Sequelize } from 'sequelize'; const PackageModel = { - id: { - type: Sequelize.INTEGER, - primaryKey: true, - autoIncrement: true, - }, vbnPackageCode: { type: Sequelize.INTEGER, }, diff --git a/models/tradeItem/packingConfigurations.js b/models/tradeItem/packingConfigurations.js index fc7e42e..3a92f55 100644 --- a/models/tradeItem/packingConfigurations.js +++ b/models/tradeItem/packingConfigurations.js @@ -1,13 +1,9 @@ import { Sequelize } from 'sequelize'; const packingConfigurations = { - id: { - type: Sequelize.INTEGER, - primaryKey: true, - autoIncrement: true, - }, packingConfigurationId: { type: Sequelize.STRING, + primaryKey: true, }, piecesPerPackage: { type: Sequelize.INTEGER, diff --git a/models/tradeItem/photos.js b/models/tradeItem/photos.js index dba34a9..0958fec 100644 --- a/models/tradeItem/photos.js +++ b/models/tradeItem/photos.js @@ -1,8 +1,9 @@ import { Sequelize } from 'sequelize'; const photos = { - photoId: { + id: { type: Sequelize.STRING, + primaryKey: true, }, url: { type: Sequelize.STRING, @@ -13,9 +14,6 @@ const photos = { primary: { type: Sequelize.BOOLEAN, }, - seasonalPeriodFk: { - type: Sequelize.INTEGER, - }, }; export default (sequelize) => { diff --git a/models/tradeItem/tradeItem.js b/models/tradeItem/tradeItem.js index a3d1ea5..d7f229f 100644 --- a/models/tradeItem/tradeItem.js +++ b/models/tradeItem/tradeItem.js @@ -1,21 +1,13 @@ import { Sequelize } from 'sequelize'; const tradeItem = { - id: { - type: Sequelize.INTEGER, - primaryKey: true, - autoIncrement: true, - }, tradeItemId: { type: Sequelize.STRING, - unique: true, + primaryKey: true, }, supplierOrganizationId: { type: Sequelize.STRING }, - supplierOrganizationUuid: { - type: Sequelize.STRING - }, code: { type: Sequelize.STRING }, diff --git a/utils.js b/utils.js index 76f609c..68186f0 100644 --- a/utils.js +++ b/utils.js @@ -2,7 +2,7 @@ import moment from 'moment'; import fetch from 'node-fetch'; import dotenv from 'dotenv'; import { models } from './models/index.js'; -//import { v4 as uuidv4 } from 'uuid'; +import { v4 as uuidv4 } from 'uuid'; //import cliProgress from 'cli-progress'; import suppliersGln from './suppliersGln.js'; dotenv.config(); @@ -223,11 +223,18 @@ async function syncSuppliers(){ 'X-Api-Key': process.env.API_KEY }; - let maximumSequenceNumber = 500; + let maximumSequenceNumber = await fetch(`${BASE_CUSTOMER_URL}organizations/current-max-sequence`, { + method: 'GET', + headers: headers + }); + + maximumSequenceNumber = await maximumSequenceNumber.json(); + + console.log('Maximum sequence number: ', maximumSequenceNumber); for (let i = 0; i < maximumSequenceNumber; i++) { - let query = `${BASE_CUSTOMER_URL}organizations/sync/${i}?organizationType=SUPPLIER&limit=500`; + let query = `${BASE_CUSTOMER_URL}organizations/sync/${i}?organizationType=SUPPLIER&limit=1000`; let response = await fetch(query, { method: 'GET', headers: headers @@ -235,12 +242,11 @@ async function syncSuppliers(){ let data = await response.json(); - maximumSequenceNumber = data.maximumSequenceNumber; let suppliers = data.results; for (let supplier of suppliers) { i = supplier.sequenceNumber; - await models.suppliers.upsert({ + await models.supplier.upsert({ isConnected: false, commercialName: supplier.commercialName, email: supplier.email, @@ -261,76 +267,159 @@ async function syncSuppliers(){ console.log('INSERTED:\t', supplier.commercialName, '\nsequenceNumber:\t', supplier.sequenceNumber); } await syncSequence(i, 'suppliers', maximumSequenceNumber); - console.log(data.maximumSequenceNumber); - console.log(data.results.length); - console.log(i); } } async function syncTradeItems(){ - let suppliers = await models.suppliers.findAll({ - where: { - isConnected: true - } - }); - + const suppliers = await models.supplier.findAll(); + let headers = { 'Content-Type': 'application/json', 'Authorization': `Bearer ${await getJWT()}`, 'X-Api-Key': process.env.API_KEY }; - for (let i = 0; i < suppliers.length; i++) { - - let queryMaxSequence = `${BASE_CUSTOMER_URL}trade-items/current-max-sequence`; - - let responseMaxSequence = await fetch(queryMaxSequence, { - method: 'GET', - headers: headers - }); + let i = 0; - let maximumSequenceNumber = await responseMaxSequence.json(); + console.log('Syncing trade items'); + for (let supplier of suppliers) { + i++; + let query = `${BASE_CUSTOMER_URL}trade-items?supplierOrganizationId=${supplier.organizationId}`; - await syncSequence(0, 'tradeItems', maximumSequenceNumber); - - let currentSequence = await models.sequenceNumber.findOne({ - where: { - model: 'tradeItems' - } - }); + try { - currentSequence = currentSequence.sequenceNumber; - - let estimatedIterations = Math.ceil(maximumSequenceNumber / 1000); - - let supplier = suppliers[i]; - - console.log('Syncing trade items for: ', supplier.name); - console.log('Supplier Id: ', supplier.organizationId); - console.log('Current sequence number: ', currentSequence); - console.log('Maximum sequence number: ', maximumSequenceNumber); - console.log('Estimated iterations: ', estimatedIterations); - - for (let j = 0; j < estimatedIterations; j++) { - - let query = `${BASE_CUSTOMER_URL}trade-items/sync/${currentSequence}?supplierOrganizationId=${supplier.organizationId}&limit=1000`; - let request = await fetch(query, { method: 'GET', headers: headers }); - let itemdata = await request.json(); + let tradeItems = await request.json(); - let results = itemdata.results; - - if (results.length == 0) { - console.log('No more trade items to sync'); - break; + if (tradeItems.length == 0) { + console.log('No trade items for supplier: ', supplier.commercialName); + continue; } + for (let tradeItem of tradeItems) { + + let tx = await models.sequelize.transaction(); + + console.log('Syncing trade item: ', tradeItem.name); + + try { + + await models.tradeItem.upsert({ + tradeItemId: tradeItem.tradeItemId, + supplierOrganizationId: tradeItem.supplierOrganizationId, + code: tradeItem.code, + gtin: tradeItem.gtin, + vbnProductCode: tradeItem.vbnProductCode, + name: tradeItem.name, + isDeleted: tradeItem.isDeleted, + sequenceNumber: tradeItem.sequenceNumber, + tradeItemVersion: tradeItem.tradeItemVersion, + isCustomerSpecific: tradeItem.isCustomerSpecific, + isHiddenInCatalog: tradeItem.isHiddenInCatalog, + }, + {transaction: tx}); + + + let characteristics = tradeItem.characteristics; + + for (let characteristic of characteristics) { + await models.characteristic.upsert({ + vbnCode: characteristic.vbnCode, + vbnValueCode: characteristic.vbnValueCode, + tradeItemFk: tradeItem.tradeItemId, + }, {transaction: tx}); + } + + let seasonalPeriods = tradeItem.seasonalPeriods; + + for (let seasonalPeriod of seasonalPeriods) { + await models.seasonalPeriod.upsert({ + startWeek: seasonalPeriod.startWeek, + endWeek: seasonalPeriod.endWeek, + tradeItemFk: tradeItem.tradeItemId, + }, {transaction: tx}); + } + + let photos = tradeItem.photos; + + for (let photo of photos) { + await models.photo.upsert({ + id: photo.id, + url: photo.url, + type: photo.type, + primary: photo.primary, + tradeItemFk: tradeItem.tradeItemId, + }, {transaction: tx}); + } + + let packingConfigurations = tradeItem.packingConfigurations; + + for (let packingConfiguration of packingConfigurations) { + + let uuid = uuidv4(); + + await models.packingConfiguration.upsert({ + packingConfigurationId: uuid, + piecesPerPackage: packingConfiguration.piecesPerPackage, + bunchesPerPackage: packingConfiguration.bunchesPerPackage, + photoUrl: packingConfiguration.photoUrl, + packagesPerLayer: packingConfiguration.packagesPerLayer, + layersPerLoadCarrier: packingConfiguration.layersPerLoadCarrier, + additionalPricePerPiece : JSON.stringify(packingConfiguration.additionalPricePerPiece), + transportHeightInCm: packingConfiguration.transportHeightInCm, + loadCarrierType: packingConfiguration.loadCarrierType, + isPrimary: packingConfiguration.isPrimary, + tradeItemFk: tradeItem.tradeItemId, + }, {transaction: tx}); + await models.package.upsert({ + vbnPackageCode: packingConfiguration.package.vbnPackageCode, + customPackageId: packingConfiguration.package.customPackageId, + packingConfigurationFk: uuid, + }, {transaction: tx}); + } + + let countryOfOriginIsoCodes = tradeItem.countryOfOriginIsoCodes; + + countryOfOriginIsoCodes ??= 0; + + for (let countryOfOriginIsoCode of countryOfOriginIsoCodes) { + await models.countryOfOriginIsoCode.upsert({ + isoCode: countryOfOriginIsoCode, + tradeItemFk: tradeItem.tradeItemId, + }, {transaction: tx}); + } + + let botanicalNames = tradeItem.botanicalNames; + + for (let botanicalName of botanicalNames) { + await models.botanicalName.upsert({ + name: botanicalName.name, + tradeItemFk: tradeItem.tradeItemId, + }, {transaction: tx}); + } + + await tx.commit(); + + } catch (error) { + await tx.rollback(); + console.log('Error while syncing trade items for: ', supplier.commercialName); + console.log(error); + } + } + + console.log('Synced trade items for: ', supplier.commercialName); + console.log('Remaining suppliers: ', suppliers.length - i, 'of', suppliers.length); + console.log('Total trade items: ', tradeItems.length); + } catch (error) { + console.log('Error while syncing trade items for: ', supplier.commercialName); + console.log(error); } + } }