refactor-syncsuppliers #2

Merged
pau merged 3 commits from refactor-syncsuppliers into master 2023-02-10 09:45:21 +00:00
8 changed files with 355 additions and 111 deletions

View File

@ -17,7 +17,9 @@ let tokenExpirationDate = await vnUtils.getClientToken(models);
process.env.SYNC_SEQUENCE ? await vnUtils.syncSequence() : null;
process.env.SYNC_SUPPLIER ? await vnUtils.syncSuppliers() : null;
//process.env.SYNC_TRADEITEM ? await vnUtils.syncTradeItems() : null;
await vnUtils.syncConnections();
process.env.SYNC_TRADEITEM ? await vnUtils.syncTradeItems() : null;
await vnUtils.syncSupplyLines();
try {
// eslint-disable-next-line no-constant-condition
@ -31,6 +33,7 @@ try {
tokenExpirationDate = await vnUtils.getClientToken(models);
}
} catch (error) {
console.error(error);
}

View File

@ -52,22 +52,30 @@ import characteristics from './tradeItem/characteristics.js';
let models = {
sequelize: sequelize,
tradeItem: tradeItem(sequelize),
packingConfigurations: packingConfigurations(sequelize),
photos: photos(sequelize),
characteristics: characteristics(sequelize),
countryOfOriginIsoCodes: countryOfOriginIsoCodes(sequelize),
packingConfiguration: packingConfigurations(sequelize),
photo: photos(sequelize),
characteristic: characteristics(sequelize),
countryOfOriginIsoCode: countryOfOriginIsoCodes(sequelize),
package: packageModel(sequelize),
seasonalPeriod: seasonalPeriod(sequelize),
clientConfig: clientConfig(sequelize),
botanicalNames: botanicalNames(sequelize),
supplyLines: supplyLine(sequelize),
volumePrices: volumePrices(sequelize),
suppliers: suppliers(sequelize),
botanicalName: botanicalNames(sequelize),
supplyLine: supplyLine(sequelize),
volumePrice: volumePrices(sequelize),
supplier: suppliers(sequelize),
sequenceNumber: sequenceNumber(sequelize),
connections: connections(sequelize),
connection: connections(sequelize),
};
models.characteristics.belongsTo(models.tradeItem, {
/* Remove ID atribute from models */
models.characteristic.removeAttribute('id');
models.seasonalPeriod.removeAttribute('id');
models.package.removeAttribute('id');
models.botanicalName.removeAttribute('id');
models.countryOfOriginIsoCode.removeAttribute('id');
/* ------------------------------ */
models.characteristic.belongsTo(models.tradeItem, {
foreignKey: 'tradeItemFk',
as: 'tradeItem_Fk',
targetKey: 'tradeItemId',
@ -79,68 +87,62 @@ models.seasonalPeriod.belongsTo(models.tradeItem, {
targetKey: 'tradeItemId',
});
models.photos.belongsTo(models.tradeItem, {
models.photo.belongsTo(models.tradeItem, {
foreignKey: 'tradeItemFk',
as: 'tradeItem_Fk',
targetKey: 'tradeItemId',
});
models.photos.hasMany(models.seasonalPeriod, {
foreignKey: 'photoFk',
as: 'seasonalPeriod_Fk',
targetKey: 'photoId',
});
models.seasonalPeriod.belongsTo(models.photos, {
foreignKey: 'photoFk',
as: 'photo_Fk',
targetKey: 'photoId',
});
models.packingConfigurations.belongsTo(models.tradeItem, {
models.packingConfiguration.belongsTo(models.tradeItem, {
foreignKey: 'tradeItemFk',
as: 'tradeItem_Fk',
targetKey: 'tradeItemId',
});
models.packingConfigurations.hasMany(models.package, {
models.packingConfiguration.hasMany(models.package, {
foreignKey: 'packingConfigurationFk',
as: 'package_Fk',
targetKey: 'packingConfigurationId',
});
models.package.belongsTo(models.packingConfigurations, {
models.package.belongsTo(models.packingConfiguration, {
foreignKey: 'packingConfigurationFk',
as: 'packingConfiguration_Fk',
targetKey: 'packingConfigurationId',
});
models.botanicalNames.belongsTo(models.tradeItem, {
models.botanicalName.belongsTo(models.tradeItem, {
foreignKey: 'tradeItemFk',
as: 'tradeItem_Fk',
targetKey: 'tradeItemId',
});
models.countryOfOriginIsoCodes.belongsTo(models.tradeItem, {
models.countryOfOriginIsoCode.belongsTo(models.tradeItem, {
foreignKey: 'tradeItemFk',
as: 'tradeItem_Fk',
targetKey: 'tradeItemId',
});
models.volumePrices.belongsTo(models.supplyLines, {
models.volumePrice.belongsTo(models.supplyLine, {
foreignKey: 'supplyLineFk',
as: 'supplyLine_Fk',
targetKey: 'supplyLineId',
});
models.supplyLines.belongsTo(models.tradeItem, {
models.supplyLine.belongsTo(models.tradeItem, {
foreignKey: 'tradeItemFk',
as: 'tradeItem_Fk',
targetKey: 'tradeItemId',
});
if (process.env.FORCE_SYNC == true) {
models.tradeItem.belongsTo(models.supplier, {
foreignKey: 'supplierOrganizationId',
as: 'supplierOrganization_Id',
targetKey: 'organizationId',
});
if (process.env.FORCE_SYNC === 'true') {
console.log('Forcing the models...');
await sequelize.sync({ force: true });
} else {

View File

@ -1,14 +1,9 @@
import { Sequelize } from 'sequelize';
const supplyLine = {
id: {
type: Sequelize.INTEGER,
primaryKey: true,
autoIncrement: true,
},
supplyLineId: {
type: Sequelize.STRING,
unique: true,
primaryKey: true,
},
status: {
type: Sequelize.STRING,

View File

@ -1,11 +1,6 @@
import { Sequelize } from 'sequelize';
const PackageModel = {
id: {
type: Sequelize.INTEGER,
primaryKey: true,
autoIncrement: true,
},
vbnPackageCode: {
type: Sequelize.INTEGER,
},

View File

@ -1,13 +1,9 @@
import { Sequelize } from 'sequelize';
const packingConfigurations = {
id: {
type: Sequelize.INTEGER,
primaryKey: true,
autoIncrement: true,
},
packingConfigurationId: {
type: Sequelize.STRING,
primaryKey: true,
},
piecesPerPackage: {
type: Sequelize.INTEGER,

View File

@ -1,8 +1,9 @@
import { Sequelize } from 'sequelize';
const photos = {
photoId: {
id: {
type: Sequelize.STRING,
primaryKey: true,
},
url: {
type: Sequelize.STRING,
@ -13,9 +14,6 @@ const photos = {
primary: {
type: Sequelize.BOOLEAN,
},
seasonalPeriodFk: {
type: Sequelize.INTEGER,
},
};
export default (sequelize) => {

View File

@ -1,21 +1,13 @@
import { Sequelize } from 'sequelize';
const tradeItem = {
id: {
type: Sequelize.INTEGER,
primaryKey: true,
autoIncrement: true,
},
tradeItemId: {
type: Sequelize.STRING,
unique: true,
primaryKey: true,
},
supplierOrganizationId: {
type: Sequelize.STRING
},
supplierOrganizationUuid: {
type: Sequelize.STRING
},
code: {
type: Sequelize.STRING
},

363
utils.js
View File

@ -2,7 +2,7 @@ import moment from 'moment';
import fetch from 'node-fetch';
import dotenv from 'dotenv';
import { models } from './models/index.js';
//import { v4 as uuidv4 } from 'uuid';
import { v4 as uuidv4 } from 'uuid';
//import cliProgress from 'cli-progress';
import suppliersGln from './suppliersGln.js';
dotenv.config();
@ -223,11 +223,18 @@ async function syncSuppliers(){
'X-Api-Key': process.env.API_KEY
};
let maximumSequenceNumber = 500;
let maximumSequenceNumber = await fetch(`${BASE_CUSTOMER_URL}organizations/current-max-sequence`, {
method: 'GET',
headers: headers
});
maximumSequenceNumber = await maximumSequenceNumber.json();
console.log('Maximum sequence number: ', maximumSequenceNumber);
for (let i = 0; i < maximumSequenceNumber; i++) {
let query = `${BASE_CUSTOMER_URL}organizations/sync/${i}?organizationType=SUPPLIER&limit=500`;
let query = `${BASE_CUSTOMER_URL}organizations/sync/${i}?organizationType=SUPPLIER&limit=1000`;
let response = await fetch(query, {
method: 'GET',
headers: headers
@ -235,12 +242,11 @@ async function syncSuppliers(){
let data = await response.json();
maximumSequenceNumber = data.maximumSequenceNumber;
let suppliers = data.results;
for (let supplier of suppliers) {
i = supplier.sequenceNumber;
await models.suppliers.upsert({
await models.supplier.upsert({
isConnected: false,
commercialName: supplier.commercialName,
email: supplier.email,
@ -261,77 +267,334 @@ async function syncSuppliers(){
console.log('INSERTED:\t', supplier.commercialName, '\nsequenceNumber:\t', supplier.sequenceNumber);
}
await syncSequence(i, 'suppliers', maximumSequenceNumber);
console.log(data.maximumSequenceNumber);
console.log(data.results.length);
console.log(i);
}
}
async function syncTradeItems(){
async function syncConnections(){
let connections = await models.connection.findAll();
let suppliers = await models.suppliers.findAll({
where: {
isConnected: true
}
});
let headers = {
'Content-Type': 'application/json',
'Authorization': `Bearer ${await getJWT()}`,
'X-Api-Key': process.env.API_KEY
};
for (let i = 0; i < suppliers.length; i++) {
let remoteConnections = await fetch(`${BASE_CUSTOMER_URL}connections`, {
method: 'GET',
headers: headers
});
remoteConnections = await remoteConnections.json();
for (let connection of connections){
if (connection.isConnected == false){
continue;
}
let remoteConnection = remoteConnections.find(remoteConnection => remoteConnection == connection.organizationId);
if (remoteConnection == undefined){
console.log('Connection: ', connection, 'does not exist in the remote server');
console.log('Creating remote connection');
await fetch(`${BASE_CUSTOMER_URL}connections/${connection.organizationId}`, {
method: 'PUT',
headers: headers
});
await models.connection.update({ isConnected: true }, {
where: {
organizationId: connection.organizationId
}
});
await models.supplier.update({ isConnected: true }, {
where: {
organizationId: connection.organizationId
}
});
} else {
console.log('Connection: ', connection, 'exists in the remote server');
await models.connection.update({ isConnected: true }, {
where: {
organizationId: connection.organizationId
}
});
await models.supplier.update({ isConnected: true }, {
where: {
organizationId: connection.organizationId
}
});
}
}
let queryMaxSequence = `${BASE_CUSTOMER_URL}trade-items/current-max-sequence`;
let responseMaxSequence = await fetch(queryMaxSequence, {
method: 'GET',
headers: headers
});
}
let maximumSequenceNumber = await responseMaxSequence.json();
async function syncTradeItems(){
await syncSequence(0, 'tradeItems', maximumSequenceNumber);
let currentSequence = await models.sequenceNumber.findOne({
where: {
model: 'tradeItems'
}
});
const suppliers = await models.supplier.findAll();
currentSequence = currentSequence.sequenceNumber;
let headers = {
'Content-Type': 'application/json',
'Authorization': `Bearer ${await getJWT()}`,
'X-Api-Key': process.env.API_KEY
};
let estimatedIterations = Math.ceil(maximumSequenceNumber / 1000);
let supplier = suppliers[i];
let i = 0;
console.log('Syncing trade items for: ', supplier.name);
console.log('Supplier Id: ', supplier.organizationId);
console.log('Current sequence number: ', currentSequence);
console.log('Maximum sequence number: ', maximumSequenceNumber);
console.log('Estimated iterations: ', estimatedIterations);
console.log('Syncing trade items');
for (let supplier of suppliers) {
i++;
if (!supplier.isConnected){
console.log('Supplier: ', supplier.commercialName, 'is not connected');
console.log('Skipping supplier', supplier.commercialName, '(', i, '/', suppliers.length, ')');
continue;
}
let query = `${BASE_CUSTOMER_URL}trade-items?supplierOrganizationId=${supplier.organizationId}`;
try {
for (let j = 0; j < estimatedIterations; j++) {
let query = `${BASE_CUSTOMER_URL}trade-items/sync/${currentSequence}?supplierOrganizationId=${supplier.organizationId}&limit=1000`;
let request = await fetch(query, {
method: 'GET',
headers: headers
});
let itemdata = await request.json();
let tradeItems = await request.json();
let results = itemdata.results;
if (results.length == 0) {
console.log('No more trade items to sync');
break;
if (tradeItems.length == 0) {
console.log('No trade items for supplier: ', supplier.commercialName);
continue;
}
for (let tradeItem of tradeItems) {
let tx = await models.sequelize.transaction();
console.log('Syncing trade item: ', tradeItem.name);
try {
await models.tradeItem.upsert({
tradeItemId: tradeItem.tradeItemId,
supplierOrganizationId: tradeItem.supplierOrganizationId,
code: tradeItem.code,
gtin: tradeItem.gtin,
vbnProductCode: tradeItem.vbnProductCode,
name: tradeItem.name,
isDeleted: tradeItem.isDeleted,
sequenceNumber: tradeItem.sequenceNumber,
tradeItemVersion: tradeItem.tradeItemVersion,
isCustomerSpecific: tradeItem.isCustomerSpecific,
isHiddenInCatalog: tradeItem.isHiddenInCatalog,
},
{transaction: tx});
let characteristics = tradeItem.characteristics;
for (let characteristic of characteristics) {
await models.characteristic.upsert({
vbnCode: characteristic.vbnCode,
vbnValueCode: characteristic.vbnValueCode,
tradeItemFk: tradeItem.tradeItemId,
}, {transaction: tx});
}
let seasonalPeriods = tradeItem.seasonalPeriods;
for (let seasonalPeriod of seasonalPeriods) {
await models.seasonalPeriod.upsert({
startWeek: seasonalPeriod.startWeek,
endWeek: seasonalPeriod.endWeek,
tradeItemFk: tradeItem.tradeItemId,
}, {transaction: tx});
}
let photos = tradeItem.photos;
for (let photo of photos) {
await models.photo.upsert({
id: photo.id,
url: photo.url,
type: photo.type,
primary: photo.primary,
tradeItemFk: tradeItem.tradeItemId,
}, {transaction: tx});
}
let packingConfigurations = tradeItem.packingConfigurations;
for (let packingConfiguration of packingConfigurations) {
let uuid = uuidv4();
await models.packingConfiguration.upsert({
packingConfigurationId: uuid,
piecesPerPackage: packingConfiguration.piecesPerPackage,
bunchesPerPackage: packingConfiguration.bunchesPerPackage,
photoUrl: packingConfiguration.photoUrl,
packagesPerLayer: packingConfiguration.packagesPerLayer,
layersPerLoadCarrier: packingConfiguration.layersPerLoadCarrier,
additionalPricePerPiece : JSON.stringify(packingConfiguration.additionalPricePerPiece),
transportHeightInCm: packingConfiguration.transportHeightInCm,
loadCarrierType: packingConfiguration.loadCarrierType,
isPrimary: packingConfiguration.isPrimary,
tradeItemFk: tradeItem.tradeItemId,
}, {transaction: tx});
await models.package.upsert({
vbnPackageCode: packingConfiguration.package.vbnPackageCode,
customPackageId: packingConfiguration.package.customPackageId,
packingConfigurationFk: uuid,
}, {transaction: tx});
}
let countryOfOriginIsoCodes = tradeItem.countryOfOriginIsoCodes;
countryOfOriginIsoCodes ??= 0;
for (let countryOfOriginIsoCode of countryOfOriginIsoCodes) {
await models.countryOfOriginIsoCode.upsert({
isoCode: countryOfOriginIsoCode,
tradeItemFk: tradeItem.tradeItemId,
}, {transaction: tx});
}
let botanicalNames = tradeItem.botanicalNames;
for (let botanicalName of botanicalNames) {
await models.botanicalName.upsert({
name: botanicalName.name,
tradeItemFk: tradeItem.tradeItemId,
}, {transaction: tx});
}
await tx.commit();
} catch (error) {
await tx.rollback();
console.log('Error while syncing trade items for: ', supplier.commercialName);
console.log(error);
}
}
console.log('Synced trade items for: ', supplier.commercialName);
console.log('Remaining suppliers: ', suppliers.length - i, 'of', suppliers.length);
console.log('Total trade items: ', tradeItems.length);
} catch (error) {
console.log('Error while syncing trade items for: ', supplier.commercialName);
console.log(error);
}
}
}
export { getClientToken, updateClientConfig, getJWT, sleep, asyncQueue, syncSequence, syncSuppliers, syncTradeItems };
async function syncSupplyLines(){
console.log('Syncing supply lines');
let suppliers = await models.supplier.findAll(/*{
where: {
isConnected: true
}
}*/);
console.log('Found', suppliers.length, 'connected suppliers');
let promises = [];
// Launch a promise for each supplier
for (let supplier of suppliers) {
let headers = {
'Content-Type': 'application/json',
'Authorization': `Bearer ${await getJWT()}`,
'X-Api-Key': process.env.API_KEY
};
console.log('Requesting supply lines for CATALOG');
// eslint-disable-next-line no-async-promise-executor
let promise = new Promise(async (resolve, reject) => {
let queryCATALOG = `${BASE_CUSTOMER_URL}supply-lines?supplierOrganizationId=${supplier.organizationId}&supplyType=CATALOG`;
let queryBATCH = `${BASE_CUSTOMER_URL}supply-lines?supplierOrganizationId=${supplier.organizationId}&supplyType=BATCH`;
try {
let requestCATALOG = await fetch(queryCATALOG, {
method: 'GET',
headers: headers
});
let requestBATCH = await fetch(queryBATCH, {
method: 'GET',
headers: headers
});
let supplyLinesCATALOG = await requestCATALOG.json();
let supplyLinesBATCH = await requestBATCH.json();
if (supplyLinesBATCH.length == 0) {
console.log('No supply lines for BATCH on: ', supplier.commercialName);
}
if (supplyLinesCATALOG.length == 0) {
console.log('No supply lines for CATALOG on:', supplier.commercialName);
}
let supplyLines = supplyLinesCATALOG.concat(supplyLinesBATCH);
if (supplyLines.length > 0) {
console.log('Syncing supply lines for: ', supplier.commercialName);
console.log('Found', supplyLines.length, 'supply lines');
}
resolve(supplyLines);
} catch (error) {
console.log('Error while syncing supply lines for: ', supplier.commercialName);
reject(error);
}
});
promises.push(promise);
}
let supplyLines = await Promise.all(promises);
for (let supplyLine of supplyLines) {
for (let line of supplyLine) {
let tradeItem = await models.tradeItem.findOne({
where: {
tradeItemId: line.tradeItemId
}
});
if (!tradeItem) {
console.log('Trade item not found for supply line: ', line.supplyLineId);
console.log('Trade item id: ', line.tradeItemId);
continue;
}
await models.supplyLine.upsert({
supplyLineId: line.supplyLineId,
status: line.status,
supplierOrganizationId: line.supplierOrganizationId,
pricePerPiece: line.pricePerPiece,
numberOfPieces: line.numberOfPieces,
deliveryPeriod: line.deliveryPeriod,
orderPeriod: line.orderPeriod,
wharehouseId: line.wharehouseId,
sequenceNumber: line.sequenceNumber,
type: line.type,
isDeleted: line.isDeleted,
salesUnit: line.salesUnit,
agreementReferenceCode: line.agreementReference.code,
agreementReferenceDescription: line.agreementReference.description,
isLimited: line.isLimited,
isCustomerSpecific: line.isCustomerSpecific,
tradeItemFk: line.tradeItemId,
});
}
}
console.log('Synced supply lines');
}
export { getClientToken, updateClientConfig, getJWT, sleep, asyncQueue, syncSequence, syncSuppliers, syncTradeItems, syncConnections, syncSupplyLines };