From 6c8230d1a09a2944f13812b20a703b3424308b63 Mon Sep 17 00:00:00 2001 From: Pau Navarro Date: Mon, 16 Jan 2023 14:52:08 +0100 Subject: [PATCH] Stock is now being obtained --- README.md | 2 +- index.js | 33 ++++++-- models/clientConfig.js | 2 +- models/index.js | 21 ++++- models/suppliers.js | 24 ++++++ models/supplyLine.js | 68 ++++++++++++++++ models/tradeItem.js | 3 + models/volumePrices.js | 23 ++++++ utils.js | 181 ++++++++++++++++++++++++++++++++++++++--- 9 files changed, 334 insertions(+), 23 deletions(-) create mode 100644 models/suppliers.js create mode 100644 models/supplyLine.js create mode 100644 models/volumePrices.js diff --git a/README.md b/README.md index 29b9c81..583b586 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Floriday -Requires [Node.js](https://nodejs.org/en/) v14.x.x or higher. +Requires [Node.js](https://nodejs.org/en/) v15.14.0 or higher. The Floriday service project should perform the following tasks: diff --git a/index.js b/index.js index 2771eb6..6987fb1 100644 --- a/index.js +++ b/index.js @@ -15,36 +15,51 @@ try { if (process.env.QUERYSUPPLIERS) { + process.env.HowManySuppliers ??= suppliers.suppliers.length; + console.log('Querying suppliers...'); - console.log(suppliers.suppliers.length + ' suppliers found'); + console.log(process.env.HowManySuppliers + ' suppliers will be queried.'); const bar1 = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); - bar1.start(suppliers.suppliers.length, 0); + bar1.start(process.env.HowManySuppliers, 0); + + let functionQueue = []; // vnUtils.getTradeitems(suppliers.suppliers[i].SupplierGLN) - for (let i = 0; i < suppliers.suppliers.length; i++) { - await vnUtils.getTradeitems(suppliers.suppliers[i].SupplierGLN).then(() => { + for (let i = 0; i < process.env.HowManySuppliers; i++) { + functionQueue.push(async () => { + await vnUtils.getTradeitems(suppliers.suppliers[i].SupplierGLN); bar1.increment(); }); } + await vnUtils.asyncQueue(functionQueue, 10); + bar1.stop(); + + console.log('Done querying trade items.'); } - setInterval(async () => { + // eslint-disable-next-line no-constant-condition + while (true) { + console.log('Querying the API to check for new data...'); - - - console.log('Current token expiration date: ', tokenExpirationDate); + await vnUtils.getStock(); + if (moment().isAfter(tokenExpirationDate)) { console.log('Token expired, getting a new one...'); tokenExpirationDate = await vnUtils.getClientToken(models); } + if (process.env.STATUS == 'development') { + await vnUtils.sleep(15000); + } else { + await vnUtils.sleep(30000); + } + } - }, process.env.STATUS == 'development' ? 15000 : 30000); } catch (error) { console.error('Unable to connect to the database:', error); } diff --git a/models/clientConfig.js b/models/clientConfig.js index 7c8510c..bc642c9 100644 --- a/models/clientConfig.js +++ b/models/clientConfig.js @@ -13,7 +13,7 @@ const clientConfig = { type: Sequelize.STRING, }, currentToken: { - type: Sequelize.STRING(5000), + type: Sequelize.STRING(2000), }, tokenExpiration: { type: Sequelize.STRING, diff --git a/models/index.js b/models/index.js index 72e27a1..6961b4b 100644 --- a/models/index.js +++ b/models/index.js @@ -1,6 +1,5 @@ import { Sequelize } from 'sequelize'; import dotenv from 'dotenv'; -import {sleep} from '../utils.js'; dotenv.config(); let sequelize = createConnection(); @@ -14,6 +13,9 @@ import seasonalPeriod from './seasonalPeriod.js'; import tradeItem from './tradeItem.js'; import clientConfig from './clientConfig.js'; import characteristics from './characteristics.js'; +import supplyLine from './supplyLine.js'; +import volumePrices from './volumePrices.js'; +import suppliers from './suppliers.js'; /** * Contains all the models that are related to the application. @@ -45,6 +47,9 @@ let models = { seasonalPeriod: seasonalPeriod(sequelize), clientConfig: clientConfig(sequelize), botanicalNames: botanicalNames(sequelize), + supplyLines: supplyLine(sequelize), + volumePrices: volumePrices(sequelize), + suppliers: suppliers(sequelize), }; models.characteristics.belongsTo(models.tradeItem, { @@ -108,6 +113,18 @@ models.countryOfOriginIsoCodes.belongsTo(models.tradeItem, { targetKey: 'tradeItemId', }); +models.volumePrices.belongsTo(models.supplyLines, { + foreignKey: 'supplyLineFk', + as: 'supplyLine_Fk', + targetKey: 'supplyLineId', +}); + +models.supplyLines.belongsTo(models.tradeItem, { + foreignKey: 'tradeItemFk', + as: 'tradeItem_Fk', + targetKey: 'tradeItemId', +}); + if (process.env.FORCE_SYNC) { console.log('Syncing the models...'); await sequelize.sync({ force: true }); @@ -135,7 +152,7 @@ function createConnection() { dialect: process.env.DB_DIALECT, logging: false, pool: { - max: 20, + max: 40, min: 0, acquire: 60000, idle: 10000, diff --git a/models/suppliers.js b/models/suppliers.js new file mode 100644 index 0000000..7414170 --- /dev/null +++ b/models/suppliers.js @@ -0,0 +1,24 @@ +import { Sequelize } from 'sequelize'; + +const suppliers = { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true, + }, + supplierId: { + type: Sequelize.STRING, + unique: true, + }, + supplierGln: { + type: Sequelize.STRING, + }, +}; + +export default (sequelize) => { + const Suppliers = sequelize.define('FDsuppliers', suppliers, { + timestamps: false, + freezeTableName: true, + }); + return Suppliers; +}; \ No newline at end of file diff --git a/models/supplyLine.js b/models/supplyLine.js new file mode 100644 index 0000000..02c5cb0 --- /dev/null +++ b/models/supplyLine.js @@ -0,0 +1,68 @@ +import { Sequelize } from 'sequelize'; + +const supplyLine = { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true, + }, + supplyLineId: { + type: Sequelize.STRING, + unique: true, + }, + status: { + type: Sequelize.STRING, + }, + supplierOrganizationId: { + type: Sequelize.STRING, + }, + pricePerPiece: { + type: Sequelize.JSON, + }, + numberOfPieces : { + type: Sequelize.INTEGER, + }, + deliveryPeriod: { + type: Sequelize.JSON, + }, + orderPeriod: { + type: Sequelize.JSON, + }, + wharehouseId: { + type: Sequelize.STRING, + }, + sequenceNumber: { + type: Sequelize.INTEGER, + }, + type: { + type: Sequelize.STRING, + }, + isDeleted: { + type: Sequelize.BOOLEAN, + }, + salesUnit: { + type: Sequelize.STRING, + }, + agreementReferenceCode: { + type: Sequelize.STRING, + }, + agreementReferenceDescription: { + type: Sequelize.STRING, + }, + isLimited: { + type: Sequelize.BOOLEAN, + }, + isCustomerSpecific: { + type: Sequelize.BOOLEAN, + } +}; + + + +export default (sequelize) => { + const SupplyLine = sequelize.define('FDsupplyLine', supplyLine, { + timestamps: false, + freezeTableName: true, + }); + return SupplyLine; +}; \ No newline at end of file diff --git a/models/tradeItem.js b/models/tradeItem.js index 1312609..a3d1ea5 100644 --- a/models/tradeItem.js +++ b/models/tradeItem.js @@ -13,6 +13,9 @@ const tradeItem = { supplierOrganizationId: { type: Sequelize.STRING }, + supplierOrganizationUuid: { + type: Sequelize.STRING + }, code: { type: Sequelize.STRING }, diff --git a/models/volumePrices.js b/models/volumePrices.js new file mode 100644 index 0000000..2462aa0 --- /dev/null +++ b/models/volumePrices.js @@ -0,0 +1,23 @@ +import { Sequelize } from 'sequelize'; + +const volumePrices = { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true, + }, + unit: { + type: Sequelize.STRING, + }, + pricePerPiece: { + type: Sequelize.INTEGER, + }, +}; + +export default (sequelize) => { + const VolumePrices = sequelize.define('FDvolumePrices', volumePrices, { + timestamps: false, + freezeTableName: true, + }); + return VolumePrices; +}; \ No newline at end of file diff --git a/utils.js b/utils.js index f7810aa..d3c50ae 100644 --- a/utils.js +++ b/utils.js @@ -11,6 +11,8 @@ dotenv.config(); */ const _accessTokenEndpoint = 'https://idm.staging.floriday.io/oauth2/ausmw6b47z1BnlHkw0h7/v1/token'; +const BASE_CUSTOMER_URL = 'https://api.staging.floriday.io/customers-api/2022v2/'; + /** * Gets the Access Token from the client config table * @@ -121,6 +123,63 @@ async function sleep(ms) { }); } +/** + * Returns the uuidv4 string with the prefix 'Vn-' + * + * @param {string} uuidv4 - uuidv4 string + * @returns + */ +async function vnUuid(uuidv4) { + return 'Vn-' + uuidv4; +} +/** + * Returns the uuidv4 string without the prefix 'Vn-' + * + * @param {string} uuidv4 - uuidv4 string with prefix 'Vn-' + * @returns + */ +async function removeVnPrefix(uuidv4) { + return uuidv4.replace('Vn-', ''); +} + +/** + * Recieves an array of functions and executes them in a queue with the given concurrency + * + * @param {Array} fnArray + * @param {Number} concurrency + * @returns + */ +async function asyncQueue(fnArray, concurrency = 1) { + + const results = []; // 1 + + const queue = fnArray.map((fn, index) => () => fn().then((result) => results[index] = result)); + + const run = async () => { // 2 + const fn = queue.shift(); + if (fn) { + await fn(); + await run(); + } + }; + + const promises = []; // 3 + while (concurrency--) { // 4 + promises.push(run()); + } + + await Promise.all(promises); // 5 + + return results; + +} +// 1. Create an array of functions that will be executed in a queue +// 2. Create a function that will execute the functions in the queue +// 3. Create an array of promises that will execute the run function +// 4. Execute the run function while the concurrency is greater than 0 +// 5. Return the results + + /** * * Inserts every tradeitem from the organization into the DB @@ -135,8 +194,6 @@ async function getTradeitems(organizationGln) { 'X-Api-Key': process.env.API_KEY, }; - const BASE_CUSTOMER_URL = 'https://api.staging.floriday.io/customers-api/2022v2/'; - // Get the organization id from the organizationGln const organizationsUrl = `${BASE_CUSTOMER_URL}organizations/gln/${organizationGln}`; @@ -181,11 +238,17 @@ async function getTradeitems(organizationGln) { try { - item.tradeItemId = 'Vn-' + item.tradeItemId; + item.tradeItemId = await vnUuid(item.tradeItemId); + + await models.suppliers.upsert({ + supplierId: await vnUuid(organizationId), + supplierGln: organizationGln, + }); await models.tradeItem.upsert({ tradeItemId: item.tradeItemId, supplierOrganizationId: organizationGln, + supplierOrganizationUuid: await vnUuid(organizationId), code: item.code, gtin: item.gtin, vbnProductCode: item.vbnProductCode, @@ -214,9 +277,9 @@ async function getTradeitems(organizationGln) { }); - await item.photos.forEach((photo) => { + await item.photos.forEach(async (photo) => { - photo.id = 'Vn-' + photo.id; + photo.id = await vnUuid(photo.id); models.photos.upsert({ tradeItemFk: item.tradeItemId, @@ -231,7 +294,7 @@ async function getTradeitems(organizationGln) { await item.packingConfigurations.forEach(async (packagingConfiguration) => { let uuid = uuidv4(); - uuid = 'Vn-' + uuid; + uuid = await vnUuid(uuid); await models.packingConfigurations.upsert({ tradeItemFk: item.tradeItemId, @@ -270,7 +333,9 @@ async function getTradeitems(organizationGln) { }); } catch (error) { - console.log('There was an error while saving the data to the database'); + console.log('Missing data for the tradeitem: ' + item.tradeItemId); + bar.stop(); + return; } }); @@ -278,9 +343,105 @@ async function getTradeitems(organizationGln) { } } catch (error) { - console.log('There was an error while saving the data to the database'); - throw new Error(error); + console.log('There was an error while saving the data to the database, trying again in 5 seconds...'); + await sleep(5000); + await getTradeitems(organizationGln); } } -export { getClientToken, updateClientConfig, getJWT, getTradeitems, sleep }; \ No newline at end of file +async function getStock() { + + const suppliers = await models.suppliers.findAll(); + + const supplierQueue = []; + + suppliers.forEach(async (supplier) => { + supplierQueue.push(await getStockBySupplier(supplier.supplierId)); + }); + + await asyncQueue(supplierQueue); + +} + +async function getStockBySupplier(supplierId) { + + supplierId = await removeVnPrefix(supplierId); + + const headers = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${await getJWT()}`, + 'X-Api-Key': process.env.API_KEY, + }; + + const stockUrl = `${BASE_CUSTOMER_URL}supply-lines?supplierOrganizationId=${supplierId}`; + + const stockRequest = await fetch(stockUrl, { + method: 'GET', + headers: headers, + }); + + const stockResponse = await stockRequest.json(); + + if (stockResponse.length == 0) { + return; + } + + try { + + if (stockResponse.length > 0 && stockResponse != null) { + + let bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_grey); + bar.start(stockResponse.length, 0); + + await stockResponse.forEach(async supplyLine => { + + bar.increment(); + + try { + + await models.supplyLines.upsert({ + supplyLineId: supplyLine.supplyLineId, + status: supplyLine.status, + supplierOrganizationId: await vnUuid(supplyLine.supplierOrganizationId), + pricePerPiece: supplyLine.pricePerPiece, + numberOfPieces: supplyLine.numberOfPieces, + deliveryPeriod: supplyLine.deliveryPeriod, + orderPeriod: supplyLine.orderPeriod, + warehouseId: await vnUuid(supplyLine.warehouseId), + sequenceNumber: supplyLine.sequenceNumber, + type: supplyLine.type, + isDeleted: supplyLine.isDeleted, + salesUnit: supplyLine.salesUnit, + agreementReferenceCode: supplyLine.agreementReference.code, + agreementReferenceDescription: supplyLine.agreementReference.description, + isLimited: supplyLine.isLimited, + isCustomerSpecific: supplyLine.isCustomerSpecific, + tradeItemFk: await vnUuid(supplyLine.tradeItemId), + }); + + await models.volumePrices.upsert({ + unit: supplyLine.volumePrices.unit, + price: supplyLine.volumePrices.price, + supplyLineFk: await vnUuid(supplyLine.supplyLineId), + }); + + } catch (error) { + if (error.name == 'SequelizeForeignKeyConstraintError') { + console.log('Missing data for the tradeitem: ' + supplyLine.tradeItemId); + } + bar.stop(); + return; + } + + }); + bar.stop(); + } + } catch (error) { + console.log('There was an error while saving the data to the database, trying again in 5 seconds...'); + await sleep(5000); + await getStockBySupplier(supplierId); + } + +} + +export { getClientToken, updateClientConfig, getJWT, getTradeitems, sleep, asyncQueue, getStock }; \ No newline at end of file