From acebc9a89f1bf99522ca373c6dba18e103b18f24 Mon Sep 17 00:00:00 2001 From: guillermo Date: Mon, 22 May 2023 09:47:16 +0200 Subject: [PATCH] refs #4823 Now save the maxSequenceNumber of orgs --- models/config/sequenceNumber.js | 22 ++++++ models/sequelize.js | 8 +- utils.js | 132 +++++++++++++++++++------------- 3 files changed, 104 insertions(+), 58 deletions(-) create mode 100644 models/config/sequenceNumber.js diff --git a/models/config/sequenceNumber.js b/models/config/sequenceNumber.js new file mode 100644 index 0000000..313ea7d --- /dev/null +++ b/models/config/sequenceNumber.js @@ -0,0 +1,22 @@ +import { Sequelize } from 'sequelize'; + +const sequenceNumber = { + model: { + type: Sequelize.STRING, + primaryKey: true, + }, + maxSequenceNumber: { + type: Sequelize.INTEGER, + }, +}; + +export default (sequelize) => { + return sequelize.define( + 'sequenceNumber', + sequenceNumber, + { + timestamps: false, + freezeTableName: true, + } + ); +}; \ No newline at end of file diff --git a/models/sequelize.js b/models/sequelize.js index 7de8067..5fc77d8 100644 --- a/models/sequelize.js +++ b/models/sequelize.js @@ -29,13 +29,14 @@ try { criticalError(err); } +// Conf Models +import clientConfig from './config/clientConfig.js'; +import sequenceNumber from './config/sequenceNumber.js'; + // Supply Line Models import supplyLine from './supplyLine/supplyLine.js'; import volumePrices from './supplyLine/volumePrices.js'; -// Conf Models -import clientConfig from './config/clientConfig.js'; - // Organization Models import organization from './organization/organization.js'; import warehouses from './organization/warehouses.js'; @@ -55,6 +56,7 @@ import characteristics from './tradeItem/characteristics.js'; */ let models = { sequelize: sequelize, + sequenceNumber: sequenceNumber(sequelize), clientConfig: clientConfig(sequelize), organization: organization(sequelize), warehouses: warehouses(sequelize), diff --git a/utils.js b/utils.js index 6f4532b..45dd62a 100644 --- a/utils.js +++ b/utils.js @@ -34,7 +34,7 @@ export async function requestToken(isForce = false) { scope: 'role:app catalog:read supply:read organization:read network:write network:read' }).toString(); const headers = { 'Content-Type': 'application/x-www-form-urlencoded' }; - const response = (await vnRequest('POST', env.API_ENDPOINT, data, headers)).data; + const response = (await vnRequest('POST', env.API_ENDPOINT, data, headers, spinner)).data; const tokenExpiration = moment() .add(response.expires_in, 's') @@ -127,14 +127,20 @@ export async function sleep(ms) { */ export async function syncOrganizations(){ let spinner = ora('Syncing organizations...').start(); - let i = 1; + let i = 0; try { - const maxSequenceNumber = (await vnRequest('GET', `${env.API_URL}/organizations/current-max-sequence`)).data; - for (let curSequenceNumber = 0; curSequenceNumber <= maxSequenceNumber; curSequenceNumber++) { + const sequenceNumberDb = await models.sequenceNumber.findOne({ + where: { model: 'organization' } + }); + const maxSequenceNumber = (await vnRequest('GET', `${env.API_URL}/organizations/current-max-sequence`, null, null, spinner)).data; + let curSequenceNumber = (sequenceNumberDb?.maxSequenceNumber) + ? sequenceNumberDb.maxSequenceNumber + : 0 + for (curSequenceNumber; curSequenceNumber <= maxSequenceNumber; curSequenceNumber++) { const params = new URLSearchParams({ organizationType: 'SUPPLIER' }).toString(); - let response = (await vnRequest('GET', `${env.API_URL}/organizations/sync/${curSequenceNumber}?${params}`)).data; + let response = (await vnRequest('GET', `${env.API_URL}/organizations/sync/${curSequenceNumber}?${params}`, null, null, spinner)).data; curSequenceNumber = response.maximumSequenceNumber; const orgs = response.results; for (let org of orgs) { @@ -145,6 +151,11 @@ export async function syncOrganizations(){ } }; } + await models.sequenceNumber.upsert({ + model: 'organization', + maxSequenceNumber, + }); + spinner.text = `Syncing ${i} organizations, 0 missing...` spinner.succeed(); } catch (err) { @@ -172,7 +183,7 @@ export async function syncConnections(){ group: ['organizationId'], }); - const connectionsInFloriday = (await vnRequest('GET', `${env.API_URL}/connections`)).data; + const connectionsInFloriday = (await vnRequest('GET', `${env.API_URL}/connections`, null, null, spinner)).data; let isExists = false, connectionsToPut = []; for (let connectionInDb of connectionsInDb) { @@ -190,7 +201,7 @@ export async function syncConnections(){ let i = 1; for (let connection of connectionsToPut) { spinner.text = `Creating ${i++} of ${connectionsToPut.length} connections in Floriday...` - await vnRequest('PUT', `${env.API_URL}/connections/${connection}`); + await vnRequest('PUT', `${env.API_URL}/connections/${connection}`, null, null, spinner); } if (spinner) spinner.succeed(); } catch (err) { @@ -214,7 +225,7 @@ export async function syncTradeItems(){ const params = new URLSearchParams({ supplierOrganizationId: org.organizationId, }).toString(); - let tradeItems = (await vnRequest('GET', `${env.API_URL}/trade-items?${params}`)).data + let tradeItems = (await vnRequest('GET', `${env.API_URL}/trade-items?${params}`, null, null, spinner)).data spinner.text = `Syncing ${i} trade items of [${x++}|${orgs.length}] organizations...` if (!tradeItems.length) continue; @@ -250,7 +261,7 @@ export async function syncSupplyLines() { const params = new URLSearchParams({ supplierOrganizationId: org.organizationId, }).toString(); - let supplyLines = (await vnRequest('GET',`${env.API_URL}/supply-lines?${params}`)).data; + let supplyLines = (await vnRequest('GET',`${env.API_URL}/supply-lines?${params}`, null, null, spinner)).data; if (!supplyLines.length) continue for (let supplyLine of supplyLines) { @@ -259,14 +270,14 @@ export async function syncSupplyLines() { where: { warehouseId: supplyLine.warehouseId } }); if (!warehouse) { - let warehouse = (await vnRequest('GET', `${env.API_URL}/warehouses/${supplyLine.warehouseId}`)).data; + let warehouse = (await vnRequest('GET', `${env.API_URL}/warehouses/${supplyLine.warehouseId}`, null, null, spinner)).data; // Check if the organization exists, and if it doesn't, create it let organization = await models.organization.findOne({ where: { organizationId: warehouse.organizationId } }); if (!organization) { - let organization = (await vnRequest('GET', `${env.API_URL}/organizations/${warehouse.organizationId}`)).data; + let organization = (await vnRequest('GET', `${env.API_URL}/organizations/${warehouse.organizationId}`, null, null, spinner)).data; await insertOrganization(organization); } @@ -278,7 +289,7 @@ export async function syncSupplyLines() { where: { tradeItemId: supplyLine.tradeItemId } }); if (!tradeItem) { - let tradeItem = (await vnRequest('GET', `${env.API_URL}/trade-items/${supplyLine.tradeItemId}`)).data; + let tradeItem = (await vnRequest('GET', `${env.API_URL}/trade-items/${supplyLine.tradeItemId}`, null, null, spinner)).data; await insertItem(tradeItem); } @@ -462,7 +473,7 @@ export async function syncWarehouses(){ let x = 0, i = 0; for (let org of orgs) { spinner.text = `Syncing ${i} warehouses of [${x++}|${orgs.length}] organizations...` - const warehouses = (await vnRequest('GET', `${env.API_URL}/organizations/supplier/${org.organizationId}/warehouses`)).data; + const warehouses = (await vnRequest('GET', `${env.API_URL}/organizations/supplier/${org.organizationId}/warehouses`, null, null, spinner)).data; for (let warehouse of warehouses) { spinner.text = `Syncing ${i++} warehouses of [${x}|${orgs.length}] organizations...` await insertWarehouse(warehouse); @@ -483,7 +494,7 @@ export async function deleteConnections() { let spinner; try { let i = 1; - const connectionsInFloriday = (await vnRequest('GET', `${env.API_URL}/connections`)).data; + const connectionsInFloriday = (await vnRequest('GET', `${env.API_URL}/connections`, null, null, spinner)).data; const connectionsInDb = await models.organization.findAll({ attributes: ['organizationId'], where: { isConnected: true } @@ -503,7 +514,7 @@ export async function deleteConnections() { if (ghostConnections.length) spinner = ora(`Deleting connections that aren't in the db...`).start(); for (let connection of ghostConnections) { - await vnRequest('DELETE', `${env.API_URL}/connections/${connection}`); + await vnRequest('DELETE', `${env.API_URL}/connections/${connection}`, null, null, spinner); spinner.text = `Deleting ${i++} of ${ghostConnections.length} that aren't in the db...` } if (spinner) spinner.succeed(); @@ -520,10 +531,11 @@ export async function deleteConnections() { * @param {String} method * @param {Array} body * @param {Array} header + * @param {Array} spinner * * @return {Array} **/ -export async function vnRequest(method, url, data, headers) { +export async function vnRequest(method, url, data, headers, spinner) { if (!headers) headers = { 'Content-Type': 'application/json', @@ -532,50 +544,60 @@ export async function vnRequest(method, url, data, headers) { }; while(true) { - try { + try { return (['GET', 'DELETE'].includes(method)) ? await axios({method, url, headers}) : await axios({method, url, data, headers}); - } catch (err) { - switch (err.code) { - case 'ECONNRESET': // Client network socket TLS - case 'EAI_AGAIN': // getaddrinfo - warning(err); - await sleep(1000); - break; - case 'ECONNABORTED': - case 'ECONNREFUSED': - case 'ERR_BAD_REQUEST': - switch (err.response.status) { - case 504: - case 502: - warning(err); - await sleep(1000); - break; - case 429: // Too Many Requests - warning(err); - await sleep(3400); // Stipulated by Floriday - break; - case 401: // Unauthorized - warning(err); - await requestToken(true); - headers.Authorization - ? headers.Authorization = `Bearer ${await getCurrentToken()}` - : criticalError(err); - break; - default: - warning(err); - await sleep(env.MS_RETRY_UNHANDLED_ERROR); - break; - } - break; - default: - warning(err); - await sleep(env.MS_RETRY_UNHANDLED_ERROR); - break; + } catch (err) { + switch (err.code) { + case 'ECONNRESET': // Client network socket TLS + case 'EAI_AGAIN': // getaddrinfo + warning(err); + await sleep(1000); + break; + case 'ECONNABORTED': + case 'ECONNREFUSED': + case 'ERR_BAD_REQUEST': + switch (err.response.status) { + case 504: + case 502: + if (spinner) spinner.warn(); + warning(err); + await sleep(1000); + if (spinner) spinner.start(); + break; + case 429: // Too Many Requests + if (spinner) spinner.warn(); + warning(err); + await sleep(3400); // Stipulated by Floriday + if (spinner) spinner.start(); + break; + case 401: // Unauthorized + if (spinner) spinner.warn(); + warning(err); + await requestToken(true); + headers.Authorization + ? headers.Authorization = `Bearer ${await getCurrentToken()}` + : criticalError(err); + if (spinner) spinner.start(); + break; + default: + if (spinner) spinner.warn(); + warning(err); + await sleep(env.MS_RETRY_UNHANDLED_ERROR); + if (spinner) spinner.start(); + break; + } + break; + default: + if (spinner) spinner.warn(); + warning(err); + await sleep(env.MS_RETRY_UNHANDLED_ERROR); + if (spinner) spinner.start(); + break; + } } } - } } /**