diff --git a/README.md b/README.md index 5a45c76..69c3657 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ The Floriday service project should perform the following tasks: 1. Create / mantain the table structure to allow the storage of the data provided by the Floriday API. This is done using the [Sequelize](https://sequelize.org/) ORM. 2. Query the Floriday API and store the data in the database. - This is done using the [node-fetch](https://www.npmjs.com/package/node-fetch) package. + This is done using the [axios](https://www.npmjs.com/package/axios) package. 2.1. The data is requested every minute, but only the data that has changed is stored in the database. This is done using the [Sequelize](https://sequelize.org/) ORM and storing the data as it is received from the API, so it can be compared with the previous data. diff --git a/floriday.js b/floriday.js index 49fd8c0..4c99a64 100644 --- a/floriday.js +++ b/floriday.js @@ -1,5 +1,5 @@ -import * as utils from './utils.js'; import { models, checkConn, closeConn } from './models/sequelize.js'; +import * as utils from './utils.js'; import moment from 'moment'; import chalk from 'chalk'; diff --git a/models/sequelize.js b/models/sequelize.js index 434fbe4..d216d9b 100644 --- a/models/sequelize.js +++ b/models/sequelize.js @@ -174,7 +174,7 @@ try { } let action, isForce; -if (JSON.parse(process.env.FORCE_SYNC)) { +if (JSON.parse(env.FORCE_SYNC)) { action = 'Forcing' isForce = true } else { @@ -198,12 +198,12 @@ catch (err) { * @returns {Sequelize} Sequelize instance with the connection to the database. */ function createConn() { - return new Sequelize(process.env.DB_SCHEMA, process.env.DB_USER, process.env.DB_PWD, { - host: process.env.DB_HOST, - dialect: process.env.DB_DIALECT, + return new Sequelize(env.DB_SCHEMA, env.DB_USER, env.DB_PWD, { + host: env.DB_HOST, + dialect: env.DB_DIALECT, logging: false, pool: { - max: parseInt(process.env.DB_MAX_CONN_POOL), + max: parseInt(env.DB_MAX_CONN_POOL), acquire: 60000, idle: 10000, }, diff --git a/package-lock.json b/package-lock.json index 5957891..8297308 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,11 +6,11 @@ "": { "name": "floriday", "dependencies": { + "axios": "^1.4.0", "chalk": "^5.2.0", "dotenv": "^16.0.3", "mariadb": "^3.1.2", "moment": "^2.29.4", - "node-fetch": "^3.3.1", "ora": "^6.3.0", "sequelize": "^6.31.1", "uuid": "^9.0.0" @@ -238,6 +238,21 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", + "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -377,6 +392,17 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -397,14 +423,6 @@ "node": ">= 8" } }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "engines": { - "node": ">= 12" - } - }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -438,6 +456,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/denque": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", @@ -670,28 +696,6 @@ "reusify": "^1.0.4" } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -739,15 +743,36 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "node_modules/follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "dependencies": { - "fetch-blob": "^3.1.2" + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" }, "engines": { - "node": ">=12.20.0" + "node": ">= 6" } }, "node_modules/fs.realpath": { @@ -1074,6 +1099,25 @@ "node": ">= 12" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -1124,41 +1168,6 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz", - "integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -1329,6 +1338,11 @@ "node": ">= 0.8.0" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -1731,14 +1745,6 @@ "defaults": "^1.0.3" } }, - "node_modules/web-streams-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", - "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", - "engines": { - "node": ">= 8" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 465548e..a272221 100644 --- a/package.json +++ b/package.json @@ -3,11 +3,11 @@ "module": "index.ts", "type": "module", "dependencies": { + "axios": "^1.4.0", "chalk": "^5.2.0", "dotenv": "^16.0.3", "mariadb": "^3.1.2", "moment": "^2.29.4", - "node-fetch": "^3.3.1", "ora": "^6.3.0", "sequelize": "^6.31.1", "uuid": "^9.0.0" diff --git a/utils.js b/utils.js index c6c74e9..3506a66 100644 --- a/utils.js +++ b/utils.js @@ -1,9 +1,10 @@ -import moment from 'moment'; -import fetch from 'node-fetch'; import { models } from './models/sequelize.js'; import { v4 as uuidv4 } from 'uuid'; +import axios from 'axios'; +import moment from 'moment'; import chalk from 'chalk'; -import ora, { spinners } from 'ora'; +import ora from 'ora'; + const env = process.env; /** @@ -39,22 +40,21 @@ export async function requestToken() { client_secret: clientSecret, scope: 'role:app catalog:read supply:read organization:read network:write network:read' }; - - const body = Object.keys(data) - .map(key => `${encodeURIComponent(key)}=${encodeURIComponent(data[key])}`) - .join('&'); - - const response = await fetch(env.API_ENDPOINT, { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - }, - body, - }); - - const responseData = await response.json(); - - if (response.ok) + + const response = await axios.post(env.API_ENDPOINT, + Object.keys(data) + .map(key => `${encodeURIComponent(key)}=${encodeURIComponent(data[key])}`) + .join('&'), + { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + } + } + ); + + const responseData = response.data; + + if (response.statusText = 'OK') spinner.succeed(); else { spinner.fail(); @@ -278,27 +278,19 @@ export async function syncSuppliers(){ 'X-Api-Key': process.env.API_KEY, }; - const response = await fetch(`${env.API_URL}/organizations/current-max-sequence`, { - method: 'GET', - headers, - }); + const response = await axios.get(`${env.API_URL}/organizations/current-max-sequence`, { headers }); if (!response.ok) { spinner.fail(); criticalError(new Error(`Max sequence request failed with status: ${response.status} - ${response.statusText}`)); } + const maxSequenceNumber = response.data; - const maxSequenceNumber = await response.json(); let timeFinish, timeToGoSec, timeToGoMin, timeLeft; for (let curSequenceNumber = 0; curSequenceNumber <= maxSequenceNumber; curSequenceNumber++) { let timeStart = new moment(); - - let query = `${env.API_URL}/organizations/sync/${curSequenceNumber}?organizationType=SUPPLIER`; - let response = await fetch(query, { - method: 'GET', - headers - }); - let data = await response.json(); + let response = await axios.get(`${env.API_URL}/organizations/sync/${curSequenceNumber}?organizationType=SUPPLIER`, { headers }); + let data = response.data; let suppliers = data.results; for (let supplier of suppliers) { @@ -351,27 +343,19 @@ export async function syncConn(){ 'X-Api-Key': process.env.API_KEY }; - let remoteConnections = await fetch(`${env.API_URL}/connections`, { - method: 'GET', - headers - }); - - remoteConnections = await remoteConnections.json(); - + let remoteConnections = await axios.get(`${env.API_URL}/connections`, { headers }); + let i = 1; for (let connection of connections){ spinner.text = `Syncing ${i++} connections...` if (connection.isConnected == false) continue; - let remoteConnection = remoteConnections.find(remoteConnection => remoteConnection == connection.supplierOrganizationId); + let remoteConnection = remoteConnections.data.find(remoteConnection => remoteConnection == connection.supplierOrganizationId); if (remoteConnection == undefined){ console.log('Connection: ', connection, 'does not exist in the remote server'); console.log('Creating remote connection'); - await fetch(`${env.API_URL}/connections/${connection.supplierOrganizationId}`, { - method: 'PUT', - headers - }); + await axios.put(`${env.API_URL}/connections/${connection.supplierOrganizationId}`, { headers }); await models.connection.update({ isConnected: true }, { where: { supplierOrganizationId: connection.supplierOrganizationId @@ -415,18 +399,15 @@ export async function syncTradeItems(){ 'X-Api-Key': process.env.API_KEY }; try { - let request = await fetch(`${env.API_URL}/trade-items?supplierOrganizationId=${supplier.supplierOrganizationId}`, { - method: 'GET', - headers - }); + let request = await axios.get(`${env.API_URL}/trade-items?supplierOrganizationId=${supplier.supplierOrganizationId}`, { headers }) - let tradeItems = await request.json(); + let tradeItems = request.data; if (!tradeItems.length) continue; for (let tradeItem of tradeItems) { - await insertItem(tradeItem, supplier); + await insertItem(tradeItem); spinner.text = `Syncing ${i++} trade items...` }; } catch (err) { @@ -439,7 +420,7 @@ export async function syncTradeItems(){ /** * Syncs the supply lines for suppliers that are connected to do this, - * it fetches all the supply lines for every tradeitem of the suppliers + * it searches all the supply lines for every tradeitem of the suppliers */ export async function syncSupplyLines(){ const spinner = ora(`Syncing supply lines...`).start(); @@ -476,17 +457,14 @@ export async function syncSupplyLines(){ tradeItemId: tradeItem.tradeItemId, postFilterSelectedTradeItems: false }); - let request = await fetch(`${url}?${params.toString()}`, { - method: 'GET', - headers - }); + let request = await axios.get(`${url}?${params.toString()}`, { headers }); if (request.status == 429) { // Too many request resolve([]); return; } - let supplyLines = await request.json(); + let supplyLines = request.data; if (!supplyLines.results.length) { resolve([]); @@ -524,26 +502,17 @@ export async function syncSupplyLines(){ console.log('Trade item not found for supply line: ', line.supplyLineId); console.log('Requesting data for trade item id: ', line.tradeItemId); - let queryTradeItem = await fetch(`${env.API_URL}/trade-items?tradeItemIds=${line.tradeItemId}`, { - method: 'GET', - headers - }); + let queryTradeItem = await axios.get(`${env.API_URL}/trade-items?tradeItemIds=${line.tradeItemId}`, { headers }); - let tradeItem = await queryTradeItem.json(); + let tradeItem = queryTradeItem.data; if (tradeItem.length == 0) { console.log('Trade item not found for supply line: ', line.supplyLineId); console.log('Trade item id: ', line.tradeItemId); continue; } - - let supplier = await models.supplier.findOne({ - where: { - supplierOrganizationId: tradeItem[0].supplierOrganizationId - } - }); - - await insertItem(tradeItem[0], supplier); + + await insertItem(tradeItem[0]); tradeItem = await models.tradeItem.findOne({ where: { @@ -604,9 +573,15 @@ export async function syncSupplyLines(){ } } -export async function insertItem(tradeItem, supplier) { +/** + * Insert the trade item + * + * @param {array} tradeItem + */ +export async function insertItem(tradeItem) { + let tx; try { - const tx = await models.sequelize.transaction(); + tx = await models.sequelize.transaction(); // Upsert supplier connection await models.connection.upsert({ @@ -623,66 +598,71 @@ export async function insertItem(tradeItem, supplier) { }, { transaction: tx }); // Upsert characteristics - for (const characteristic of tradeItem.characteristics) { - await models.characteristic.upsert({ - ...characteristic, - tradeItemId: tradeItem.tradeItemId, - }, { transaction: tx }); - } - + if (tradeItem.characteristics) + for (const characteristic of tradeItem.characteristics) { + await models.characteristic.upsert({ + ...characteristic, + tradeItemId: tradeItem.tradeItemId, + }, { transaction: tx }); + } // Upsert seasonal periods - for (const seasonalPeriod of tradeItem.seasonalPeriods) { - await models.seasonalPeriod.upsert({ - ...seasonalPeriod, - tradeItemId: tradeItem.tradeItemId, - }, { transaction: tx }); - } + if (tradeItem.seasonalPeriods) + for (const seasonalPeriod of tradeItem.seasonalPeriods) { + await models.seasonalPeriod.upsert({ + ...seasonalPeriod, + tradeItemId: tradeItem.tradeItemId, + }, { transaction: tx }); + } // Upsert photos - for (const photo of tradeItem.photos) { - await models.photo.upsert({ - ...photo, - tradeItemId: tradeItem.tradeItemId, - }, { transaction: tx }); - } + if (tradeItem.photos) + for (const photo of tradeItem.photos) { + await models.photo.upsert({ + ...photo, + tradeItemId: tradeItem.tradeItemId, + }, { transaction: tx }); + } // Upsert packing configurations - for (const packingConfiguration of tradeItem.packingConfigurations) { - const uuid = uuidv4(); + if (tradeItem.packingConfigurations) + for (const packingConfiguration of tradeItem.packingConfigurations) { + const uuid = uuidv4(); - await models.packingConfiguration.upsert({ - packingConfigurationId: uuid, - ...packingConfiguration, - additionalPricePerPiece: JSON.stringify(packingConfiguration.additionalPricePerPiece), - tradeItemId: tradeItem.tradeItemId, - }, { transaction: tx }); + await models.packingConfiguration.upsert({ + packingConfigurationId: uuid, + ...packingConfiguration, + additionalPricePerPiece: JSON.stringify(packingConfiguration.additionalPricePerPiece), + tradeItemId: tradeItem.tradeItemId, + }, { transaction: tx }); - await models.package.upsert({ - ...packingConfiguration.package, - packingConfigurationFk: uuid, - }, { transaction: tx }); - } + await models.package.upsert({ + ...packingConfiguration.package, + packingConfigurationFk: uuid, + }, { transaction: tx }); + } // Upsert country of origin ISO codes - for (const isoCode of tradeItem.countryOfOriginIsoCodes || []) { - await models.countryOfOriginIsoCode.upsert({ - isoCode, - tradeItemId: tradeItem.tradeItemId, - }, { transaction: tx }); - } + if (tradeItem.countryOfOriginIsoCodes) + for (const isoCode of tradeItem.countryOfOriginIsoCodes || []) { + await models.countryOfOriginIsoCode.upsert({ + isoCode, + tradeItemId: tradeItem.tradeItemId, + }, { transaction: tx }); + } // Upsert botanical names - for (const botanicalName of tradeItem.botanicalNames) { - await models.botanicalName.upsert({ - ...botanicalName, - tradeItemId: tradeItem.tradeItemId, - }, { transaction: tx }); - } + if (tradeItem.botanicalNames) + for (const botanicalName of tradeItem.botanicalNames) { + await models.botanicalName.upsert({ + ...botanicalName, + tradeItemId: tradeItem.tradeItemId, + }, { transaction: tx }); + } await tx.commit(); } catch (err) { - console.log(err); await tx.rollback(); + throw err; } }