refs #4823 Reeplaced node-fetch to axios

This commit is contained in:
Guillermo Bonet 2023-05-11 08:39:02 +02:00
parent 8448dc52c6
commit 1a746968b8
6 changed files with 189 additions and 203 deletions

View File

@ -5,7 +5,7 @@ The Floriday service project should perform the following tasks:
1. Create / mantain the table structure to allow the storage of the data provided by the Floriday API.
This is done using the [Sequelize](https://sequelize.org/) ORM.
2. Query the Floriday API and store the data in the database.
This is done using the [node-fetch](https://www.npmjs.com/package/node-fetch) package.
This is done using the [axios](https://www.npmjs.com/package/axios) package.
2.1. The data is requested every minute, but only the data that has changed is stored in the database.
This is done using the [Sequelize](https://sequelize.org/) ORM and storing the data as it is received from the API,
so it can be compared with the previous data.

View File

@ -1,5 +1,5 @@
import * as utils from './utils.js';
import { models, checkConn, closeConn } from './models/sequelize.js';
import * as utils from './utils.js';
import moment from 'moment';
import chalk from 'chalk';

View File

@ -174,7 +174,7 @@ try {
}
let action, isForce;
if (JSON.parse(process.env.FORCE_SYNC)) {
if (JSON.parse(env.FORCE_SYNC)) {
action = 'Forcing'
isForce = true
} else {
@ -198,12 +198,12 @@ catch (err) {
* @returns {Sequelize} Sequelize instance with the connection to the database.
*/
function createConn() {
return new Sequelize(process.env.DB_SCHEMA, process.env.DB_USER, process.env.DB_PWD, {
host: process.env.DB_HOST,
dialect: process.env.DB_DIALECT,
return new Sequelize(env.DB_SCHEMA, env.DB_USER, env.DB_PWD, {
host: env.DB_HOST,
dialect: env.DB_DIALECT,
logging: false,
pool: {
max: parseInt(process.env.DB_MAX_CONN_POOL),
max: parseInt(env.DB_MAX_CONN_POOL),
acquire: 60000,
idle: 10000,
},

166
package-lock.json generated
View File

@ -6,11 +6,11 @@
"": {
"name": "floriday",
"dependencies": {
"axios": "^1.4.0",
"chalk": "^5.2.0",
"dotenv": "^16.0.3",
"mariadb": "^3.1.2",
"moment": "^2.29.4",
"node-fetch": "^3.3.1",
"ora": "^6.3.0",
"sequelize": "^6.31.1",
"uuid": "^9.0.0"
@ -238,6 +238,21 @@
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
"dev": true
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
"node_modules/axios": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
"integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
"dependencies": {
"follow-redirects": "^1.15.0",
"form-data": "^4.0.0",
"proxy-from-env": "^1.1.0"
}
},
"node_modules/balanced-match": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
@ -377,6 +392,17 @@
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@ -397,14 +423,6 @@
"node": ">= 8"
}
},
"node_modules/data-uri-to-buffer": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
"engines": {
"node": ">= 12"
}
},
"node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@ -438,6 +456,14 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
@ -670,28 +696,6 @@
"reusify": "^1.0.4"
}
},
"node_modules/fetch-blob": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "paypal",
"url": "https://paypal.me/jimmywarting"
}
],
"dependencies": {
"node-domexception": "^1.0.0",
"web-streams-polyfill": "^3.0.3"
},
"engines": {
"node": "^12.20 || >= 14.13"
}
},
"node_modules/file-entry-cache": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
@ -739,15 +743,36 @@
"integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
"dev": true
},
"node_modules/formdata-polyfill": {
"version": "4.0.10",
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
"node_modules/follow-redirects": {
"version": "1.15.2",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
"integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
}
},
"node_modules/form-data": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
"dependencies": {
"fetch-blob": "^3.1.2"
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">=12.20.0"
"node": ">= 6"
}
},
"node_modules/fs.realpath": {
@ -1074,6 +1099,25 @@
"node": ">= 12"
}
},
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dependencies": {
"mime-db": "1.52.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mimic-fn": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
@ -1124,41 +1168,6 @@
"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
"dev": true
},
"node_modules/node-domexception": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "github",
"url": "https://paypal.me/jimmywarting"
}
],
"engines": {
"node": ">=10.5.0"
}
},
"node_modules/node-fetch": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz",
"integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==",
"dependencies": {
"data-uri-to-buffer": "^4.0.0",
"fetch-blob": "^3.1.4",
"formdata-polyfill": "^4.0.10"
},
"engines": {
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/node-fetch"
}
},
"node_modules/once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@ -1329,6 +1338,11 @@
"node": ">= 0.8.0"
}
},
"node_modules/proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
},
"node_modules/punycode": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@ -1731,14 +1745,6 @@
"defaults": "^1.0.3"
}
},
"node_modules/web-streams-polyfill": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
"engines": {
"node": ">= 8"
}
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

View File

@ -3,11 +3,11 @@
"module": "index.ts",
"type": "module",
"dependencies": {
"axios": "^1.4.0",
"chalk": "^5.2.0",
"dotenv": "^16.0.3",
"mariadb": "^3.1.2",
"moment": "^2.29.4",
"node-fetch": "^3.3.1",
"ora": "^6.3.0",
"sequelize": "^6.31.1",
"uuid": "^9.0.0"

108
utils.js
View File

@ -1,9 +1,10 @@
import moment from 'moment';
import fetch from 'node-fetch';
import { models } from './models/sequelize.js';
import { v4 as uuidv4 } from 'uuid';
import axios from 'axios';
import moment from 'moment';
import chalk from 'chalk';
import ora, { spinners } from 'ora';
import ora from 'ora';
const env = process.env;
/**
@ -40,21 +41,20 @@ export async function requestToken() {
scope: 'role:app catalog:read supply:read organization:read network:write network:read'
};
const body = Object.keys(data)
const response = await axios.post(env.API_ENDPOINT,
Object.keys(data)
.map(key => `${encodeURIComponent(key)}=${encodeURIComponent(data[key])}`)
.join('&');
const response = await fetch(env.API_ENDPOINT, {
method: 'POST',
.join('&'),
{
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
},
body,
});
}
}
);
const responseData = await response.json();
const responseData = response.data;
if (response.ok)
if (response.statusText = 'OK')
spinner.succeed();
else {
spinner.fail();
@ -278,27 +278,19 @@ export async function syncSuppliers(){
'X-Api-Key': process.env.API_KEY,
};
const response = await fetch(`${env.API_URL}/organizations/current-max-sequence`, {
method: 'GET',
headers,
});
const response = await axios.get(`${env.API_URL}/organizations/current-max-sequence`, { headers });
if (!response.ok) {
spinner.fail();
criticalError(new Error(`Max sequence request failed with status: ${response.status} - ${response.statusText}`));
}
const maxSequenceNumber = response.data;
const maxSequenceNumber = await response.json();
let timeFinish, timeToGoSec, timeToGoMin, timeLeft;
for (let curSequenceNumber = 0; curSequenceNumber <= maxSequenceNumber; curSequenceNumber++) {
let timeStart = new moment();
let query = `${env.API_URL}/organizations/sync/${curSequenceNumber}?organizationType=SUPPLIER`;
let response = await fetch(query, {
method: 'GET',
headers
});
let data = await response.json();
let response = await axios.get(`${env.API_URL}/organizations/sync/${curSequenceNumber}?organizationType=SUPPLIER`, { headers });
let data = response.data;
let suppliers = data.results;
for (let supplier of suppliers) {
@ -351,27 +343,19 @@ export async function syncConn(){
'X-Api-Key': process.env.API_KEY
};
let remoteConnections = await fetch(`${env.API_URL}/connections`, {
method: 'GET',
headers
});
remoteConnections = await remoteConnections.json();
let remoteConnections = await axios.get(`${env.API_URL}/connections`, { headers });
let i = 1;
for (let connection of connections){
spinner.text = `Syncing ${i++} connections...`
if (connection.isConnected == false)
continue;
let remoteConnection = remoteConnections.find(remoteConnection => remoteConnection == connection.supplierOrganizationId);
let remoteConnection = remoteConnections.data.find(remoteConnection => remoteConnection == connection.supplierOrganizationId);
if (remoteConnection == undefined){
console.log('Connection: ', connection, 'does not exist in the remote server');
console.log('Creating remote connection');
await fetch(`${env.API_URL}/connections/${connection.supplierOrganizationId}`, {
method: 'PUT',
headers
});
await axios.put(`${env.API_URL}/connections/${connection.supplierOrganizationId}`, { headers });
await models.connection.update({ isConnected: true }, {
where: {
supplierOrganizationId: connection.supplierOrganizationId
@ -415,18 +399,15 @@ export async function syncTradeItems(){
'X-Api-Key': process.env.API_KEY
};
try {
let request = await fetch(`${env.API_URL}/trade-items?supplierOrganizationId=${supplier.supplierOrganizationId}`, {
method: 'GET',
headers
});
let request = await axios.get(`${env.API_URL}/trade-items?supplierOrganizationId=${supplier.supplierOrganizationId}`, { headers })
let tradeItems = await request.json();
let tradeItems = request.data;
if (!tradeItems.length)
continue;
for (let tradeItem of tradeItems) {
await insertItem(tradeItem, supplier);
await insertItem(tradeItem);
spinner.text = `Syncing ${i++} trade items...`
};
} catch (err) {
@ -439,7 +420,7 @@ export async function syncTradeItems(){
/**
* Syncs the supply lines for suppliers that are connected to do this,
* it fetches all the supply lines for every tradeitem of the suppliers
* it searches all the supply lines for every tradeitem of the suppliers
*/
export async function syncSupplyLines(){
const spinner = ora(`Syncing supply lines...`).start();
@ -476,17 +457,14 @@ export async function syncSupplyLines(){
tradeItemId: tradeItem.tradeItemId,
postFilterSelectedTradeItems: false
});
let request = await fetch(`${url}?${params.toString()}`, {
method: 'GET',
headers
});
let request = await axios.get(`${url}?${params.toString()}`, { headers });
if (request.status == 429) { // Too many request
resolve([]);
return;
}
let supplyLines = await request.json();
let supplyLines = request.data;
if (!supplyLines.results.length) {
resolve([]);
@ -524,12 +502,9 @@ export async function syncSupplyLines(){
console.log('Trade item not found for supply line: ', line.supplyLineId);
console.log('Requesting data for trade item id: ', line.tradeItemId);
let queryTradeItem = await fetch(`${env.API_URL}/trade-items?tradeItemIds=${line.tradeItemId}`, {
method: 'GET',
headers
});
let queryTradeItem = await axios.get(`${env.API_URL}/trade-items?tradeItemIds=${line.tradeItemId}`, { headers });
let tradeItem = await queryTradeItem.json();
let tradeItem = queryTradeItem.data;
if (tradeItem.length == 0) {
console.log('Trade item not found for supply line: ', line.supplyLineId);
@ -537,13 +512,7 @@ export async function syncSupplyLines(){
continue;
}
let supplier = await models.supplier.findOne({
where: {
supplierOrganizationId: tradeItem[0].supplierOrganizationId
}
});
await insertItem(tradeItem[0], supplier);
await insertItem(tradeItem[0]);
tradeItem = await models.tradeItem.findOne({
where: {
@ -604,9 +573,15 @@ export async function syncSupplyLines(){
}
}
export async function insertItem(tradeItem, supplier) {
/**
* Insert the trade item
*
* @param {array} tradeItem
*/
export async function insertItem(tradeItem) {
let tx;
try {
const tx = await models.sequelize.transaction();
tx = await models.sequelize.transaction();
// Upsert supplier connection
await models.connection.upsert({
@ -623,14 +598,15 @@ export async function insertItem(tradeItem, supplier) {
}, { transaction: tx });
// Upsert characteristics
if (tradeItem.characteristics)
for (const characteristic of tradeItem.characteristics) {
await models.characteristic.upsert({
...characteristic,
tradeItemId: tradeItem.tradeItemId,
}, { transaction: tx });
}
// Upsert seasonal periods
if (tradeItem.seasonalPeriods)
for (const seasonalPeriod of tradeItem.seasonalPeriods) {
await models.seasonalPeriod.upsert({
...seasonalPeriod,
@ -639,6 +615,7 @@ export async function insertItem(tradeItem, supplier) {
}
// Upsert photos
if (tradeItem.photos)
for (const photo of tradeItem.photos) {
await models.photo.upsert({
...photo,
@ -647,6 +624,7 @@ export async function insertItem(tradeItem, supplier) {
}
// Upsert packing configurations
if (tradeItem.packingConfigurations)
for (const packingConfiguration of tradeItem.packingConfigurations) {
const uuid = uuidv4();
@ -664,6 +642,7 @@ export async function insertItem(tradeItem, supplier) {
}
// Upsert country of origin ISO codes
if (tradeItem.countryOfOriginIsoCodes)
for (const isoCode of tradeItem.countryOfOriginIsoCodes || []) {
await models.countryOfOriginIsoCode.upsert({
isoCode,
@ -672,6 +651,7 @@ export async function insertItem(tradeItem, supplier) {
}
// Upsert botanical names
if (tradeItem.botanicalNames)
for (const botanicalName of tradeItem.botanicalNames) {
await models.botanicalName.upsert({
...botanicalName,
@ -681,8 +661,8 @@ export async function insertItem(tradeItem, supplier) {
await tx.commit();
} catch (err) {
console.log(err);
await tx.rollback();
throw err;
}
}