Merge pull request '4164 - Endpoint refactor' (#996) from 4164-edi_updateData into dev
gitea/salix/pipeline/head This commit looks good
Details
gitea/salix/pipeline/head This commit looks good
Details
Reviewed-on: #996 Reviewed-by: Carlos Jimenez Ruiz <carlosjr@verdnatura.es> Reviewed-by: Juan Ferrer <juan@verdnatura.es>
This commit is contained in:
commit
d08e1cee23
|
|
@ -19,31 +19,42 @@ module.exports = Self => {
|
|||
Self.updateData = async() => {
|
||||
const models = Self.app.models;
|
||||
|
||||
// Get files checksum
|
||||
const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig');
|
||||
|
||||
const updatableFiles = [];
|
||||
for (const file of files) {
|
||||
const fileChecksum = await getChecksum(file);
|
||||
|
||||
if (file.checksum != fileChecksum) {
|
||||
updatableFiles.push({
|
||||
name: file.name,
|
||||
checksum: fileChecksum
|
||||
});
|
||||
} else
|
||||
console.debug(`File already updated, skipping...`);
|
||||
}
|
||||
|
||||
if (updatableFiles.length === 0)
|
||||
return false;
|
||||
|
||||
// Download files
|
||||
const container = await models.TempContainer.container('edi');
|
||||
const tempPath = path.join(container.client.root, container.name);
|
||||
|
||||
// Temporary file clean
|
||||
await fs.rmdir(`${tempPath}/*`, {recursive: true});
|
||||
|
||||
const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig');
|
||||
console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`);
|
||||
|
||||
const FtpClient = require('ftps');
|
||||
const ftpClient = new FtpClient({
|
||||
host: ftpConfig.host,
|
||||
username: ftpConfig.user,
|
||||
password: ftpConfig.password,
|
||||
procotol: 'ftp'
|
||||
});
|
||||
|
||||
const files = await Self.rawSql('SELECT fileName, toTable, file, updated FROM edi.fileConfig');
|
||||
|
||||
let remoteFile;
|
||||
let tempDir;
|
||||
let tempFile;
|
||||
for (const file of files) {
|
||||
try {
|
||||
const fileName = file.file;
|
||||
|
||||
const fileNames = updatableFiles.map(file => file.name);
|
||||
|
||||
const tables = await Self.rawSql(`
|
||||
SELECT fileName, toTable, file
|
||||
FROM edi.tableConfig
|
||||
WHERE file IN (?)`, [fileNames]);
|
||||
|
||||
for (const table of tables) {
|
||||
const fileName = table.file;
|
||||
|
||||
console.debug(`Downloading file ${fileName}...`);
|
||||
|
||||
|
|
@ -51,58 +62,143 @@ module.exports = Self => {
|
|||
tempDir = `${tempPath}/${fileName}`;
|
||||
tempFile = `${tempPath}/${fileName}.zip`;
|
||||
|
||||
// if (fs.existsSync(tempFile))
|
||||
// await fs.unlink(tempFile);
|
||||
|
||||
// if (fs.existsSync(tempDir))
|
||||
// await fs.rmdir(tempDir, {recursive: true});
|
||||
|
||||
await extractFile({
|
||||
ftpClient: ftpClient,
|
||||
file: file,
|
||||
paths: {
|
||||
remoteFile: remoteFile,
|
||||
tempDir: tempDir,
|
||||
tempFile: tempFile
|
||||
}
|
||||
});
|
||||
try {
|
||||
await fs.readFile(tempFile);
|
||||
} catch (error) {
|
||||
if (fs.existsSync(tempFile))
|
||||
await fs.unlink(tempFile);
|
||||
|
||||
console.error(error);
|
||||
if (error.code === 'ENOENT') {
|
||||
const downloadOutput = await downloadFile(remoteFile, tempFile);
|
||||
if (downloadOutput.error)
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
console.debug(`Extracting file ${fileName}...`);
|
||||
await extractFile(tempFile, tempDir);
|
||||
|
||||
console.debug(`Updating table ${table.toTable}...`);
|
||||
await dumpData(tempDir, table);
|
||||
}
|
||||
|
||||
// Update files checksum
|
||||
for (const file of updatableFiles) {
|
||||
await Self.rawSql(`
|
||||
UPDATE edi.fileConfig
|
||||
SET checksum = ?
|
||||
WHERE name = ?`,
|
||||
[file.checksum, file.name]);
|
||||
}
|
||||
|
||||
// Clean files
|
||||
try {
|
||||
await fs.remove(tempPath);
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT')
|
||||
throw e;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
async function extractFile({ftpClient, file, paths}) {
|
||||
// Download the zip file
|
||||
ftpClient.get(paths.remoteFile, paths.tempFile);
|
||||
let ftpClient;
|
||||
async function getFtpClient() {
|
||||
if (!ftpClient) {
|
||||
const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig');
|
||||
console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`);
|
||||
|
||||
// Execute download command
|
||||
ftpClient.exec(async(err, response) => {
|
||||
if (response.error) {
|
||||
console.debug(`Error downloading file... ${response.error}`);
|
||||
return;
|
||||
}
|
||||
const FtpClient = require('ftps');
|
||||
|
||||
const AdmZip = require('adm-zip');
|
||||
const zip = new AdmZip(paths.tempFile);
|
||||
const entries = zip.getEntries();
|
||||
|
||||
zip.extractAllTo(paths.tempDir, false);
|
||||
|
||||
await dumpData({file, entries, paths});
|
||||
|
||||
await fs.rmdir(paths.tempDir, {recursive: true});
|
||||
ftpClient = new FtpClient({
|
||||
host: ftpConfig.host,
|
||||
username: ftpConfig.user,
|
||||
password: ftpConfig.password,
|
||||
procotol: 'ftp'
|
||||
});
|
||||
}
|
||||
|
||||
async function dumpData({file, entries, paths}) {
|
||||
const toTable = file.toTable;
|
||||
const baseName = file.fileName;
|
||||
return ftpClient;
|
||||
}
|
||||
|
||||
async function getChecksum(file) {
|
||||
const ftpClient = await getFtpClient();
|
||||
console.debug(`Checking checksum for file ${file.name}...`);
|
||||
|
||||
ftpClient.cat(`codes/${file.name}.txt`);
|
||||
|
||||
const response = await new Promise((resolve, reject) => {
|
||||
ftpClient.exec((err, response) => {
|
||||
if (response.error) {
|
||||
console.debug(`Error downloading checksum file... ${response.error}`);
|
||||
reject(err);
|
||||
}
|
||||
|
||||
resolve(response);
|
||||
});
|
||||
});
|
||||
|
||||
if (response && response.data) {
|
||||
const fileContents = response.data;
|
||||
const rows = fileContents.split('\n');
|
||||
const row = rows[4];
|
||||
const columns = row.split(/\s+/);
|
||||
|
||||
let fileChecksum;
|
||||
if (file.keyValue)
|
||||
fileChecksum = columns[1];
|
||||
|
||||
if (!file.keyValue)
|
||||
fileChecksum = columns[0];
|
||||
|
||||
return fileChecksum;
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadFile(remoteFile, tempFile) {
|
||||
const ftpClient = await getFtpClient();
|
||||
|
||||
ftpClient.get(remoteFile, tempFile);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
ftpClient.exec((err, response) => {
|
||||
if (response.error) {
|
||||
console.debug(`Error downloading file... ${response.error}`);
|
||||
reject(err);
|
||||
}
|
||||
|
||||
resolve(response);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function extractFile(tempFile, tempDir) {
|
||||
const JSZip = require('jszip');
|
||||
|
||||
try {
|
||||
await fs.mkdir(tempDir);
|
||||
} catch (error) {
|
||||
if (error.code !== 'EEXIST')
|
||||
throw e;
|
||||
}
|
||||
|
||||
const fileStream = await fs.readFile(tempFile);
|
||||
if (fileStream) {
|
||||
const zip = new JSZip();
|
||||
const zipContents = await zip.loadAsync(fileStream);
|
||||
|
||||
if (!zipContents) return;
|
||||
|
||||
const fileNames = Object.keys(zipContents.files);
|
||||
|
||||
for (const fileName of fileNames) {
|
||||
const fileContent = await zip.file(fileName).async('nodebuffer');
|
||||
const dest = path.join(tempDir, fileName);
|
||||
await fs.writeFile(dest, fileContent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function dumpData(tempDir, table) {
|
||||
const toTable = table.toTable;
|
||||
const baseName = table.fileName;
|
||||
|
||||
const tx = await Self.beginTransaction({});
|
||||
|
||||
|
|
@ -112,44 +208,22 @@ module.exports = Self => {
|
|||
const tableName = `edi.${toTable}`;
|
||||
await Self.rawSql(`DELETE FROM ??`, [tableName], options);
|
||||
|
||||
for (const zipEntry of entries) {
|
||||
const entryName = zipEntry.entryName;
|
||||
console.log(`Reading file ${entryName}...`);
|
||||
const dirFiles = await fs.readdir(tempDir);
|
||||
const files = dirFiles.filter(file => file.startsWith(baseName));
|
||||
|
||||
const startIndex = (entryName.length - 10);
|
||||
const endIndex = (entryName.length - 4);
|
||||
const dateString = entryName.substring(startIndex, endIndex);
|
||||
const lastUpdated = new Date();
|
||||
for (const file of files) {
|
||||
console.log(`Dumping data from file ${file}...`);
|
||||
|
||||
// Format string date to a date object
|
||||
let updated = null;
|
||||
if (file.updated) {
|
||||
updated = new Date(file.updated);
|
||||
updated.setHours(0, 0, 0, 0);
|
||||
}
|
||||
|
||||
lastUpdated.setFullYear(`20${dateString.substring(4, 6)}`);
|
||||
lastUpdated.setMonth(parseInt(dateString.substring(2, 4)) - 1);
|
||||
lastUpdated.setDate(dateString.substring(0, 2));
|
||||
lastUpdated.setHours(0, 0, 0, 0);
|
||||
|
||||
if (updated && lastUpdated <= updated) {
|
||||
console.debug(`Table ${toTable} already updated, skipping...`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log('Dumping data...');
|
||||
const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
|
||||
const sqlTemplate = fs.readFileSync(templatePath, 'utf8');
|
||||
const sqlTemplate = await fs.readFile(templatePath, 'utf8');
|
||||
const filePath = path.join(tempDir, file);
|
||||
|
||||
const rawPath = path.join(paths.tempDir, entryName);
|
||||
|
||||
await Self.rawSql(sqlTemplate, [rawPath], options);
|
||||
await Self.rawSql(sqlTemplate, [filePath], options);
|
||||
await Self.rawSql(`
|
||||
UPDATE edi.fileConfig
|
||||
UPDATE edi.tableConfig
|
||||
SET updated = ?
|
||||
WHERE fileName = ?
|
||||
`, [lastUpdated, baseName], options);
|
||||
`, [new Date(), baseName], options);
|
||||
}
|
||||
|
||||
tx.commit();
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
RENAME TABLE `edi`.`fileConfig` to `edi`.`tableConfig`;
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
CREATE TABLE `edi`.`fileConfig`
|
||||
(
|
||||
name varchar(25) NOT NULL,
|
||||
checksum text NULL,
|
||||
keyValue tinyint(1) default true NOT NULL,
|
||||
constraint fileConfig_pk
|
||||
primary key (name)
|
||||
);
|
||||
|
||||
create unique index fileConfig_name_uindex
|
||||
on `edi`.`fileConfig` (name);
|
||||
|
||||
|
||||
INSERT INTO `edi`.`fileConfig` (name, checksum, keyValue)
|
||||
VALUES ('FEC010104', null, 0);
|
||||
|
||||
INSERT INTO `edi`.`fileConfig` (name, checksum, keyValue)
|
||||
VALUES ('VBN020101', null, 1);
|
||||
|
||||
INSERT INTO `edi`.`fileConfig` (name, checksum, keyValue)
|
||||
VALUES ('florecompc2', null, 1);
|
||||
|
||||
|
|
@ -85,7 +85,6 @@
|
|||
}
|
||||
.icon-bucket:before {
|
||||
content: "\e97a";
|
||||
color: #000;
|
||||
}
|
||||
.icon-buscaman:before {
|
||||
content: "\e93b";
|
||||
|
|
@ -95,32 +94,26 @@
|
|||
}
|
||||
.icon-calc_volum .path1:before {
|
||||
content: "\e915";
|
||||
color: rgb(0, 0, 0);
|
||||
}
|
||||
.icon-calc_volum .path2:before {
|
||||
content: "\e916";
|
||||
margin-left: -1em;
|
||||
color: rgb(0, 0, 0);
|
||||
}
|
||||
.icon-calc_volum .path3:before {
|
||||
content: "\e917";
|
||||
margin-left: -1em;
|
||||
color: rgb(0, 0, 0);
|
||||
}
|
||||
.icon-calc_volum .path4:before {
|
||||
content: "\e918";
|
||||
margin-left: -1em;
|
||||
color: rgb(0, 0, 0);
|
||||
}
|
||||
.icon-calc_volum .path5:before {
|
||||
content: "\e919";
|
||||
margin-left: -1em;
|
||||
color: rgb(0, 0, 0);
|
||||
}
|
||||
.icon-calc_volum .path6:before {
|
||||
content: "\e91a";
|
||||
margin-left: -1em;
|
||||
color: rgb(255, 255, 255);
|
||||
}
|
||||
.icon-calendar:before {
|
||||
content: "\e93d";
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@
|
|||
"node": ">=14"
|
||||
},
|
||||
"dependencies": {
|
||||
"adm-zip": "^0.5.9",
|
||||
"axios": "^0.25.0",
|
||||
"bmp-js": "^0.1.0",
|
||||
"compression": "^1.7.3",
|
||||
|
|
@ -23,6 +22,7 @@
|
|||
"i18n": "^0.8.4",
|
||||
"image-type": "^4.1.0",
|
||||
"imap": "^0.8.19",
|
||||
"jszip": "^3.10.0",
|
||||
"ldapjs": "^2.2.0",
|
||||
"loopback": "^3.26.0",
|
||||
"loopback-boot": "3.3.1",
|
||||
|
|
|
|||
Loading…
Reference in New Issue