salix/back/methods/edi/updateData.js

/* eslint no-console: "off" */
const path = require('path');
const fs = require('fs-extra');

module.exports = Self => {
    Self.remoteMethodCtx('updateData', {
        description: 'Updates schema data from external provider',
        accessType: 'WRITE',
        returns: {
            type: 'object',
            root: true
        },
        http: {
            path: `/updateData`,
            verb: 'GET'
        }
    });

    Self.updateData = async() => {
        const models = Self.app.models;

        // Get files checksum
        const tx = await Self.beginTransaction({});

        try {
            const options = {transaction: tx};
            const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig', null, options);

            const updatableFiles = [];
            for (const file of files) {
                const fileChecksum = await getChecksum(file);

                if (file.checksum != fileChecksum) {
                    updatableFiles.push({
                        name: file.name,
                        checksum: fileChecksum
                    });
                } else
                    console.debug(`File already updated, skipping...`);
            }

            if (updatableFiles.length === 0)
                return false;

            // Download files
            const container = await models.TempContainer.container('edi');
            const tempPath = path.join(container.client.root, container.name);

            let remoteFile;
            let tempDir;
            let tempFile;

            const fileNames = updatableFiles.map(file => file.name);

            const tables = await Self.rawSql(`
                SELECT fileName, toTable, file
                    FROM edi.tableConfig
                WHERE file IN (?)`, [fileNames], options);

            for (const table of tables) {
                const fileName = table.file;

                remoteFile = `codes/${fileName}.ZIP`;
                tempDir = `${tempPath}/${fileName}`;
                tempFile = `${tempPath}/${fileName}.zip`;

                try {
                    await fs.readFile(tempFile);
                } catch (error) {
                    if (error.code === 'ENOENT') {
                        console.debug(`Downloading file ${fileName}...`);
                        const downloadOutput = await downloadFile(remoteFile, tempFile);
                        if (downloadOutput.error)
                            continue;
                    }
                }

                await extractFile(fileName, tempFile, tempDir);

                console.debug(`Updating table ${table.toTable}...`);
                await dumpData(tempDir, table, options);
            }

            // Update files checksum
            for (const file of updatableFiles) {
                console.log(`Updating file ${file.name} checksum...`);
                await Self.rawSql(`
                        UPDATE edi.fileConfig
                            SET checksum = ?
                        WHERE name = ?`,
                [file.checksum, file.name], options);
            }

            await tx.commit();

            // Clean files
            try {
                console.debug(`Cleaning files...`);
                await fs.remove(tempPath);
            } catch (error) {
                if (error.code !== 'ENOENT')
                    throw e;
            }
        } catch (error) {
            await tx.rollback();
            throw error;
        }

        return true;
    };

    let ftpClient;
    async function getFtpClient() {
        if (!ftpClient) {
            const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig');
            console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`);

            const FtpClient = require('ftps');

            ftpClient = new FtpClient({
                host: ftpConfig.host,
                username: ftpConfig.user,
                password: ftpConfig.password,
                procotol: 'ftp'
            });
        }

        return ftpClient;
    }

    async function getChecksum(file) {
        const ftpClient = await getFtpClient();
        console.debug(`Checking checksum for file ${file.name}...`);

        ftpClient.cat(`codes/${file.name}.txt`);

        const response = await new Promise((resolve, reject) => {
            ftpClient.exec((err, response) => {
                if (err || response.error) {
                    console.debug(`Error downloading checksum file... ${response.error}`);
                    return reject(err);
                }

                resolve(response);
            });
        });

        if (response && response.data) {
            const fileContents = response.data;
            const rows = fileContents.split('\n');
            const row = rows[4];
            const columns = row.split(/\s+/);

            let fileChecksum;
            if (file.keyValue)
                fileChecksum = columns[1];

            if (!file.keyValue)
                fileChecksum = columns[0];

            return fileChecksum;
        }
    }

    async function downloadFile(remoteFile, tempFile) {
        const ftpClient = await getFtpClient();

        ftpClient.get(remoteFile, tempFile);

        return new Promise((resolve, reject) => {
            ftpClient.exec((err, response) => {
                if (err || response.error) {
                    console.debug(`Error downloading file... ${response.error}`);
                    return reject(err);
                }

                resolve(response);
            });
        });
    }

    async function extractFile(fileName, tempFile, tempDir) {
        const JSZip = require('jszip');

        try {
            await fs.mkdir(tempDir);
            console.debug(`Extracting file ${fileName}...`);
        } catch (error) {
            if (error.code !== 'EEXIST')
                throw e;
        }

        const fileStream = await fs.readFile(tempFile);
        if (fileStream) {
            const zip = new JSZip();
            const zipContents = await zip.loadAsync(fileStream);

            if (!zipContents) return;

            const fileNames = Object.keys(zipContents.files);

            for (const fileName of fileNames) {
                const fileContent = await zip.file(fileName).async('nodebuffer');
                const dest = path.join(tempDir, fileName);
                await fs.writeFile(dest, fileContent);
            }
        }
    }

    async function dumpData(tempDir, table, options) {
        const toTable = table.toTable;
        const baseName = table.fileName;

        console.log(`Emptying table ${toTable}...`);
        const tableName = `edi.${toTable}`;
        await Self.rawSql(`DELETE FROM ??`, [tableName]);

        const dirFiles = await fs.readdir(tempDir);
        const files = dirFiles.filter(file => file.startsWith(baseName));

        for (const file of files) {
            console.log(`Dumping data from file ${file}...`);

            const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
            const sqlTemplate = await fs.readFile(templatePath, 'utf8');
            const filePath = path.join(tempDir, file);

            await Self.rawSql(sqlTemplate, [filePath], options);
            await Self.rawSql(`
                        UPDATE edi.tableConfig
                            SET updated = ?
                        WHERE fileName = ?
                `, [new Date(), baseName], options);
        }

        console.log(`Updated table ${toTable}\n`);
    }
};