salix/back/methods/edi/updateData.js

/* eslint no-console: "off" */
const path = require('path');
const fs = require('fs-extra');

module.exports = Self => {
    Self.remoteMethodCtx('updateData', {
        description: 'Updates schema data from external provider',
        accessType: 'WRITE',
        returns: {
            type: 'object',
            root: true
        },
        http: {
            path: `/updateData`,
            verb: 'GET'
        }
    });

    Self.updateData = async() => {
        const models = Self.app.models;

        // Get files checksum
        const files = await Self.rawSql('SELECT name, checksum, keyValue FROM edi.fileConfig');

        const updatableFiles = [];
        for (const file of files) {
            const fileChecksum = await getChecksum(file);

            if (file.checksum != fileChecksum) {
                updatableFiles.push({
                    name: file.name,
                    checksum: fileChecksum
                });
            } else
                console.debug(`File already updated, skipping...`);
        }

        if (updatableFiles.length === 0)
            return false;

        // Download files
        const container = await models.TempContainer.container('edi');
        const tempPath = path.join(container.client.root, container.name);

        let remoteFile;
        let tempDir;
        let tempFile;

        const fileNames = updatableFiles.map(file => file.name);

        const tables = await Self.rawSql(`
            SELECT fileName, toTable, file
                FROM edi.tableConfig
            WHERE file IN (?)`, [fileNames]);

        for (const table of tables) {
            const fileName = table.file;

            console.debug(`Downloading file ${fileName}...`);

            remoteFile = `codes/${fileName}.ZIP`;
            tempDir = `${tempPath}/${fileName}`;
            tempFile = `${tempPath}/${fileName}.zip`;

            try {
                await fs.readFile(tempFile);
            } catch (error) {
                if (error.code === 'ENOENT') {
                    const downloadOutput = await downloadFile(remoteFile, tempFile);
                    if (downloadOutput.error)
                        continue;
                }
            }

            console.debug(`Extracting file ${fileName}...`);
            await extractFile(tempFile, tempDir);

            console.debug(`Updating table ${table.toTable}...`);
            await dumpData(tempDir, table);
        }

        // Update files checksum
        for (const file of updatableFiles) {
            await Self.rawSql(`
                    UPDATE edi.fileConfig
                        SET checksum = ?
                    WHERE name = ?`,
            [file.checksum, file.name]);
        }

        // Clean files
        try {
            await fs.remove(tempPath);
        } catch (error) {
            if (error.code !== 'ENOENT')
                throw e;
        }

        return true;
    };

    let ftpClient;
    async function getFtpClient() {
        if (!ftpClient) {
            const [ftpConfig] = await Self.rawSql('SELECT host, user, password FROM edi.ftpConfig');
            console.debug(`Openning FTP connection to ${ftpConfig.host}...\n`);

            const FtpClient = require('ftps');

            ftpClient = new FtpClient({
                host: ftpConfig.host,
                username: ftpConfig.user,
                password: ftpConfig.password,
                procotol: 'ftp'
            });
        }

        return ftpClient;
    }

    async function getChecksum(file) {
        const ftpClient = await getFtpClient();
        console.debug(`Checking checksum for file ${file.name}...`);

        ftpClient.cat(`codes/${file.name}.txt`);

        const response = await new Promise((resolve, reject) => {
            ftpClient.exec((err, response) => {
                if (response.error) {
                    console.debug(`Error downloading checksum file... ${response.error}`);
                    reject(err);
                }

                resolve(response);
            });
        });

        if (response && response.data) {
            const fileContents = response.data;
            const rows = fileContents.split('\n');
            const row = rows[4];
            const columns = row.split(/\s+/);

            let fileChecksum;
            if (file.keyValue)
                fileChecksum = columns[1];

            if (!file.keyValue)
                fileChecksum = columns[0];

            return fileChecksum;
        }
    }

    async function downloadFile(remoteFile, tempFile) {
        const ftpClient = await getFtpClient();

        ftpClient.get(remoteFile, tempFile);

        return new Promise((resolve, reject) => {
            ftpClient.exec((err, response) => {
                if (response.error) {
                    console.debug(`Error downloading file... ${response.error}`);
                    reject(err);
                }

                resolve(response);
            });
        });
    }

    async function extractFile(tempFile, tempDir) {
        const JSZip = require('jszip');

        try {
            await fs.mkdir(tempDir);
        } catch (error) {
            if (error.code !== 'EEXIST')
                throw e;
        }

        const fileStream = await fs.readFile(tempFile);
        if (fileStream) {
            const zip = new JSZip();
            const zipContents = await zip.loadAsync(fileStream);

            if (!zipContents) return;

            const fileNames = Object.keys(zipContents.files);

            for (const fileName of fileNames) {
                const fileContent = await zip.file(fileName).async('nodebuffer');
                const dest = path.join(tempDir, fileName);
                await fs.writeFile(dest, fileContent);
            }
        }
    }

    async function dumpData(tempDir, table) {
        const toTable = table.toTable;
        const baseName = table.fileName;

        const firstEntry = entries[0];
        const entryName = firstEntry.entryName;
        const startIndex = (entryName.length - 10);
        const endIndex = (entryName.length - 4);
        const dateString = entryName.substring(startIndex, endIndex);

        const lastUpdated = new Date();

        let updated = null;
        if (file.updated) {
            updated = new Date(file.updated);
            updated.setHours(0, 0, 0, 0);
        }

        // Format string date to a date object
        lastUpdated.setFullYear(`20${dateString.substring(4, 6)}`);
        lastUpdated.setMonth(parseInt(dateString.substring(2, 4)) - 1);
        lastUpdated.setDate(dateString.substring(0, 2));
        lastUpdated.setHours(0, 0, 0, 0);

        if (updated && lastUpdated <= updated) {
            console.debug(`Table ${toTable} already updated, skipping...`);
            return;
        }

        const tx = await Self.beginTransaction({});

        try {
            const options = {transaction: tx};

            const tableName = `edi.${toTable}`;
            await Self.rawSql(`DELETE FROM ??`, [tableName], options);

            const dirFiles = await fs.readdir(tempDir);
            const files = dirFiles.filter(file => file.startsWith(baseName));

            for (const file of files) {
                console.log(`Dumping data from file ${file}...`);

                const templatePath = path.join(__dirname, `./sql/${toTable}.sql`);
                const sqlTemplate = await fs.readFile(templatePath, 'utf8');
                const filePath = path.join(tempDir, file);

                await Self.rawSql(sqlTemplate, [filePath], options);
                await Self.rawSql(`
                        UPDATE edi.tableConfig
                            SET updated = ?
                        WHERE fileName = ?
                `, [new Date(), baseName], options);
            }

            tx.commit();
        } catch (error) {
            tx.rollback();
            throw error;
        }
        console.log(`Updated table ${toTable}\n`);
    }
};