require('require-yaml'); require('colors'); const fs = require('fs'); const path = require('path'); const mysql = require('mysql2/promise'); const amqp = require('amqplib'); const {cpus} = require('os'); class Consumer { async start() { const defaultConfig = require('./config.yml'); const config = this.config = Object.assign({}, defaultConfig); const localPath = path.join(__dirname, 'config.local.yml'); if (fs.existsSync(localPath)) { const localConfig = require(localPath); Object.assign(config, localConfig); } if (config.testMode) console.log('Test mode enabled, just logging queries to console.'); console.log('Starting process.'); await this.init(); console.log('Process started.'); await this.consumeQueues(); } async stop() { console.log('Stopping process.'); await this.end(); console.log('Process stopped.'); } async init() { const config = this.config; this.onErrorListener = err => this.onError(err); const dbConfig = Object.assign({ connectionLimit: cpus().length }, config.consumerDb); this.db = await mysql.createPool(dbConfig); this.db.on('error', this.onErrorListener); this.consumer = await amqp.connect(config.amqp); this.channel = await this.consumer.createChannel(); this.channel.prefetch(1); } async consumeQueues() { for (const queueName in this.config.queues) { await this.channel.assertQueue(queueName, { durable: true }); await this.channel.consume(queueName, msg => this.onConsume(msg, queueName)); } } async end(silent) { await this.consumer.close(); this.db.off('error', this.onErrorListener); // FIXME: mysql2/promise bug, db.end() ends process this.db.on('error', () => {}); try { await this.db.end(); } catch (err) { if (!silent) console.error(err); } } async onConsume(msg, queueName) { const config = this.config; const data = JSON.parse(msg.content.toString()); if (config.debug) console.debug('Message:'.blue, queueName.yellow, data.table); const queue = config.queues[queueName]; let query = queue.query; if (!query) return; // XXX: Testing //query = 'SELECT 1 sleep'; switch(queue.mode) { case 'fk': for (const fk of data.fks) { const sql = this.db.format(query, fk); this.debug('SQL', sql); if (!config.testMode) await this.db.query(query, fk); } break; case 'changes': const queueTable = queue.includeSchema[data.schema][data.table]; for (const row of data.rows) { const sql = this.db.format(query, [ data.table, row[queueTable.key], JSON.stringify(row) ]); this.debug('SQL', sql); if (!config.testMode) await this.db.query(query, row); } break; } await this.channel.ack(msg); } async tryRestart() { try { await this.init(); console.log('Process restarted.'); } catch(err) { setTimeout(() => this.tryRestart(), 30); } } async onError(err) { console.log(`Error: ${err.code}: ${err.message}`); try { await this.end(true); } catch(e) {} switch (err.code) { case 'PROTOCOL_CONNECTION_LOST': case 'ECONNRESET': console.log('Trying to restart process.'); await this.tryRestart(); break; default: process.exit(); } } debug(namespace, message) { if (this.config.debug) console.debug(`${namespace}:`.blue, message.yellow); } } async function main() { const consumer = new Consumer() await consumer.start(); process.on('SIGINT', async function() { console.log('Got SIGINT.'); try { await consumer.stop(); } catch (err) { console.error(err); } process.exit(); }); } main();