mycdc/consumer.js

require('require-yaml');
require('colors');
const fs = require('fs');
const path = require('path');
const mysql = require('mysql2/promise');
const amqp = require('amqplib');
const {cpus} = require('os');


class Consumer {
  async start() {
    const defaultConfig = require('./config.yml');
    const config = this.config = Object.assign({}, defaultConfig);
    const localPath = path.join(__dirname, 'config.local.yml');
    if (fs.existsSync(localPath)) {
      const localConfig = require(localPath);
      Object.assign(config, localConfig);
    }

    if (config.testMode)
      console.log('Test mode enabled, just logging queries to console.');

    console.log('Starting process.');
    await this.init();
    console.log('Process started.');

    await this.consumeQueues();
  }

  async stop() {
    console.log('Stopping process.');
    await this.end();
    console.log('Process stopped.');
  }

  async init() {
    const config = this.config;
    this.onErrorListener = err => this.onError(err);

    const dbConfig = Object.assign({
      connectionLimit: cpus().length
    }, config.consumerDb);

    this.db = await mysql.createPool(dbConfig);
    this.db.on('error', this.onErrorListener);

    this.consumer = await amqp.connect(config.amqp);
    this.channel = await this.consumer.createChannel();
    this.channel.prefetch(1);
  }

  async consumeQueues() {
    for (const queueName in this.config.queues) {
      await this.channel.assertQueue(queueName, {
        durable: true
      });
      await this.channel.consume(queueName,
        msg => this.onConsume(msg, queueName));
    }
  }

  async end(silent) {
    await this.consumer.close();

    this.db.off('error', this.onErrorListener);
    // FIXME: mysql2/promise bug, db.end() ends process
    this.db.on('error', () => {});
    try {
      await this.db.end();
    } catch (err) {
      if (!silent)
        console.error(err);
    }
  }

  async onConsume(msg, queueName) {
    const config = this.config;
    const data = JSON.parse(msg.content.toString());

    if (config.debug)
      console.debug('Message:'.blue, queueName.yellow, data.table);

    const queue = config.queues[queueName];
    let query = queue.query;
    if (!query) return;

    // XXX: Testing
    //query = 'SELECT 1 sleep';

    switch(queue.mode) {
    case 'fk':
      for (const fk of data.fks) {
        const sql = this.db.format(query, fk);
        this.debug('SQL', sql);
        if (!config.testMode)
          await this.db.query(query, fk);
      }
      break;
    case 'changes':
      const queueTable = queue.includeSchema[data.schema][data.table];
      for (const row of data.rows) {
        const sql = this.db.format(query, [
          data.table,
          row[queueTable.key],
          JSON.stringify(row)
        ]);
        this.debug('SQL', sql);
        if (!config.testMode)
          await this.db.query(query, row);
      }
      break;
    }

    await this.channel.ack(msg);
  }

  async tryRestart() {
    try {
      await this.init();
      console.log('Process restarted.');
    } catch(err) {
      setTimeout(() => this.tryRestart(), 30);
    }
  }

  async onError(err) {
    console.log(`Error: ${err.code}: ${err.message}`);
    try {
      await this.end(true);
    } catch(e) {}

    switch (err.code) {
      case 'PROTOCOL_CONNECTION_LOST':
      case 'ECONNRESET':
      console.log('Trying to restart process.');
      await this.tryRestart();
      break;
    default:
      process.exit();
    }
  }

  debug(namespace, message) {
    if (this.config.debug)
      console.debug(`${namespace}:`.blue, message.yellow);
  }
}

async function main() {
  const consumer = new Consumer()
  await consumer.start();

  process.on('SIGINT', async function() {
    console.log('Got SIGINT.');
    try {
      await consumer.stop();
    } catch (err) {
      console.error(err);
    }
    process.exit();
  });
}
main();