Highways4Good/api/node_modules/kafka-node/lib/consumer.js

'use strict';

var util = require('util');
var _ = require('lodash');
var EventEmitter = require('events');
var logger = require('./logging')('kafka-node:Consumer');
var utils = require('./utils');

var DEFAULTS = {
  groupId: 'kafka-node-group',
  // Auto commit config
  autoCommit: true,
  autoCommitIntervalMs: 5000,
  // Fetch message config
  fetchMaxWaitMs: 100,
  fetchMinBytes: 1,
  fetchMaxBytes: 1024 * 1024,
  fromOffset: false,
  encoding: 'utf8'
};

var nextId = (function () {
  var id = 0;
  return function () {
    return id++;
  };
})();

function Consumer (client, topics, options) {
  EventEmitter.call(this);
  if (!topics) {
    throw new Error('Must have payloads');
  }

  utils.validateTopics(topics);

  this.fetchCount = 0;
  this.client = client;
  this.options = _.defaults((options || {}), DEFAULTS);
  this.ready = false;
  this.paused = this.options.paused;
  this.id = nextId();
  this.payloads = this.buildPayloads(topics);
  this.connect();
  this.encoding = this.options.encoding;

  if (this.options.groupId) {
    utils.validateConfig('options.groupId', this.options.groupId);
  }
}
util.inherits(Consumer, EventEmitter);

Consumer.prototype.buildPayloads = function (payloads) {
  var self = this;
  return payloads.map(function (p) {
    if (typeof p !== 'object') p = { topic: p };
    p.partition = p.partition || 0;
    p.offset = p.offset || 0;
    p.maxBytes = self.options.fetchMaxBytes;
    p.metadata = 'm'; // metadata can be arbitrary
    return p;
  });
};

Consumer.prototype.connect = function () {
  var self = this;
  // Client already exists
  this.ready = this.client.ready;
  if (this.ready) this.init();

  this.client.on('ready', function () {
    logger.debug('consumer ready');
    if (!self.ready) self.init();
    self.ready = true;
  });

  this.client.on('error', function (err) {
    logger.error('client error %s', err.message);
    self.emit('error', err);
  });

  this.client.on('close', function () {
    logger.debug('connection closed');
  });

  this.client.on('brokersChanged', function () {
    var topicNames = self.payloads.map(function (p) {
      return p.topic;
    });

    this.refreshMetadata(topicNames, function (err) {
      if (err) return self.emit('error', err);
      self.fetch();
    });
  });
  // 'done' will be emit when a message fetch request complete
  this.on('done', function (topics) {
    self.updateOffsets(topics);
    setImmediate(function () {
      self.fetch();
    });
  });
};

Consumer.prototype.init = function () {
  if (!this.payloads.length) {
    return;
  }

  var self = this;
  var topics = self.payloads.map(function (p) { return p.topic; });

  self.client.topicExists(topics, function (err) {
    if (err) {
      return self.emit('error', err);
    }

    if (self.options.fromOffset) {
      return self.fetch();
    }

    self.fetchOffset(self.payloads, function (err, topics) {
      if (err) {
        return self.emit('error', err);
      }

      self.updateOffsets(topics, true);
      self.fetch();
    });
  });
};

/*
 * Update offset info in current payloads
 * @param {Object} Topic-partition-offset
 * @param {Boolean} Don't commit when initing consumer
 */
Consumer.prototype.updateOffsets = function (topics, initing) {
  this.payloads.forEach(function (p) {
    if (!_.isEmpty(topics[p.topic]) && topics[p.topic][p.partition] !== undefined) {
      var offset = topics[p.topic][p.partition];
      if (offset === -1) offset = 0;
      if (!initing) p.offset = offset + 1;
      else p.offset = offset;
    }
  });

  if (this.options.autoCommit && !initing) {
    this.autoCommit(false, function (err) {
      err && logger.debug('auto commit offset', err);
    });
  }
};

function autoCommit (force, cb) {
  if (arguments.length === 1) {
    cb = force;
    force = false;
  }

  if (this.committing && !force) return cb(null, 'Offset committing');

  this.committing = true;
  setTimeout(function () {
    this.committing = false;
  }.bind(this), this.options.autoCommitIntervalMs);

  var payloads = this.payloads;
  if (this.pausedPayloads) payloads = payloads.concat(this.pausedPayloads);

  var commits = payloads.filter(function (p) { return p.offset !== 0; });
  if (commits.length) {
    this.client.sendOffsetCommitRequest(this.options.groupId, commits, cb);
  } else {
    cb(null, 'Nothing to be committed');
  }
}
Consumer.prototype.commit = Consumer.prototype.autoCommit = autoCommit;

Consumer.prototype.fetch = function () {
  if (!this.ready || this.paused) return;
  this.client.sendFetchRequest(this, this.payloads, this.options.fetchMaxWaitMs, this.options.fetchMinBytes);
};

Consumer.prototype.fetchOffset = function (payloads, cb) {
  this.client.sendOffsetFetchRequest(this.options.groupId, payloads, cb);
};

Consumer.prototype.addTopics = function (topics, cb, fromOffset) {
  fromOffset = !!fromOffset;
  var self = this;
  if (!this.ready) {
    setTimeout(function () {
      self.addTopics(topics, cb, fromOffset);
    }
    , 100);
    return;
  }

  // The default is that the topics is a string array of topic names
  var topicNames = topics;

  // If the topics is actually an object and not string we assume it is an array of payloads
  if (typeof topics[0] === 'object') {
    topicNames = topics.map(function (p) { return p.topic; });
  }

  this.client.addTopics(
    topicNames,
    function (err, added) {
      if (err) return cb && cb(err, added);

      var payloads = self.buildPayloads(topics);
      var reFetch = !self.payloads.length;

      if (fromOffset) {
        payloads.forEach(function (p) {
          self.payloads.push(p);
        });
        if (reFetch) self.fetch();
        cb && cb(null, added);
        return;
      }

      // update offset of topics that will be added
      self.fetchOffset(payloads, function (err, offsets) {
        if (err) return cb(err);
        payloads.forEach(function (p) {
          var offset = offsets[p.topic][p.partition];
          if (offset === -1) offset = 0;
          p.offset = offset;
          self.payloads.push(p);
        });
        if (reFetch) self.fetch();
        cb && cb(null, added);
      });
    }
  );
};

Consumer.prototype.removeTopics = function (topics, cb) {
  topics = typeof topics === 'string' ? [topics] : topics;
  this.payloads = this.payloads.filter(function (p) {
    return !~topics.indexOf(p.topic);
  });

  this.client.removeTopicMetadata(topics, cb);
};

Consumer.prototype.close = function (force, cb) {
  this.ready = false;
  if (typeof force === 'function') {
    cb = force;
    force = false;
  }

  if (force) {
    this.commit(force, function (err) {
      if (err) {
        return cb(err);
      }
      this.client.close(cb);
    }.bind(this));
  } else {
    this.client.close(cb);
  }
};

Consumer.prototype.setOffset = function (topic, partition, offset) {
  this.payloads.every(function (p) {
    if (p.topic === topic && p.partition == partition) { // eslint-disable-line eqeqeq
      p.offset = offset;
      return false;
    }
    return true;
  });
};

Consumer.prototype.pause = function () {
  this.paused = true;
};

Consumer.prototype.resume = function () {
  this.paused = false;
  this.fetch();
};

Consumer.prototype.pauseTopics = function (topics) {
  if (!this.pausedPayloads) this.pausedPayloads = [];
  pauseOrResume(this.payloads, this.pausedPayloads, topics);
};

Consumer.prototype.resumeTopics = function (topics) {
  if (!this.pausedPayloads) this.pausedPayloads = [];
  var reFetch = !this.payloads.length;
  pauseOrResume(this.pausedPayloads, this.payloads, topics);
  reFetch = reFetch && this.payloads.length;
  if (reFetch) this.fetch();
};

function pauseOrResume (payloads, nextPayloads, topics) {
  if (!topics || !topics.length) return;

  for (var i = 0, j = 0, l = payloads.length; j < l; i++, j++) {
    if (isInTopics(payloads[i])) {
      nextPayloads.push(
        payloads.splice(i, 1)[0]
      );
      i--;
    }
  }

  function isInTopics (p) {
    return topics.some(function (topic) {
      if (typeof topic === 'string') {
        return p.topic === topic;
      } else {
        return p.topic === topic.topic && p.partition === topic.partition;
      }
    });
  }
}

module.exports = Consumer;