Highways4Good/api/node_modules/kafka-node/lib/consumerGroupMigrator.js

'use strict';

const assert = require('assert');
const util = require('util');
const async = require('async');
const logger = require('./logging')('kafka-node:ConsumerGroupMigrator');
const zookeeper = require('node-zookeeper-client');
const _ = require('lodash');
const EventEmitter = require('events');
const NUMER_OF_TIMES_TO_VERIFY = 4;
const VERIFY_WAIT_TIME_MS = 1500;

function ConsumerGroupMigrator (consumerGroup) {
  EventEmitter.call(this);
  assert(consumerGroup);
  const self = this;
  this.consumerGroup = consumerGroup;
  this.client = consumerGroup.client;
  var verified = 0;

  if (consumerGroup.options.migrateRolling) {
    this.zk = zookeeper.createClient(consumerGroup.client.connectionString, { retries: 10 });
    this.zk.on('connected', function () {
      self.filterByExistingZkTopics(function (error, topics) {
        if (error) {
          return self.emit('error', error);
        }

        if (topics.length) {
          self.checkForOwnersAndListenForChange(topics);
        } else {
          logger.debug('No HLC topics exist in zookeeper.');
          self.connectConsumerGroup();
        }
      });
    });

    this.on('noOwnersForTopics', function (topics) {
      logger.debug('No owners for topics %s reported.', topics);
      if (++verified <= NUMER_OF_TIMES_TO_VERIFY) {
        logger.debug(
          '%s verify %d of %d HLC has given up ownership by checking again in %d',
          self.client.clientId,
          verified,
          NUMER_OF_TIMES_TO_VERIFY,
          VERIFY_WAIT_TIME_MS
        );

        setTimeout(function () {
          self.checkForOwners(topics);
        }, VERIFY_WAIT_TIME_MS);
      } else {
        self.connectConsumerGroup();
      }
    });

    this.on(
      'topicOwnerChange',
      _.debounce(function (topics) {
        verified = 0;
        self.checkForOwnersAndListenForChange(topics);
      }, 250)
    );

    this.zk.connect();
  } else {
    this.connectConsumerGroup();
  }
}

util.inherits(ConsumerGroupMigrator, EventEmitter);

ConsumerGroupMigrator.prototype.connectConsumerGroup = function () {
  logger.debug('%s connecting consumer group', this.client.clientId);
  const self = this;
  if (this.client.ready) {
    this.consumerGroup.connect();
  } else {
    this.client.once('ready', function () {
      self.consumerGroup.connect();
    });
  }
  this.zk && this.zk.close();
};

ConsumerGroupMigrator.prototype.filterByExistingZkTopics = function (callback) {
  const self = this;
  const path = '/consumers/' + this.consumerGroup.options.groupId + '/owners/';

  async.filter(
    this.consumerGroup.topics,
    function (topic, cb) {
      const topicPath = path + topic;
      logger.debug('%s checking zk path %s', self.client.clientId, topicPath);
      self.zk.exists(topicPath, function (error, stat) {
        if (error) {
          return callback(error);
        }
        cb(stat);
      });
    },
    callback
  );
};

ConsumerGroupMigrator.prototype.checkForOwnersAndListenForChange = function (topics) {
  this.checkForOwners(topics, true);
};

ConsumerGroupMigrator.prototype.checkForOwners = function (topics, listenForChange) {
  const self = this;
  const path = '/consumers/' + this.consumerGroup.options.groupId + '/owners/';
  var ownedPartitions = 0;

  function topicWatcher (event) {
    self.emit('topicOwnerChange', topics);
  }

  async.each(
    topics,
    function (topic, callback) {
      const args = [path + topic];

      if (listenForChange) {
        logger.debug('%s listening for changes in topic %s', self.client.clientId, topic);
        args.push(topicWatcher);
      }

      args.push(function (error, children, stats) {
        if (error) {
          return callback(error);
        }
        ownedPartitions += children.length;
        callback(null);
      });

      self.zk.getChildren.apply(self.zk, args);
    },
    function (error) {
      if (error) {
        return self.emit('error', error);
      }
      if (ownedPartitions === 0) {
        self.emit('noOwnersForTopics', topics);
      } else {
        logger.debug('%s %d partitions are owned by old HLC... waiting...', self.client.clientId, ownedPartitions);
      }
    }
  );
};

ConsumerGroupMigrator.prototype.saveHighLevelConsumerOffsets = function (topicPartitionList, callback) {
  const self = this;
  this.client.sendOffsetFetchRequest(this.consumerGroup.options.groupId, topicPartitionList, function (error, results) {
    logger.debug('sendOffsetFetchRequest response:', results, error);
    if (error) {
      return callback(error);
    }
    self.offsets = results;
    callback(null);
  });
};

ConsumerGroupMigrator.prototype.getOffset = function (tp, defaultOffset) {
  const offset = _.get(this.offsets, [tp.topic, tp.partition], defaultOffset);
  if (offset === -1) {
    return defaultOffset;
  }
  return offset;
};

module.exports = ConsumerGroupMigrator;