et-go/master/data_source/kafka_producer.go

package data_source

import (
	"encoding/json"
	"fmt"
	"github.com/IBM/sarama"
	"log"
	"time"
)

type KafkaProducer struct {
	producer sarama.SyncProducer
	brokers  []string
}

func NewKafkaProducer(brokers []string) (*KafkaProducer, error) {
	// 配置 Kafka 生产者
	producerConfig := sarama.NewConfig()
	producerConfig.Producer.Return.Successes = true                // 返回成功发送的消息
	producerConfig.Producer.Return.Errors = true                   // 返回发送失败的消息
	producerConfig.Producer.RequiredAcks = sarama.WaitForAll       // 等待所有副本确认
	producerConfig.Producer.Timeout = 10 * time.Second             // 生产者超时时间
	producerConfig.Producer.Retry.Max = 3                          // 最大重试次数
	producerConfig.Producer.Retry.Backoff = 100 * time.Millisecond // 重试间隔时间
	producerConfig.Producer.Compression = sarama.CompressionSnappy // 使用 Snappy 压缩
	producerConfig.Producer.MaxMessageBytes = 1024 * 1024 * 4      // 单条消息最大 4MB

	producer, err := sarama.NewSyncProducer(brokers, producerConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to create Kafka producer:%v", err)
	}

	return &KafkaProducer{
		producer: producer,
		brokers:  brokers,
	}, nil
}

// 实现将 messages 发送到 Kafka 的指定 topic
func (kp *KafkaProducer) SendStringArrayMessage(topic, msgKey string, values []string) error {
	// 将 value 序列化为 JSON
	valueBytes, err := json.Marshal(values)
	if err != nil {
		return fmt.Errorf("failed to marshal value: %v", err)
	}

	// 构造 Kafka 消息
	msg := &sarama.ProducerMessage{
		Topic: topic,
		Key:   sarama.StringEncoder(msgKey),
		Value: sarama.ByteEncoder(valueBytes),
	}

	// 发送消息
	_, _, err = kp.producer.SendMessage(msg)
	if err != nil {
		return fmt.Errorf("failed to send message to Kafka: %v", err)
	}

	log.Printf("Message sent successfully: key=%s, len(value)=%v", msgKey, len(values))
	return nil
}

// Close 关闭 Kafka 生产者
func (kp *KafkaProducer) Close() error {
	return kp.producer.Close()
}