Create multiple partitions in kafka topic and publish message to all of them using kafka-node

Question

I am new to kafka and implementing it in nodeJS using kafka-node. I want to create 3 partitions in one topic and publish messages to all the topics at the same time. I tried following code, but here only one partition is creating and all messages are going to that one partition. Can anyone please tell me where I am going wrong. Thank you so much.

Abc.abcData = async() => {
    try 
    {
        var client = new kafka.KafkaClient();
        var topic = 'newTopic';
        var topicsToCreate = [
        {
            topic: topic,
            partitions: 3,
            replicationFactor: 2,
            replicaAssignment: [
            {
              partition: 0,
              replicas: [0]
            },
            {
                partition: 1,
                replicas: [1]
            },
            {
                partition: 2,
                replicas: [2]
            }
          ]
        },
        ]
        client.createTopics(topicsToCreate, (error, result) => {
        console.log(result);
        });
        var HighLevelProducer = kafka.HighLevelProducer;
        var producer = new HighLevelProducer(client);
        var payloads = [
        { topic: topic, messages: 'this is partition 1!!', partitions: 0},
        { topic: topic, messages: 'this is partition 2!!', partitions: 1},
        { topic: topic, messages: 'this is partition 3!!', partitions: 2}
        ];
        producer.on('ready', function () {
            producer.send(payloads, function (err, result) {
                if (err)
                    console.log(err);
                console.log(result);

            });
        });
        
    } 
    catch (err) 
    {
        console.error(err.message);
    }
};

I am getting response as below -

[ { topic: 'newTopic', error: "Topic 'newTopic' already exists." } ]
{"newTopic":{"0":6}}

score 3 · Accepted Answer · answered Aug 21 '20 at 12:09

Here you used createTopics() on kafka server and it will only work when auto.create.topics.enable, on the Kafka server, is set to true. Client simply sends a metadata request to the server which will auto create topics. When async is set to false, this method does not return until all topics are created, otherwise it returns immediately. So, here default one topic with one partition is creating. To create multiple partition or to make it customize you have to add following line in server.property file -

auto.create.topics.enable=false

Create multiple partitions in kafka topic and publish message to all of them using kafka-node

1 Answers1