0

I'm lifting several containers in docker-compose.yaml
These are DB, spring application, kafka, kowl and zookeeper.

version: '3.8'

networks:
  kafka-net:
    name: kafka-net
    driver: bridge

services:
  postgres:
    image: 'postgres:14'
    container_name: 'docker-compose-pg'
    command:
      - "postgres"
      - "-c"
      - "max_connections=50"
      - "-c"
      - "shared_buffers=1GB"
      - "-c"
      - "effective_cache_size=4GB"
      - "-c"
      - "work_mem=16MB"
      - "-c"
      - "maintenance_work_mem=512MB"
      - "-c"
      - "random_page_cost=1.1"
      - "-c"
      - "temp_file_limit=10GB"
      - "-c"
      - "log_min_duration_statement=200ms"
      - "-c"
      - "idle_in_transaction_session_timeout=10s"
      - "-c"
      - "lock_timeout=1s"
      - "-c"
      - "statement_timeout=60s"
      - "-c"
      - "shared_preload_libraries=pg_stat_statements"
      - "-c"
      - "pg_stat_statements.max=10000"
      - "-c"
      - "pg_stat_statements.track=all"
    ports:
      - 5432:5432
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 4G
    environment:
      POSTGRES_BD: "postgres"
      POSTGRES_USER: "postgres"
      POSTGRES_PASSWORD: "Kirill"

  application:
    container_name: application
    build:
      context: ./main
      dockerfile: Dockerfile
    environment:
      KAFKA_BOOTSTRAP_SERVER: kafka:19092
      DB_URL: postgres
      DB_USER: postgres
      DB_PASSWORD: Kirill
    depends_on:
      - postgres
    ports:
      - "8080:8080"

  zookeeper:
    image: zookeeper:3.7.0
    container_name: zookeeper
    restart: "no"
    networks:
     - kafka-net
    ports:
      - "2181:2181"

  kafka:
    image: obsidiandynamics/kafka
    container_name: kafka
    restart: "no"
    networks:
      - kafka-net
    ports:
      - "9092:9092"
      - "19092:19092"
    environment:
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER_INTERNAL:PLAINTEXT,DOCKER_EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: DOCKER_INTERNAL://:19092,DOCKER_EXTERNAL://:9092
      KAFKA_ADVERTISED_LISTENERS: DOCKER_INTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:19092,DOCKER_EXTERNAL://kafka:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    depends_on:
      - zookeeper

  kafdrop:
    image: obsidiandynamics/kafdrop
    container_name: kafdrop
    restart: "no"
    networks:
      - kafka-net
    ports:
      - "9000:9000"
    environment:
      KAFKA_BROKERCONNECT: "kafka:19092"
    depends_on:
      - "kafka"

  kowl:
    image: quay.io/cloudhut/kowl:v1.4.0
    restart: on-failure
    volumes:
      - ./kowl_config:/etc/kowl/
    ports:
      - "8088:8080"
    entrypoint: ./kowl --config.filepath=/etc/kowl/config.yaml
    networks:
      - kafka-net
    depends_on:
      - kafka
      - zookeeper

Building docker compose up -d
Assembling docker compose up -d
The problem is that I can't connect the application to kafka, I seem to have set up a network bridge and kowl connects to kafka, but the application does not see kafka..
Config kowl:

kafka:
  brokers:
    - kafka:9092

Log kowl:

{"level":"fatal","ts":"2022-09-16T21:30:51.128Z","msg":"failed to create kafka service","error":"failed to test kafka connection: failed to request metadata: unable to dial: dial tcp 172.18.0.3:9092: connect: connection refused"}
2022-09-16T21:30:57.871911562Z {"level":"info","ts":"2022-09-16T21:30:57.871Z","msg":"started Kowl","version":"v1.4.0","git_sha":"1177dfdcda669904b0fbfc54934dbeba322bf2af","built":"2021-05-27T18:51:39Z"}
2022-09-16T21:30:57.871961485Z {"level":"info","ts":"2022-09-16T21:30:57.871Z","msg":"connecting to Kafka seed brokers, trying to fetch cluster metadata"}
2022-09-16T21:30:57.888834598Z {"level":"info","ts":"2022-09-16T21:30:57.888Z","msg":"successfully connected to kafka cluster","advertised_broker_count":1,"topic_count":1,"controller_id":1,"kafka_version":"v2.3"}
2022-09-16T21:30:57.927838532Z {"level":"info","ts":"2022-09-16T21:30:57.927Z","msg":"Server listening on address","address":"[::]:8080","port":8080}

Log app:

2022-09-16 21:31:27.605  INFO 1 --- [ntainer#0-0-C-1] org.apache.kafka.clients.NetworkClient   : [Consumer clientId=consumer-default-1, groupId=default] Node -1 disconnected.
2022-09-16T21:31:27.605521680Z 2022-09-16 21:31:27.605  WARN 1 --- [ntainer#0-0-C-1] org.apache.kafka.clients.NetworkClient   : [Consumer clientId=consumer-default-1, groupId=default] Connection to node -1 (localhost/127.0.0.1:9092) could not be established. Broker may not be available.
2022-09-16T21:31:27.605574615Z 2022-09-16 21:31:27.605  WARN 1 --- [ntainer#0-0-C-1] org.apache.kafka.clients.NetworkClient   : [Consumer clientId=consumer-default-1, groupId=default] Bootstrap broker localhost:9092 (id: -1 rack: null) disconnected

Question: How to connect to kafka?
Thank you in advance

  • Kowl uses port 9092. So, why do your other services try to use 19092. Also not clear how your app uses KAFKA_BOOTSTRAP_SERVER variable – OneCricketeer Sep 17 '22 at 15:21
  • Kafka uses can use multiple ports and I use it, different microservices can overload only one port with requests. The problem was not in the variable, but in KAFKA_ADVERTISED_LISTENERS, when using ${DOCKER_HOST_IP:-127.0.0.1}:19092, and because I thought that the docker network would be configured dynamically and this would be enough to be able to use such a variable as localhost:19092 (when containerizing only kafka, this worked and so it could be accessed from the application without launched in docker) – Anonymous Anonymous Sep 18 '22 at 14:15
  • Regarding the KAFKA_BOOTSTRAP_SERVER variable. I use it in the usual way and maybe it will be useful to someone
    ``` kafka: bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVER:localhost:19092} ```
    – Anonymous Anonymous Sep 18 '22 at 14:17
  • Just making sure you set it. (but it should be `spring.kafka.bootstrap.servers` . Otherwise, it defaults to `localhost:9092` – OneCricketeer Sep 18 '22 at 15:13

0 Answers0