I'm lifting several containers in docker-compose.yaml
These are DB, spring application, kafka, kowl and zookeeper.
version: '3.8'
networks:
kafka-net:
name: kafka-net
driver: bridge
services:
postgres:
image: 'postgres:14'
container_name: 'docker-compose-pg'
command:
- "postgres"
- "-c"
- "max_connections=50"
- "-c"
- "shared_buffers=1GB"
- "-c"
- "effective_cache_size=4GB"
- "-c"
- "work_mem=16MB"
- "-c"
- "maintenance_work_mem=512MB"
- "-c"
- "random_page_cost=1.1"
- "-c"
- "temp_file_limit=10GB"
- "-c"
- "log_min_duration_statement=200ms"
- "-c"
- "idle_in_transaction_session_timeout=10s"
- "-c"
- "lock_timeout=1s"
- "-c"
- "statement_timeout=60s"
- "-c"
- "shared_preload_libraries=pg_stat_statements"
- "-c"
- "pg_stat_statements.max=10000"
- "-c"
- "pg_stat_statements.track=all"
ports:
- 5432:5432
deploy:
resources:
limits:
cpus: '1'
memory: 4G
environment:
POSTGRES_BD: "postgres"
POSTGRES_USER: "postgres"
POSTGRES_PASSWORD: "Kirill"
application:
container_name: application
build:
context: ./main
dockerfile: Dockerfile
environment:
KAFKA_BOOTSTRAP_SERVER: kafka:19092
DB_URL: postgres
DB_USER: postgres
DB_PASSWORD: Kirill
depends_on:
- postgres
ports:
- "8080:8080"
zookeeper:
image: zookeeper:3.7.0
container_name: zookeeper
restart: "no"
networks:
- kafka-net
ports:
- "2181:2181"
kafka:
image: obsidiandynamics/kafka
container_name: kafka
restart: "no"
networks:
- kafka-net
ports:
- "9092:9092"
- "19092:19092"
environment:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER_INTERNAL:PLAINTEXT,DOCKER_EXTERNAL:PLAINTEXT
KAFKA_LISTENERS: DOCKER_INTERNAL://:19092,DOCKER_EXTERNAL://:9092
KAFKA_ADVERTISED_LISTENERS: DOCKER_INTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:19092,DOCKER_EXTERNAL://kafka:9092
KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER_INTERNAL
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_BROKER_ID: 1
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
depends_on:
- zookeeper
kafdrop:
image: obsidiandynamics/kafdrop
container_name: kafdrop
restart: "no"
networks:
- kafka-net
ports:
- "9000:9000"
environment:
KAFKA_BROKERCONNECT: "kafka:19092"
depends_on:
- "kafka"
kowl:
image: quay.io/cloudhut/kowl:v1.4.0
restart: on-failure
volumes:
- ./kowl_config:/etc/kowl/
ports:
- "8088:8080"
entrypoint: ./kowl --config.filepath=/etc/kowl/config.yaml
networks:
- kafka-net
depends_on:
- kafka
- zookeeper
Building docker compose up -d
Assembling docker compose up -d
The problem is that I can't connect the application to kafka, I seem to have set up a network bridge and kowl connects to kafka, but the application does not see kafka..
Config kowl:
kafka:
brokers:
- kafka:9092
Log kowl:
{"level":"fatal","ts":"2022-09-16T21:30:51.128Z","msg":"failed to create kafka service","error":"failed to test kafka connection: failed to request metadata: unable to dial: dial tcp 172.18.0.3:9092: connect: connection refused"}
2022-09-16T21:30:57.871911562Z {"level":"info","ts":"2022-09-16T21:30:57.871Z","msg":"started Kowl","version":"v1.4.0","git_sha":"1177dfdcda669904b0fbfc54934dbeba322bf2af","built":"2021-05-27T18:51:39Z"}
2022-09-16T21:30:57.871961485Z {"level":"info","ts":"2022-09-16T21:30:57.871Z","msg":"connecting to Kafka seed brokers, trying to fetch cluster metadata"}
2022-09-16T21:30:57.888834598Z {"level":"info","ts":"2022-09-16T21:30:57.888Z","msg":"successfully connected to kafka cluster","advertised_broker_count":1,"topic_count":1,"controller_id":1,"kafka_version":"v2.3"}
2022-09-16T21:30:57.927838532Z {"level":"info","ts":"2022-09-16T21:30:57.927Z","msg":"Server listening on address","address":"[::]:8080","port":8080}
Log app:
2022-09-16 21:31:27.605 INFO 1 --- [ntainer#0-0-C-1] org.apache.kafka.clients.NetworkClient : [Consumer clientId=consumer-default-1, groupId=default] Node -1 disconnected.
2022-09-16T21:31:27.605521680Z 2022-09-16 21:31:27.605 WARN 1 --- [ntainer#0-0-C-1] org.apache.kafka.clients.NetworkClient : [Consumer clientId=consumer-default-1, groupId=default] Connection to node -1 (localhost/127.0.0.1:9092) could not be established. Broker may not be available.
2022-09-16T21:31:27.605574615Z 2022-09-16 21:31:27.605 WARN 1 --- [ntainer#0-0-C-1] org.apache.kafka.clients.NetworkClient : [Consumer clientId=consumer-default-1, groupId=default] Bootstrap broker localhost:9092 (id: -1 rack: null) disconnected
Question: How to connect to kafka?
Thank you in advance
``` kafka: bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVER:localhost:19092} ``` – Anonymous Anonymous Sep 18 '22 at 14:17