0

I have a docker-compose file where it contains 4 containers -

  1. flask app

  2. nginx for reverse proxy

  3. fluentd for log processing

  4. mongodb for storing the logs

version: '3'
services:

  fluentd:
    build: fluentd
    container_name: fluentd
    hostname: fluend
    ports:
      - 1514:1514/udp
      - 24224:24224
    volumes:
      - ./fluentd/fluent.conf:/fluentd/etc/fluent.conf
    networks:
      - frontend

  reverse:
    container_name: reverse
    hostname: reverse
    restart: unless-stopped
    image: nginx
    ports:
      - 80:80
      - 443:443
    logging:
          driver: "fluentd"      
    volumes:
      - ./nginx/flask.conf:/etc/nginx/conf.d/default.conf
      - ./nginx/.htpasswd:/etc/nginx/conf.d/.htpasswd
    depends_on:
      - fluentd
    networks:
      - frontend    

  mongodb:
    restart: unless-stopped
    container_name: mongodb
    hostname: mongodb 
    image: mongo:4.2.8
    environment:
      MONGO_INITDB_ROOT_USERNAME: root
      MONGO_INITDB_ROOT_PASSWORD: root
      MONGO_INITDB_DATABASE: fluentd
    ports:
      - 27017:27017
    networks:
      - frontend 

  flask:
    build:
      context: app
      dockerfile: Dockerfile
    container_name: flask
    hostname: flask
    restart: unless-stopped
    environment:
      APP_PORT: 5000
    networks:
      - frontend

networks:
  frontend:
    driver: bridge
    name: frontend

and here's the fluent.conf

 <source>
   @type forward
  #  @type tail
  #  format nginx
  #  tag nginx.access
  #  path /var/log/nginx/access.log
 </source>


 <match *>
  #  @type stdout
   @type mongo
   host mongodb
   port 27017
   database admin
   collection logs   

  # for capped collection
  capped
  capped_size 1024m

  # authentication
  user root
  password root

  <inject>
    # key name of timestamp
    time_key time
  </inject>

  <buffer>
    # flush
    flush_interval 10s
  </buffer>   
 </match>

But nginx ONLY starts when the <source> is set to @type forward and not @type tail

and with @type tail it gives the following error:

ERROR: for reverse Cannot start service reverse: failed to initialize logging driver: dial tcp 127.0.0.1:24224: connect: connection refused

As of now with @type forward, the logs are being stored in the mongodb as below:

enter image description here

But how can I save the values of log key and save as separate keys? So it would look like:

time_local : some value

remote_addr : some ip

So basically what I want is, create separate keys for the whole log key now, which is:

{\"time_local\":\"06/Mar/2022:06:19:16 +0000\",\"remote_addr\":\"192.168.112.1\",\"status\": \"200\",\"http_user_agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.109 Safari/537.36\"}"
Jananath Banuka
  • 2,951
  • 8
  • 57
  • 105
  • Relevant: https://docs.docker.com/config/containers/logging/configure/#supported-logging-drivers and https://docs.docker.com/config/containers/logging/fluentd/. You've configured `fluentd` logging driver and that should be with the `forward` input plugin. – Azeem Mar 06 '22 at 15:05
  • For logs, maybe you need to configure the parser `nginx` or `json`. See: https://docs.fluentd.org/configuration/parse-section#parameters – Azeem Mar 06 '22 at 15:05

0 Answers0