0

I'm having a bit of trouble with my Fluentd configuration. It seems to be sending logs to OpenSearch just fine, but I can't get it to send the data with the same tags to S3. just to clarify, Fluentd is running inside a container.

Now, how to send the each log type (myapp.** & docker.** & django.logger) to s3 separately?

The original config file I'm using:

fluent.conf

<source>
  @type forward
  @label @mainstream
  port 24224
  bind 0.0.0.0
</source>

# expose metrics in prometheus format
<source>
  @type prometheus
  bind 0.0.0.0
  port 24231
  metrics_path /metrics
</source>

<source>
  @type prometheus_output_monitor
  interval 15
  <labels>
    hostname ${hostname}
  </labels>
</source>

<label @mainstream>
  # count the number of incoming records per tag
  <filter *.**>
    @type prometheus
    <metric>
      name fluentd_input_status_num_records_total
      type counter
      desc The total number of incoming records
      <labels>
        tag ${tag}
        hostname ${hostname}
      </labels>
    </metric>
  </filter>

  <filter /docker\..*nginx.*/>
    @type parser
    key_name log
    reserve_data true
    remove_key_name_field true
    inject_key_prefix nginx.
    <parse>
      @type nginx
    </parse>
  </filter>

  <filter /docker\..*(traefik|prometheus).*/>
    @type parser
    key_name log
    reserve_data true
    remove_key_name_field true
    <parse>
      @type json
      # "json" parser sets time_key to "time" by default
      # set it to current time instead
      time_key nil
    </parse>
  </filter>

  <match myapp.**>
    @type copy

    <store>
      @type opensearch
      @include opensearch.conf

      id_key entryhash

      logstash_format true
      logstash_prefix myapp
      logstash_dateformat %Y-%m
      include_tag_key true
      tag_key @log_name

      template_name myapp
      template_overwrite true
      template_file /fluentd/etc/templates/myapp_template.json
    </store>

    @include prometheus_count_store.conf

  </match>

  <match docker.**>
    @type copy

    <store>
      @type opensearch
      @include opensearch.conf

      logstash_format true
      logstash_prefix docker
      logstash_dateformat %Y-%m
      include_timestamp true
      include_tag_key true
      tag_key @tag

      template_name docker
      template_overwrite true
      template_file /fluentd/etc/templates/docker_template.json
    </store>

    @include prometheus_count_store.conf

  </match>

  <match django.logger>
    @type copy

    <store>
      @type opensearch
      @include opensearch.conf

      logstash_format true
      logstash_prefix django
      logstash_dateformat %Y-%m
      include_timestamp true

      template_name django
      template_overwrite true
      template_file /fluentd/etc/templates/django_template.json
    </store>

    @include prometheus_count_store.conf

  </match>
</label>

I've been experimenting with different tags to include this configuration but haven't had any luck. Any suggestions on how to make it work?

<match *>
    @type s3
    aws_key_id *********
    aws_sec_key ********
    s3_bucket "******"
    s3_region ********
    s3_object_key_format %{path}/%{time_slice}/%{index}.%{file_extension}
    path 
    store_as json
    <buffer>
      flush_mode interval
      flush_interval 60s
      chunk_limit_size 1M
      timekey 2m
    </buffer>
  </match>
Azeem
  • 11,148
  • 4
  • 27
  • 40

0 Answers0