Promtail: Loki Server returned HTTP status 429 Too Many Requests

Question

I'm running Loki for test purposes in Docker and am recently getting following error from the Promtail and Loki containers:

level=warn ts=2022-02-18T09:41:39.186511145Z caller=client.go:349 component=client host=loki:3100 msg="error sending batch, will retry" status=429 error="server returned HTTP status 429 Too Many Requests (429): Maximum active stream limit exceeded, reduce the number of active streams (reduce labels or reduce label values), or contact your Loki administrator to see if the limit can be increased"

I have tried increasing limit settings (ingestion_rate_mb and ingestion_burst_size_mb) in my Loki config.

I setup two Promtail jobs - one job ingesting MS Exchange logs from a local directory (currently 8TB and increasing), the other job gets logs spooled from syslog-ng.

I've read that reducing labels help. But I'm only using two labels.

Configuration

Below my config files (docker-compose, loki, promtail):

docker-compose.yaml

version: "3"

networks:
  loki:

services:

  loki:
    image: grafana/loki:2.4.2
    container_name: loki
    restart: always
    user: "10001:10001"
    ports:
      - "3100:3100"
    command: -config.file=/etc/loki/local-config.yaml
    volumes:
      - ${DATADIR}/loki/etc:/etc/loki:rw
      - ${DATADIR}/loki/chunks:/loki/chunks
    networks:
      - loki

  promtail:
    image: grafana/promtail:2.4.2
    container_name: promtail
    restart: always
    volumes:
      - /var/log/loki:/var/log/loki
      - ${DATADIR}/promtail/etc:/etc/promtail
    ports:
      - "1514:1514" # for syslog-ng
      - "9080:9080" # for http web interface
    command: -config.file=/etc/promtail/config.yml
    networks:
      - loki

  grafana:
    image: grafana/grafana:8.3.4
    container_name: grafana
    restart: always
    user: "476:0"
    volumes:
      - ${DATADIR}/grafana/var:/var/lib/grafana
    ports:
      - "3000:3000"
    networks:
      - loki

Loki Config

auth_enabled: false

server:
  http_listen_port: 3100

common:
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

# https://grafana.com/docs/loki/latest/configuration/#limits_config
limits_config:
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  ingestion_rate_mb: 12
  ingestion_burst_size_mb: 24
  per_stream_rate_limit: 24MB
chunk_store_config:
  max_look_back_period: 336h
table_manager:
  retention_deletes_enabled: true
  retention_period: 2190h
ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_encoding: snappy

Promtail Config

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:

- job_name: exchange
  static_configs:
  - targets:
      - localhost
    labels:
      job: exchange
      __path__: /var/log/loki/exchange/*/*/*log

- job_name: syslog-ng
  syslog:
    listen_address: 0.0.0.0:1514
    idle_timeout: 60s
    label_structured_data: yes
    labels:
      job: "syslog-ng"

  relabel_configs:
    - source_labels: ['__syslog_message_hostname']
      target_label: 'host'