Loki chunks flushed as "idle" despite chunk_idle_period > max_chunk_age

Grafana Loki
1

Hi,

Using the metric sum by (reason) (rate(loki_ingester_chunks_flushed_total[1h])) I can see, that almost all chunks get flushed for reason “idle”. For testing purposes I set max_chunk_age: 5h and chunk_idle_period: 6h in the ingester config.
From my understanding the reason for chunks being flushed should never be “idle”, because the limit of max_chunk_age should be hit first. Looking at the metric from above it still shows most chunks being flushed for reason “idle”, even after a day.

Can anybody explain why this is the case?

I’m using Loki Version 3.3.2 installed via loki helm chart 6.25.0

2

I’ve never tried this before, so I am not sure. Can you hit the /config endpoint on your ingester and post the results here, please?

3 
ingester_client:
  pool_config:
    client_cleanup_period: 15s
    health_check_ingesters: true
    remote_timeout: 1s
  remote_timeout: 5s
  grpc_client_config:
    max_recv_msg_size: 104857600
    max_send_msg_size: 104857600
    grpc_compression: ""
    rate_limit: 0
    rate_limit_burst: 0
    backoff_on_ratelimits: false
    backoff_config:
      min_period: 100ms
      max_period: 10s
      max_retries: 10
    initial_stream_window_size: 63KiB1023B
    initial_connection_window_size: 63KiB1023B
    tls_enabled: false
    tls_cert_path: ""
    tls_key_path: ""
    tls_ca_path: ""
    tls_server_name: ""
    tls_insecure_skip_verify: false
    tls_cipher_suites: ""
    tls_min_version: ""
    connect_timeout: 5s
    connect_backoff_base_delay: 1s
    connect_backoff_max_delay: 5s
ingester:
  lifecycler:
    ring:
      kvstore:
        store: memberlist
        prefix: collectors/
        consul:
          host: localhost:8500
          acl_token: ""
          http_client_timeout: 20s
          consistent_reads: false
          watch_rate_limit: 1
          watch_burst_size: 1
          cas_retry_delay: 1s
        etcd:
          endpoints: []
          dial_timeout: 10s
          max_retries: 10
          tls_enabled: false
          tls_cert_path: ""
          tls_key_path: ""
          tls_ca_path: ""
          tls_server_name: ""
          tls_insecure_skip_verify: false
          tls_cipher_suites: ""
          tls_min_version: ""
          username: ""
          password: ""
        multi:
          primary: ""
          secondary: ""
          mirror_enabled: false
          mirror_timeout: 2s
      heartbeat_timeout: 1m0s
      replication_factor: 1
      zone_awareness_enabled: false
      excluded_zones: ""
    num_tokens: 128
    heartbeat_period: 5s
    heartbeat_timeout: 1m0s
    observe_period: 0s
    join_after: 0s
    min_ready_duration: 15s
    interface_names:
    - eth0
    - lo
    enable_inet6: false
    final_sleep: 0s
    tokens_file_path: ""
    availability_zone: ""
    unregister_on_shutdown: true
    readiness_check_ring_health: true
    address: ""
    port: 0
    id: loki-0
  concurrent_flushes: 32
  flush_check_period: 30s
  flush_op_backoff:
    min_period: 10s
    max_period: 1m0s
    max_retries: 10
  flush_op_timeout: 10m0s
  chunk_retain_period: 0s
  chunk_idle_period: 6h0m0s
  chunk_block_size: 262144
  chunk_target_size: 1572864
  chunk_encoding: snappy
  max_chunk_age: 5h0m0s
  autoforget_unhealthy: false
  sync_period: 1h0m0s
  sync_min_utilization: 0.1
  max_returned_stream_errors: 10
  query_store_max_look_back_period: 5h41m0s
  wal:
    enabled: true
    dir: /var/loki/wal
    checkpoint_duration: 5m0s
    flush_on_shutdown: false
    replay_memory_ceiling: 4GB
  index_shards: 32
  max_dropped_streams: 10
  shutdown_marker_path: /var/loki
  owned_streams_check_interval: 30s
  kafka_ingestion:
    enabled: false
    partition_ring:
      kvstore:
        store: memberlist
        prefix: collectors/
        consul:
          host: localhost:8500
          acl_token: ""
          http_client_timeout: 20s
          consistent_reads: false
          watch_rate_limit: 1
          watch_burst_size: 1
          cas_retry_delay: 1s
        etcd:
          endpoints: []
          dial_timeout: 10s
          max_retries: 10
          tls_enabled: false
          tls_cert_path: ""
          tls_key_path: ""
          tls_ca_path: ""
          tls_server_name: ""
          tls_insecure_skip_verify: false
          tls_cipher_suites: ""
          tls_min_version: ""
          username: ""
          password: ""
        multi:
          primary: ""
          secondary: ""
          mirror_enabled: false
          mirror_timeout: 2s
      min_partition_owners_count: 1
      min_partition_owners_duration: 10s
      delete_inactive_partition_after: 13h0m0s