How to debug the ingester memory problem

Hi,
what does the ingester memory contain other than chunks/indexes? The amount of data pushed is much less than the consumed ingester memory. Metric loki_ingester_memory_chunks shows 4K chunks, while the ingester pod memory (RAM) is showing around 20G, this is much more. Each chunk size is around 1.5MB (chunk_target_size: 1572864)
So I’m wondering how to check what objects are stored inside the memory (RAM). Is there a way to debug? The metric is showing loki_ingester_chunk_stored_bytes_total{tenant="mytenant"} 3.696300781e+09
Also I’m trying to understand why the ingester memory is not decrease after the max_chunk_age expiration. Since the memory does not decrease then the ingester is going to crash because OOMemory since push is continuing. (Ingester pod is configured with 32GB memory size)

Could someone give some suggestions?

This is configuration file:

"analytics":
  "reporting_enabled": false
"auth_enabled": true
"chunk_store_config":
  "chunk_cache_config":
    "enable_fifocache": true
    "fifocache":
      "max_size_bytes": "500MB"
  "max_look_back_period": "0s"
"common":
  "compactor_grpc_address": "server2:port1"
"compactor":
  "compaction_interval": "10m"
  "compactor_ring":
    "kvstore":
      "store": "memberlist"
  "max_compaction_parallelism": 10
  "retention_enabled": true
  "shared_store": "cos"
  "working_directory": "/myloki/compactor"
"distributor":
  "ring":
    "kvstore":
      "store": "memberlist"
"frontend":
  "compress_responses": true
  "scheduler_address": "server3:port1"
  "tail_proxy_url": "server4:port2"
"frontend_worker":
  "grpc_client_config":
    "max_send_msg_size": 104857600
  "match_max_concurrent": true
  "parallelism": 10
  "scheduler_address": "server3:port1"
"ingester":
  "chunk_block_size": 262144
  "chunk_encoding": "snappy"
  "chunk_idle_period": "1m"
  "chunk_retain_period": "0s"
  "chunk_target_size": 1572864
  "lifecycler":
    "heartbeat_period": "5s"
    "interface_names":
    - "eth0"
    "join_after": "60s"
    "num_tokens": 512
    "ring":
      "heartbeat_timeout": "1m"
      "kvstore":
        "store": "memberlist"
  "max_transfer_retries": 0
  "wal":
    "checkpoint_duration": "5m"
    "dir": "/myloki/wal"
    "enabled": true
    "replay_memory_ceiling": "4GB"
"ingester_client":
  "grpc_client_config":
    "max_recv_msg_size": 67108864
  "remote_timeout": "1s"
"limits_config":
  "cardinality_limit": 100000
  "creation_grace_period": "10m"
  "enforce_metric_name": false
  "ingestion_burst_size_mb": 1500
  "ingestion_rate_mb": 1100
  "ingestion_rate_strategy": "global"
  "max_cache_freshness_per_query": "10m"
  "max_chunks_per_query": 2000000
  "max_entries_limit_per_query": 5000
  "max_global_streams_per_user": 10000
  "max_label_name_length": 1024
  "max_label_names_per_series": 30
  "max_label_value_length": 2048
  "max_line_size": 256000
  "max_queriers_per_tenant": 0
  "max_query_length": "721h"
  "max_query_parallelism": 16
  "max_query_series": 500
  "max_streams_per_user": 1000
  "per_stream_rate_limit": "3MB"
  "per_stream_rate_limit_burst": "15MB"
  "query_timeout": "1h"
  "reject_old_samples": true
  "reject_old_samples_max_age": "24h"
  "retention_period": "720h"
  "split_queries_by_interval": "30m"
"memberlist":
  "abort_if_cluster_join_fails": false
  "bind_port": 7946
  "join_members":
  - "server5:port3"
  "max_join_backoff": "1m"
  "max_join_retries": 10
  "min_join_backoff": "1s"
"querier":
  "engine":
    "max_look_back_period": "30s"
    "timeout": "3m"
  "extra_query_delay": "0s"
  "max_concurrent": 16
  "query_ingesters_within": "3h"
  "query_timeout": "1h"
  "tail_max_duration": "1h"
"query_range":
  "align_queries_with_step": true
  "cache_results": true
  "max_retries": 5
  "parallelise_shardable_queries": true
  "results_cache":
    "cache":
      "embedded_cache":
        "enabled": true
        "max_size_mb": 100
        "ttl": "3h"
    "compression": "snappy"
"query_scheduler":
  "max_outstanding_requests_per_tenant": 32768
"schema_config":
  "configs":
  - "from": "2020-10-01"
    "index":
      "period": "24h"
      "prefix": "loki_index_"
    "object_store": "cos"
    "schema": "v11"
    "store": "boltdb-shipper"
  - "from": "2023-06-15"
    "index":
      "period": "24h"
      "prefix": "loki_index_"
    "object_store": "cos"
    "schema": "v12"
    "store": "tsdb"
"server":
  "graceful_shutdown_timeout": "5s"
  "grpc_server_max_concurrent_streams": 1000
  "grpc_server_max_recv_msg_size": 104857600
  "grpc_server_max_send_msg_size": 104857600
  "grpc_server_min_time_between_pings": "10s"
  "grpc_server_ping_without_stream_allowed": true
  "http_listen_port": 3100
  "http_server_idle_timeout": "120s"
  "http_server_read_timeout": "600s"
  "http_server_write_timeout": "600s"
  "log_format": "json"
  "log_level": "debug"
"storage_config":
  "boltdb_shipper":
    "active_index_directory": "/myloki/index"
    "cache_location": "/myloki/index_cache"
    "cache_ttl": "24h"
    "index_gateway_client":
      "server_address": "server1:port1"
    "resync_interval": "5m"
    "shared_store": "cos"
  "cos":
    "forcepathstyle": true
  "index_cache_validity": "3h"
  "index_queries_cache_config":
    "embedded_cache":
      "enabled": true
      "max_size_mb": 100
      "ttl": "3h"
  "tsdb_shipper":
    "active_index_directory": "/myloki/index/tsdb"
    "cache_location": "/myloki/index/tsdb/cache"
    "index_gateway_client":
      "server_address": "server1:port1"
    "shared_store": "cos"
2 Likes

This topic was automatically closed 365 days after the last reply. New replies are no longer allowed.