# How to debug the ingester memory problem **URL:** \ **Category:** Grafana Loki\ **Created:** [June 26, 2023, 2:10pm UTC](https://community.grafana.com/t/how-to-debug-the-ingester-memory-problem/90333 "2023-06-26T14:10:02Z")\ **Posts on this page:** 2\ **Page:** 1
**Author:** ![carterix](https://avatars.discourse-cdn.com/v4/letter/c/e8c25b/32.png) [@carterix](https://community.grafana.com/u/carterix)\ **Post date:** [June 26, 2023, 2:10pm UTC](https://community.grafana.com/t/how-to-debug-the-ingester-memory-problem/90333/1 "2023-06-26T14:10:03Z")
Hi, what does the ingester memory contain other than chunks/indexes? The amount of data pushed is much less than the consumed ingester memory. Metric `loki_ingester_memory_chunks` shows `4K` chunks, while the ingester pod memory (RAM) is showing around 20G, this is much more. Each chunk size is around 1.5MB (`chunk_target_size: 1572864`) So I’m wondering how to check what objects are stored inside the memory (RAM). Is there a way to debug? The metric is showing `loki_ingester_chunk_stored_bytes_total{tenant="mytenant"} 3.696300781e+09` Also I’m trying to understand why the ingester memory is not decrease after the `max_chunk_age` expiration. Since the memory does not decrease then the ingester is going to crash because OOMemory since push is continuing. (Ingester pod is configured with 32GB memory size) Could someone give some suggestions? This is configuration file: ```auto "analytics": "reporting_enabled": false "auth_enabled": true "chunk_store_config": "chunk_cache_config": "enable_fifocache": true "fifocache": "max_size_bytes": "500MB" "max_look_back_period": "0s" "common": "compactor_grpc_address": "server2:port1" "compactor": "compaction_interval": "10m" "compactor_ring": "kvstore": "store": "memberlist" "max_compaction_parallelism": 10 "retention_enabled": true "shared_store": "cos" "working_directory": "/myloki/compactor" "distributor": "ring": "kvstore": "store": "memberlist" "frontend": "compress_responses": true "scheduler_address": "server3:port1" "tail_proxy_url": "server4:port2" "frontend_worker": "grpc_client_config": "max_send_msg_size": 104857600 "match_max_concurrent": true "parallelism": 10 "scheduler_address": "server3:port1" "ingester": "chunk_block_size": 262144 "chunk_encoding": "snappy" "chunk_idle_period": "1m" "chunk_retain_period": "0s" "chunk_target_size": 1572864 "lifecycler": "heartbeat_period": "5s" "interface_names": - "eth0" "join_after": "60s" "num_tokens": 512 "ring": "heartbeat_timeout": "1m" "kvstore": "store": "memberlist" "max_transfer_retries": 0 "wal": "checkpoint_duration": "5m" "dir": "/myloki/wal" "enabled": true "replay_memory_ceiling": "4GB" "ingester_client": "grpc_client_config": "max_recv_msg_size": 67108864 "remote_timeout": "1s" "limits_config": "cardinality_limit": 100000 "creation_grace_period": "10m" "enforce_metric_name": false "ingestion_burst_size_mb": 1500 "ingestion_rate_mb": 1100 "ingestion_rate_strategy": "global" "max_cache_freshness_per_query": "10m" "max_chunks_per_query": 2000000 "max_entries_limit_per_query": 5000 "max_global_streams_per_user": 10000 "max_label_name_length": 1024 "max_label_names_per_series": 30 "max_label_value_length": 2048 "max_line_size": 256000 "max_queriers_per_tenant": 0 "max_query_length": "721h" "max_query_parallelism": 16 "max_query_series": 500 "max_streams_per_user": 1000 "per_stream_rate_limit": "3MB" "per_stream_rate_limit_burst": "15MB" "query_timeout": "1h" "reject_old_samples": true "reject_old_samples_max_age": "24h" "retention_period": "720h" "split_queries_by_interval": "30m" "memberlist": "abort_if_cluster_join_fails": false "bind_port": 7946 "join_members": - "server5:port3" "max_join_backoff": "1m" "max_join_retries": 10 "min_join_backoff": "1s" "querier": "engine": "max_look_back_period": "30s" "timeout": "3m" "extra_query_delay": "0s" "max_concurrent": 16 "query_ingesters_within": "3h" "query_timeout": "1h" "tail_max_duration": "1h" "query_range": "align_queries_with_step": true "cache_results": true "max_retries": 5 "parallelise_shardable_queries": true "results_cache": "cache": "embedded_cache": "enabled": true "max_size_mb": 100 "ttl": "3h" "compression": "snappy" "query_scheduler": "max_outstanding_requests_per_tenant": 32768 "schema_config": "configs": - "from": "2020-10-01" "index": "period": "24h" "prefix": "loki_index_" "object_store": "cos" "schema": "v11" "store": "boltdb-shipper" - "from": "2023-06-15" "index": "period": "24h" "prefix": "loki_index_" "object_store": "cos" "schema": "v12" "store": "tsdb" "server": "graceful_shutdown_timeout": "5s" "grpc_server_max_concurrent_streams": 1000 "grpc_server_max_recv_msg_size": 104857600 "grpc_server_max_send_msg_size": 104857600 "grpc_server_min_time_between_pings": "10s" "grpc_server_ping_without_stream_allowed": true "http_listen_port": 3100 "http_server_idle_timeout": "120s" "http_server_read_timeout": "600s" "http_server_write_timeout": "600s" "log_format": "json" "log_level": "debug" "storage_config": "boltdb_shipper": "active_index_directory": "/myloki/index" "cache_location": "/myloki/index_cache" "cache_ttl": "24h" "index_gateway_client": "server_address": "server1:port1" "resync_interval": "5m" "shared_store": "cos" "cos": "forcepathstyle": true "index_cache_validity": "3h" "index_queries_cache_config": "embedded_cache": "enabled": true "max_size_mb": 100 "ttl": "3h" "tsdb_shipper": "active_index_directory": "/myloki/index/tsdb" "cache_location": "/myloki/index/tsdb/cache" "index_gateway_client": "server_address": "server1:port1" "shared_store": "cos" ``` ---
**Author:** ![system](https://us1.discourse-cdn.com/grafana/original/2X/e/ef5308cc58c9e5a13187b5012eb99f406928462f.png) [@system](https://community.grafana.com/u/system)\ **Post date:** [June 25, 2024, 2:10pm UTC](https://community.grafana.com/t/how-to-debug-the-ingester-memory-problem/90333/2 "2024-06-25T14:10:46Z")
Closed on June 25, 2024, 2:10pm UTC This topic was automatically closed 365 days after the last reply. New replies are no longer allowed.