Hi,
what does the ingester memory contain other than chunks/indexes? The amount of data pushed is much less than the consumed ingester memory. Metric loki_ingester_memory_chunks
shows 4K
chunks, while the ingester pod memory (RAM) is showing around 20G, this is much more. Each chunk size is around 1.5MB (chunk_target_size: 1572864
)
So I’m wondering how to check what objects are stored inside the memory (RAM). Is there a way to debug? The metric is showing loki_ingester_chunk_stored_bytes_total{tenant="mytenant"} 3.696300781e+09
Also I’m trying to understand why the ingester memory is not decrease after the max_chunk_age
expiration. Since the memory does not decrease then the ingester is going to crash because OOMemory since push is continuing. (Ingester pod is configured with 32GB memory size)
Could someone give some suggestions?
This is configuration file:
"analytics":
"reporting_enabled": false
"auth_enabled": true
"chunk_store_config":
"chunk_cache_config":
"enable_fifocache": true
"fifocache":
"max_size_bytes": "500MB"
"max_look_back_period": "0s"
"common":
"compactor_grpc_address": "server2:port1"
"compactor":
"compaction_interval": "10m"
"compactor_ring":
"kvstore":
"store": "memberlist"
"max_compaction_parallelism": 10
"retention_enabled": true
"shared_store": "cos"
"working_directory": "/myloki/compactor"
"distributor":
"ring":
"kvstore":
"store": "memberlist"
"frontend":
"compress_responses": true
"scheduler_address": "server3:port1"
"tail_proxy_url": "server4:port2"
"frontend_worker":
"grpc_client_config":
"max_send_msg_size": 104857600
"match_max_concurrent": true
"parallelism": 10
"scheduler_address": "server3:port1"
"ingester":
"chunk_block_size": 262144
"chunk_encoding": "snappy"
"chunk_idle_period": "1m"
"chunk_retain_period": "0s"
"chunk_target_size": 1572864
"lifecycler":
"heartbeat_period": "5s"
"interface_names":
- "eth0"
"join_after": "60s"
"num_tokens": 512
"ring":
"heartbeat_timeout": "1m"
"kvstore":
"store": "memberlist"
"max_transfer_retries": 0
"wal":
"checkpoint_duration": "5m"
"dir": "/myloki/wal"
"enabled": true
"replay_memory_ceiling": "4GB"
"ingester_client":
"grpc_client_config":
"max_recv_msg_size": 67108864
"remote_timeout": "1s"
"limits_config":
"cardinality_limit": 100000
"creation_grace_period": "10m"
"enforce_metric_name": false
"ingestion_burst_size_mb": 1500
"ingestion_rate_mb": 1100
"ingestion_rate_strategy": "global"
"max_cache_freshness_per_query": "10m"
"max_chunks_per_query": 2000000
"max_entries_limit_per_query": 5000
"max_global_streams_per_user": 10000
"max_label_name_length": 1024
"max_label_names_per_series": 30
"max_label_value_length": 2048
"max_line_size": 256000
"max_queriers_per_tenant": 0
"max_query_length": "721h"
"max_query_parallelism": 16
"max_query_series": 500
"max_streams_per_user": 1000
"per_stream_rate_limit": "3MB"
"per_stream_rate_limit_burst": "15MB"
"query_timeout": "1h"
"reject_old_samples": true
"reject_old_samples_max_age": "24h"
"retention_period": "720h"
"split_queries_by_interval": "30m"
"memberlist":
"abort_if_cluster_join_fails": false
"bind_port": 7946
"join_members":
- "server5:port3"
"max_join_backoff": "1m"
"max_join_retries": 10
"min_join_backoff": "1s"
"querier":
"engine":
"max_look_back_period": "30s"
"timeout": "3m"
"extra_query_delay": "0s"
"max_concurrent": 16
"query_ingesters_within": "3h"
"query_timeout": "1h"
"tail_max_duration": "1h"
"query_range":
"align_queries_with_step": true
"cache_results": true
"max_retries": 5
"parallelise_shardable_queries": true
"results_cache":
"cache":
"embedded_cache":
"enabled": true
"max_size_mb": 100
"ttl": "3h"
"compression": "snappy"
"query_scheduler":
"max_outstanding_requests_per_tenant": 32768
"schema_config":
"configs":
- "from": "2020-10-01"
"index":
"period": "24h"
"prefix": "loki_index_"
"object_store": "cos"
"schema": "v11"
"store": "boltdb-shipper"
- "from": "2023-06-15"
"index":
"period": "24h"
"prefix": "loki_index_"
"object_store": "cos"
"schema": "v12"
"store": "tsdb"
"server":
"graceful_shutdown_timeout": "5s"
"grpc_server_max_concurrent_streams": 1000
"grpc_server_max_recv_msg_size": 104857600
"grpc_server_max_send_msg_size": 104857600
"grpc_server_min_time_between_pings": "10s"
"grpc_server_ping_without_stream_allowed": true
"http_listen_port": 3100
"http_server_idle_timeout": "120s"
"http_server_read_timeout": "600s"
"http_server_write_timeout": "600s"
"log_format": "json"
"log_level": "debug"
"storage_config":
"boltdb_shipper":
"active_index_directory": "/myloki/index"
"cache_location": "/myloki/index_cache"
"cache_ttl": "24h"
"index_gateway_client":
"server_address": "server1:port1"
"resync_interval": "5m"
"shared_store": "cos"
"cos":
"forcepathstyle": true
"index_cache_validity": "3h"
"index_queries_cache_config":
"embedded_cache":
"enabled": true
"max_size_mb": 100
"ttl": "3h"
"tsdb_shipper":
"active_index_directory": "/myloki/index/tsdb"
"cache_location": "/myloki/index/tsdb/cache"
"index_gateway_client":
"server_address": "server1:port1"
"shared_store": "cos"