- Loki - 2.9.14, MacOS
- single-node setup, no HA, all components on same host.
- MinIO as object store & kvstore = memberlist
- using Fluent-bit as a log scraper and sending to Loki
- max log velocity- for testing = 10 MB/sec & for production = 500 MB/sec
- gets the following warn/error after 15-20 minutes after starting
- We get the same type of error even for kvstore = in-memory
- /ring runs fine.
- Is this a default behaviour?
- Do we need to run the read, write & backend components as separate entities?
- I don’t want to run separate components; all should be on the same host (No microservices, Docker, or Kubernetes)
- Can I get such a Loki configuration?
level=error ts=2025-05-10T03:06:29.242323Z caller=compactor.go:462 msg="error asking ring for who should run the compactor, will check again" err="at least 1 healthy replica required, could only find 0 - unhealthy instances: 127.0.0.1:9095"
level=warn ts=2025-05-10T03:06:29.803726Z caller=server.go:2220 traceID=1ff7b37df06a6002 orgID=fake msg="POST /loki/api/v1/push (500) 471.282µs Response: \"at least 1 live replicas required, could only find 0 - unhealthy instances: 127.0.0.1:9095\\n\" ws: false; Connection: keep-alive; Content-Length: 16055; Content-Type: application/json; User-Agent: Fluent-Bit; "
# Loki 2.9.14
# MinIO + memberlist
# target:
auth_enabled: false
ballast_bytes: 10000
server:
http_listen_address: 127.0.0.1
http_listen_port: 3100
grpc_listen_address: 127.0.0.1
grpc_listen_port: 9095
http_server_read_timeout: 10m
http_server_write_timeout: 10m
http_server_idle_timeout: 5m
grpc_server_max_recv_msg_size: 41943040
grpc_server_max_send_msg_size: 41943040
grpc_server_max_concurrent_streams: 0
grpc_server_max_connection_age: 5m
log_level: warn
distributor:
ring:
kvstore:
store: memberlist
heartbeat_period: 5s # default = 5s
heartbeat_timeout: 5m # default = 1m
querier:
tail_max_duration: 12h # default = 1h
query_ingesters_within: 0h # default = 3h
engine:
timeout: 5m # default = 5m
max_look_back_period: 168h # 7 days, default = 30s
max_concurrent: 4 # default = 4, 2-core CPU → 4, 4-core CPU → 8
query_scheduler:
max_outstanding_requests_per_tenant: 64000 # default = 32000
grpc_client_config:
max_recv_msg_size: 209715200 # default = 104857600 (bytes)
max_send_msg_size: 209715200 # default = 104857600 (bytes)
grpc_compression: 'snappy' # default = ""
rate_limit: 0 # default = 0
rate_limit_burst: 0 # default = 0
backoff_on_ratelimits: true # default = false
backoff_config:
min_period: 100ms # default = 100ms
max_period: 10s # default = 10s
max_retries: 10 # default = 10
tls_enabled: false # default = false
connect_timeout: 0s # default = 5s
use_scheduler_ring: false # default = false
scheduler_ring:
kvstore:
store: memberlist # default = "consul"
prefix: "collectors/" # default = "collectors/"
frontend:
log_queries_longer_than: 60s # default = 0s
query_stats_enabled: true # default = false
max_outstanding_per_tenant: 8192 # default = 2048
grpc_client_config:
max_recv_msg_size: 209715200 # default = 104857600 (bytes)
max_send_msg_size: 209715200 # default = 104857600 (bytes)
grpc_compression: 'snappy' # default = ""
rate_limit: 0 # default = 0
rate_limit_burst: 0 # default = 0
backoff_on_ratelimits: true # default = false
backoff_config:
min_period: 100ms # default = 100ms
max_period: 10s # default = 10s
max_retries: 10 # default = 10
tls_enabled: false # default = false
connect_timeout: 0s # default = 5s
compress_responses: true
query_range:
align_queries_with_step: true # default = false
results_cache:
cache:
embedded_cache:
enabled: true # default = false
max_size_mb: 300 # default = 100
ttl: 1h # default = 1h
compression: 'snappy' # default = ""
cache_results: true # default = false
max_retries: 15 # default = 5
parallelise_shardable_queries: true # default = true
cache_index_stats_results: true # default = true
index_stats_results_cache:
cache:
embedded_cache:
enabled: true # default = false
max_size_mb: 300 # default = 100
ttl: 1h # default = 1h
compression: 'snappy' # default = ""
ruler:
storage:
type: local
local:
directory: /Users/sagar/data/loki/rules
rule_path: /Users/sagar/data/loki/rules
alertmanager_url: http://127.0.0.1:9093
ring:
kvstore:
store: memberlist
enable_api: true
ingester_client:
pool_config:
client_cleanup_period: 20s # default = 15s
health_check_ingesters: false # default = true
remote_timeout: 5s # default = 1s
remote_timeout: 10s # default = 5s
grpc_client_config:
max_recv_msg_size: 209715200 # default = 104857600 (bytes)
max_send_msg_size: 209715200 # default = 104857600 (bytes)
grpc_compression: 'snappy' # default = ""
rate_limit: 0 # default = 0
rate_limit_burst: 0 # default = 0
backoff_on_ratelimits: true # default = false
backoff_config:
min_period: 100ms # default = 100ms
max_period: 10s # default = 10s
max_retries: 10 # default = 10
tls_enabled: false # default = false
connect_timeout: 0s # default = 5s
ingester:
lifecycler:
ring:
kvstore:
store: memberlist
prefix: "collectors/" # default = "collectors/"
heartbeat_timeout: 5m # default = 1m
replication_factor: 1 # default = 3
heartbeat_period: 5s # default = 5s
heartbeat_timeout: 5m # default = 1m
min_ready_duration: 15s # default = 15s
final_sleep: 0s # default = 0s
address: 127.0.0.1 # default = ""
port: 9095 # default = 0
concurrent_flushes: 64 # default = 32
flush_check_period: 30s # default = 30s
flush_op_timeout: 10m # default = 10m
chunk_retain_period: 5s # To ease recovery, default = 0s
chunk_idle_period: 5m # Close chunks after no more writes, default = 30m
chunk_block_size: 262144 # 256KB, default = 262144
chunk_target_size: 1000000 # ~1.5MB target chunk size, default = 1572864
chunk_encoding: "snappy" # default = "gzip"
max_chunk_age: 1h # default = 2h
autoforget_unhealthy: false # default = false
query_store_max_look_back_period: 0s # default = 0s
wal:
enabled: true # default = true
dir: /Users/sagar/data/loki/wal
index_gateway:
mode: "ring" # default = "simple"
ring:
kvstore:
store: memberlist # default = "consul"
prefix: "collectors/" # default = "collectors/"
heartbeat_period: 15s # default = 15s
heartbeat_timeout: 5m # default = 1m
storage_config:
tsdb_shipper:
active_index_directory: /Users/sagar/data/loki/index
shared_store: s3
shared_store_key_prefix: "index/" # default = "index/"
cache_location: /Users/sagar/data/loki/index_cache
cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
resync_interval: 10m # default = 5m
query_ready_num_days: 1 # default = 0
index_gateway_client:
grpc_client_config:
max_recv_msg_size: 209715200 # default = 104857600 (bytes)
max_send_msg_size: 209715200 # default = 104857600 (bytes)
grpc_compression: 'snappy' # default = ""
rate_limit: 0 # default = 0
rate_limit_burst: 0 # default = 0
backoff_on_ratelimits: true # default = false
backoff_config:
min_period: 100ms # default = 100ms
max_period: 10s # default = 10s
max_retries: 10 # default = 10
tls_enabled: false # default = false
connect_timeout: 0s # default = 5s
server_address: 127.0.0.1 # default = ""
filesystem:
directory: /Users/sagar/data/loki/chunks
aws:
s3: http://127.0.0.1:9000
bucketnames: loki-data
access_key_id: minioadmin
secret_access_key: minioadmin
s3forcepathstyle: true
insecure: true
index_queries_cache_config:
max_parallel_get_chunk: 150 # default = 150
max_chunk_batch_size: 100 # default = 50
chunk_store_config:
chunk_cache_config:
embedded_cache:
enabled: true # default = false
max_size_mb: 300 # default = 100
ttl: 1h # default = 1h
schema_config:
configs:
- from: 2022-01-01
store: tsdb
object_store: s3
schema: v12
index:
prefix: index_
period: 24h
chunks:
prefix: chunks_
period: 24h
compactor:
working_directory: /Users/sagar/data/loki/compactor
compaction_interval: 5m # default = 10m
retention_enabled: true # default = false
retention_delete_delay: 1h # default = 2h
retention_delete_worker_count: 150 # default = 150
delete_request_store: s3 # default = ""
delete_batch_size: 100 # default = 70
max_compaction_parallelism: 2 # default = 1
compactor_ring:
kvstore:
store: memberlist
prefix: "collectors/" # default = "collectors/"
heartbeat_period: 15s # default = 15s
heartbeat_timeout: 5m # default = 1m
tables_to_compact: 2 # default = 0
limits_config:
ingestion_rate_mb: 500 # 500MB/s, default = 4
ingestion_burst_size_mb: 1000 # Burst capability for spikes, default = 6
max_label_name_length: 1024 # default = 1024
max_label_value_length: 5120 # default = 2048
reject_old_samples: true
reject_old_samples_max_age: 2w # default = 1w (window) = max_chunk_age/2
max_line_size: 16777212 # 10MB to handle long log lines, default = 256KB
max_global_streams_per_user: 100000 # default = 5000
per_stream_rate_limit: 20MB # default = 3MB
per_stream_rate_limit_burst: 40MB # default = 15MB
max_query_parallelism: 64 # default = 32
tsdb_max_query_parallelism: 32 # default = 128
max_entries_limit_per_query: 10000 # default = 5000
split_queries_by_interval: 6h # default = 1h
volume_enabled: true
retention_period: 168h # 7 days retention
frontend_worker:
frontend_address: 127.0.0.1:9095 # default = ""
grpc_client_config:
max_recv_msg_size: 209715200 # default = 104857600 (bytes)
max_send_msg_size: 209715200 # default = 104857600 (bytes)
grpc_compression: 'snappy' # default = ""
rate_limit: 0 # default = 0
rate_limit_burst: 0 # default = 0
backoff_on_ratelimits: true # default = false
backoff_config:
min_period: 100ms # default = 100ms
max_period: 10s # default = 10s
max_retries: 10 # default = 10
tls_enabled: false # default = false
connect_timeout: 0s # default = 5s
table_manager:
retention_deletes_enabled: true # default = false
retention_period: 168h # 7 days retention
memberlist:
join_members:
- 127.0.0.1:7946
bind_addr: ['127.0.0.1']
bind_port: 7946
gossip_interval: 1s
gossip_to_dead_nodes_time: 15s
left_ingesters_timeout: 30s
# runtime_config:
# tracing:
analytics:
reporting_enabled: false
common:
path_prefix: /loki
storage:
s3:
endpoint: http://127.0.0.1:9000
insecure: true
bucketnames: loki-data
access_key_id: loki
secret_access_key: supersecret
s3forcepathstyle: true
replication_factor: 1
ring:
kvstore:
store: inmemory
heartbeat_period: 15s # default = 15s
heartbeat_timeout: 5m # default = 1m
instance_addr: 127.0.0.1
instance_addr: 127.0.0.1
compactor_address: 127.0.0.1:3100 # default = ""
compactor_grpc_address: 127.0.0.1:9095 # default = ""
shutdown_delay: 3s # # default = 0s
If you are running multiple instances then your listen address needs to not be localhost or 127.0.0.1, because each Loki instance needs to be uniquely identifiable from each other for ring membership to work (/ring should show the actual IP for all your Loki instances). You also need to change things like your compactor_address to a DNS record that can resolve to all your Loki instances.
Also I am not sure prefix should be set to the same value for components, but I don’t set prefix so I could be wrong.
