My Loki cluster is not respecting my log retention settings and is storing the logs indefinitely.
My config:
Loki version 3.5.5
target: read
auth_enabled: true
common:
compactor_address: http://localhost:9443
replication_factor: 3
ring:
heartbeat_timeout: 10m
kvstore:
store: memberlist
compactor:
compaction_interval: 1m
delete_request_store: s3
retention_enabled: true
working_directory: /loki/compactor
frontend:
compress_responses: true
log_queries_longer_than: 15s
frontend_worker:
frontend_address: test.frontend.loki.it.ufl.edu:9443
grpc_client_config:
max_send_msg_size: 104857600.0
ingester:
chunk_idle_period: 1h
flush_check_period: 10s
max_chunk_age: 2h
wal:
replay_memory_ceiling: 6047MB
limits_config:
allow_structured_metadata: false
bloom_gateway_enable_filtering: true
ingestion_burst_size_mb: 60
ingestion_rate_mb: 40
max_cache_freshness_per_query: 10m
max_entries_limit_per_query: 100000
max_global_streams_per_user: 20000
max_query_parallelism: 6
max_query_series: 10000
per_stream_rate_limit: 40MB
per_stream_rate_limit_burst: 60MB
query_timeout: 3m
reject_old_samples: true
retention_period: 48h
split_queries_by_interval: 15m
memberlist:
abort_if_cluster_join_fails: false
bind_port: 7946
join_members:
- az1-ict-o11y-test-loki-read-pve-1.server
- az1-ict-o11y-test-loki-read-pve-4.server
- az1-ict-o11y-test-loki-read-pve-2.server
- az1-ict-o11y-test-loki-read-pve-3.server
- az2-ict-o11y-test-loki-read-pve-1.server
- az2-ict-o11y-test-loki-read-pve-2.server
- az2-ict-o11y-test-loki-read-pve-3.server
- az1-ict-o11y-test-loki-write-pve-1.server
- az1-ict-o11y-test-loki-write-pve-2.server
- az1-ict-o11y-test-loki-write-pve-3.server
- az2-ict-o11y-test-loki-write-pve-1.server
- az2-ict-o11y-test-loki-write-pve-2.server
max_join_backoff: 1m
max_join_retries: 10
min_join_backoff: 1s
rejoin_interval: 1m
querier:
max_concurrent: 6000
multi_tenant_queries_enabled: true
query_range:
align_queries_with_step: true
cache_results: true
max_retries: 5
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 2048
ttl: 1h
query_scheduler:
max_outstanding_requests_per_tenant: 42768
schema_config:
configs: - from: ‘2025-09-27’
index:
period: 24h
prefix: index_tsdb_
object_store: aws
schema: v13
store: tsdb
server:
grpc_listen_port: 9443
grpc_server_max_concurrent_streams: 1500
grpc_server_max_recv_msg_size: 104857600
grpc_server_max_send_msg_size: 104857600
http_listen_port: 8443
http_server_read_timeout: 610s
http_server_write_timeout: 610s
http_tls_config:
cert_file: /etc/loki/ssl/cert.crt
key_file: /etc/loki/ssl/cert.key
log_level: info
storage_config:
aws:
access_key_id: OMITTED
bucketnames: irs-loki-test
endpoint: ict-mc2-oss02.server.ufl.edu
http_config:
insecure_skip_verify: true
insecure: false
region: default
s3forcepathstyle: true
secret_access_key: OMITTED
hedging:
at: 250ms
max_per_second: 20
up_to: 3
tsdb_shipper:
active_index_directory: /loki/tsdb-shipper-active
cache_location: /loki/tsdb-shipper-cache
Any help is greatly appreciated!