Compactor is not working

Grafana Loki
1

My config Loki:
Loki v.3.4.2
Grafana v.11..
#################

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper                     
      object_store: filesystem                  
      schema: v13                               
      index:
        prefix: index_loki_blob
        period: 24h                            

storage_config:
  boltdb_shipper:
    active_index_directory: /data/loki/boltdb-shipper-active
    cache_location: /data/loki/boltdb-cache
    cache_ttl: 24h                             
  filesystem:
    directory: /data/loki/chunks

compactor:
  compaction_interval: 1h               
  delete_request_cancel_period: 12h             
  delete_request_store: filesystem            
  retention_delete_delay: 2h                 
  retention_delete_worker_count: 10           
  retention_enabled: true                       
  working_directory: /data/loki/compactor      

limits_config:
  allow_structured_metadata: false
  ingestion_burst_size_mb: 200                 
  ingestion_rate_mb: 20                         
  ingestion_rate_strategy: local
  max_chunks_per_query: 200000                 
  max_entries_limit_per_query: 1000           
  max_global_streams_per_user: 524288000      
  max_query_lookback: 14d                     
  max_query_parallelism: 8                    
  max_query_series: 100000                     
  max_streams_per_user: 100000                
  per_stream_rate_limit: "512MB"            
  per_stream_rate_limit_burst: "1024MB"      
  reject_old_samples: true                     
  reject_old_samples_max_age: 336h      
  retention_period: 336h                        
  split_queries_by_interval: 15m  

querier:
  engine:                                   
    max_look_back_period: 336h               
  max_concurrent: 20                         
  query_ingesters_within: 2h 

table_manager:
  retention_deletes_enabled: true               
  retention_period: 336h

compactor does not work, after 14 days, the old data is not deleted and the disk fills up until it runs out. What to do? What should I do?

2

Try disabling table_manager. If that still doesn’t work, see if you can find any error log from compactor.

3

The table_manager retention settings do not apply when you’re using boltdb-shipper.
boltdb-shipper retention relies only on the compactor, not table_manager.
You can try

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v13
      index:
        prefix: index_loki_blob
        period: 24h
      **retention_period: 336h**  # <- You should move/add this here

The retention_period under limits_config is global, but schema_config.configs.retention_period is what the compactor uses.

Try check the log for loki, and look for something like
"“retention applied to table index_20240610, marked for deletion”.

5

when adding retention_period: 336h to schema_configure, I have an error:
failed parsing config: /etc/loki/loki-local-config.yaml: yaml: unmarshal errors:
line 55: field retention_period not found in type struct { PathPrefix string “yaml:“path_prefix””; Prefix string “yaml:“prefix””; Period model.Duration “yaml:“period””; Tags config.Tags “yaml:“tags”” }. Use -config.expand-env=true flag if you want to expand environment variables in your config file

6

i have this loki config on k8s render from helm loki chart 6.46.0

auth_enabled: false

bloom_build:

builder:

planner_address: loki-backend-headless.platform-logging.svc.cluster.local:9095

enabled: false

bloom_gateway:

client:

addresses: dnssrvnoa+_grpc._tcp.loki-backend-headless.platform-logging.svc.cluster.local

enabled: false

chunk_store_config:

chunk_cache_config:

background:

writeback_buffer: 500000

writeback_goroutines: 1

writeback_size_limit: 500MB

default_validity: 0s

memcached:

batch_size: 4

parallelism: 5

memcached_client:

addresses: dnssrvnoa+_memcached-client._tcp.loki-chunks-cache.platform-logging.svc.cluster.local

consistent_hash: true

max_idle_conns: 72

timeout: 2000ms

common:

compactor_grpc_address: ‘loki-backend.platform-logging.svc.cluster.local:9095’

path_prefix: /var/loki

replication_factor: 3

storage:

s3:

access_key_id: nqTUcUxxxxxx

bucketnames: loki-logging

endpoint: http://uat-minio-logging.com.vn:9000

http_config:

idle_conn_timeout: 90s

insecure_skip_verify: false

response_header_timeout: 0s

insecure: false

s3forcepathstyle: true

secret_access_key: 1Wr8lhGYK8sJN362ZFg

compactor:

compaction_interval: 5m

delete_request_store: filesystem

retention_delete_delay: 10m

retention_enabled: true

working_directory: /tmp/loki/compactor

frontend:

scheduler_address: “”

tail_proxy_url: “”

frontend_worker:

scheduler_address: “”

index_gateway:

mode: simple

limits_config:

allow_structured_metadata: true

max_cache_freshness_per_query: 10m

query_timeout: 300s

reject_old_samples: true

reject_old_samples_max_age: 168h

retention_period: 360h

split_queries_by_interval: 15m

volume_enabled: true

memberlist:

join_members:

  • loki-memberlist.platform-logging.svc.cluster.local

pattern_ingester:

enabled: false

query_range:

align_queries_with_step: true

cache_results: true

results_cache:

cache:

background:

writeback_buffer: 500000

writeback_goroutines: 1

writeback_size_limit: 500MB

default_validity: 12h

memcached_client:

addresses: dnssrvnoa+_memcached-client._tcp.loki-results-cache.platform-logging.svc.cluster.local

consistent_hash: true

timeout: 500ms

update_interval: 1m

ruler:

storage:

s3:

access_key_id: nqTUcUGSxxxxx

bucketnames: loki-logging

endpoint: http://uat-minio-logging.com.vn:9000

http_config:

idle_conn_timeout: 90s

insecure_skip_verify: false

response_header_timeout: 0s

insecure: false

s3forcepathstyle: true

secret_access_key: 1Wr8lxxxxxYK8sJN362ZFg

type: s3

wal:

dir: /var/loki/ruler-wal

runtime_config:

file: /etc/loki/runtime-config/runtime-config.yaml

schema_config:

configs:

  • from: “2024-04-01”

index:

period: 24h

prefix: index_

object_store: s3

schema: v13

store: tsdb

server:

grpc_listen_port: 9095

http_listen_port: 3100

http_server_read_timeout: 600s

http_server_write_timeout: 600s

storage_config:

bloom_shipper:

working_directory: /var/loki/data/bloomshipper

boltdb_shipper:

index_gateway_client:

server_address: dns+loki-backend-headless.platform-logging.svc.cluster.local:9095

hedging:

at: 250ms

max_per_second: 20

up_to: 3

tsdb_shipper:

index_gateway_client:

server_address: dns+loki-backend-headless.platform-logging.svc.cluster.local:9095

use_thanos_objstore: false

tracing:

enabled: false

but retention not working too, can you check for me. Thank you!