Hi,
I’m getting the following error after upgrading from boltdb to tsdb:
"level=error ts=2024-08-02T15:33:51.19951968Z caller=table.go:360 table-name=loki_index_19937 org_id=fake msg=\"index set has some problem, cleaning it up\" err=\"invalid tsdb path: /data/tsdb-cache/loki_index_19937/loki-write-2-1698694025434275042-1722564000\""
I’m using Loki v3.0.0 and this is my Loki configuration:
schemaConfig:
configs:
- from: "2021-05-12"
store: boltdb-shipper
object_store: s3
schema: v13
index:
prefix: loki_index_
period: 24h
- from: "2024-07-12"
store: tsdb
object_store: s3
schema: v13
index:
period: 24h
prefix: loki_index_
storage_config:
aws:
s3: s3://us-east-1/addi-loki-logs
insecure: false
s3forcepathstyle: false
http_config:
response_header_timeout: 10s
boltdb_shipper:
active_index_directory: /var/loki/index
tsdb_shipper:
active_index_directory: /data/tsdb-index
cache_location: /data/tsdb-cache
index_queries_cache_config:
memcached:
batch_size: 256
parallelism: 10
memcached_client:
host: index-cache-memcached.loki.svc.cluster.local
service: index-cache-memcached
period_config:
object_store: s3
limits_config:
max_global_streams_per_user: 100000
ingestion_rate_mb: 130
ingestion_burst_size_mb: 135
query_timeout: 3m
split_queries_by_interval: 5m
max_query_series: 100000
max_query_parallelism: 10000
per_stream_rate_limit: 40MB
per_stream_rate_limit_burst: 70MB
max_entries_limit_per_query: 20000
max_line_size: 0
reject_old_samples: false
reject_old_samples_max_age: 90d
ingester:
max_chunk_age: 730h
chunk_target_size: 71000000
querier:
max_concurrent: 10000
query_scheduler:
max_outstanding_requests_per_tenant: 10000
structuredConfig:
query_range:
cache_results: true
align_queries_with_step: true
server:
http_server_read_timeout: 30m
http_server_write_timeout: 30m
http_server_idle_timeout: 30m
grpc_server_max_recv_msg_size: 31457281
grpc_server_max_send_msg_size: 31457281
frontend_worker:
grpc_client_config:
max_send_msg_size: 268435456
frontend:
log_queries_longer_than: 10s