I am attempting to use this guide - Migrate from `loki-distributed` Helm chart | Grafana Loki documentation - to migrate from loki-distributed to the new Loki 3 Helm chart. I am also migrating from boltdb shipper to the tsdb.
I configured the new loki to join the old loki’s memberlist and it can see logs in the cluster. However the new Grafana datasource that uses the new loki is having 5 second query times for 7 days worth of logs - kilobytes of data. The old loki takes less than a second for the same query. Additionally the new loki does not see the old logs. What could I be doing wrong?
Loki config:
migrate:
fromDistributed:
enabled: true
memberlistService: loki-loki-distributed-memberlist
nameOverride: loki-next
fullnameOverride: loki-next
loki:
containerSecurityContext:
readOnlyRootFilesystem: false
auth_enabled: false
schemaConfig:
configs:
- from: 2024-07-15
store: boltdb-shipper
object_store: s3
schema: v13
index:
prefix: index_
period: 24h
- from: "2025-05-28"
store: tsdb
object_store: s3
schema: v13
index:
prefix: index_
period: 24h
storage_config:
aws:
region: ${region}
bucketnames: ${bucket_chunks}
s3forcepathstyle: true
boltdb_shipper:
active_index_directory: /var/loki/index
cache_location: /var/loki/boltdb-cache
cache_ttl: 168h
index_gateway_client:
server_address: dns:///loki-next-index-gateway:9095
tsdb_shipper:
active_index_directory: /var/loki/tsdb-index
cache_location: /var/loki/tsdb-cache
storage:
type: "s3"
s3:
region: ${region}
bucketNames:
chunks: ${bucket_chunks}
ruler: ${bucket_ruler}
ingester:
chunk_encoding: snappy
querier:
max_concurrent: 16
query_ingesters_within: 3h
query_scheduler:
max_outstanding_requests_per_tenant: 64000
pattern_ingester:
enabled: true
limits_config:
allow_structured_metadata: false
split_queries_by_interval: 15m
allow_deletes: true
ingestion_burst_size_mb: 200
ingestion_rate_mb: 100
ingestion_rate_strategy: local
max_cache_freshness_per_query: 10m
max_global_streams_per_user: 10000
max_query_length: 12000h
max_query_parallelism: 16
max_streams_per_user: 0
per_stream_rate_limit: 1000MB
per_stream_rate_limit_burst: 200MB
query_timeout: 5m
reject_old_samples: true
reject_old_samples_max_age: 168h
retention_period: 2880h
compactor:
working_directory: /var/loki/retention
compaction_interval: 10m
retention_delete_delay: 2h
retention_delete_worker_count: 150
retention_enabled: true
delete_request_store: s3
query_range:
# make queries more cache-able by aligning them with their step intervals
align_queries_with_step: true
max_retries: 5
cache_results: true
results_cache:
cache:
enable_fifocache: true
fifocache:
max_size_items: 1024
validity: 24h
commonConfig:
path_prefix: /var/loki
replication_factor: 3
compactor_address: "http://loki-next-compactor:3100"
serviceAccount:
name: logs-manager
annotations:
"eks.amazonaws.com/role-arn": ${service_account_arn}
deploymentMode: Distributed
chunksCache:
allocatedMemory: 11264
batchSize: 6
parallelism: 9
maxItemMemory: 9
nodeSelector:
role: metricsmem
tolerations:
- key: role
operator: Equal
value: metricsmem
effect: NoSchedule
queryScheduler:
replicas: 2
nodeSelector:
role: metrics
tolerations:
- key: role
operator: Equal
value: metrics
effect: NoSchedule
querier:
nodeSelector:
role: metrics
tolerations:
- key: role
operator: Equal
value: metrics
effect: NoSchedule
replicas: 2
maxUnavailable: 1
resources:
requests:
cpu: 2
memory: 2Gi
limits:
cpu: 3
memory: 4Gi
indexGateway:
maxUnavailable: 2
replicas: 3
nodeSelector:
role: metrics
tolerations:
- key: role
operator: Equal
value: metrics
effect: NoSchedule
resources:
requests:
cpu: 1
memory: 1Gi
limits:
cpu: 2
memory: 2Gi
compactor:
replicas: 1
nodeSelector:
role: metrics
tolerations:
- key: role
operator: Equal
value: metrics
effect: NoSchedule
ingester:
resources:
limits:
cpu: "2"
memory: 2Gi
requests:
cpu: "1"
memory: 1Gi
replicas: 3
maxUnavailable: 1
nodeSelector:
role: metrics
tolerations:
- key: role
operator: Equal
value: metrics
effect: NoSchedule
distributor:
replicas: 2
maxUnavailable: 1
nodeSelector:
role: metrics
tolerations:
- key: role
operator: Equal
value: metrics
effect: NoSchedule
queryFrontend:
replicas: 3
maxUnavailable: 2
resources:
limits:
cpu: "3"
memory: 6Gi
requests:
cpu: "2"
memory: 4Gi
nodeSelector:
role: metrics
tolerations:
- key: role
operator: Equal
value: metrics
effect: NoSchedule
gateway:
service:
type: NodePort
nodePort: 32438
image:
# -- The Docker registry for the gateway image
registry: docker.io
# -- The gateway image repository
repository: nginxinc/nginx-unprivileged
# -- The gateway image tag
tag: 1.27.3-alpine
# -- The gateway image pull policy
pullPolicy: IfNotPresent
nodeSelector:
role: metrics
tolerations:
- key: role
operator: Equal
value: metrics
effect: NoSchedule
bloomCompactor:
replicas: 0
bloomGateway:
replicas: 0
backend:
replicas: 0
read:
replicas: 0
write:
replicas: 0
singleBinary:
replicas: 0
minio:
enabled: false
Thanks in advance!