Hey, I am facing this problem: msg=“error getting ingester clients” err=“empty ring”
fullnameOverride: "test-loki"
nameOverride: "test-loki"
deploymentMode: SimpleScalable
######################################################################################################################
#
# Base Loki Configs including kubernetes configurations and configurations for Loki itself,
# see below for more specifics on Loki's configuration.
#
######################################################################################################################
loki:
configStorageType: ConfigMap
nodeSelector:
node_pool: "node-test"
# Should authentication be enabled
auth_enabled: false
commonConfig:
replication_factor: 1
ring:
kvstore:
store: inmemory
# -- Check https://grafana.com/docs/loki/latest/configuration/#server for more info on the server configuration.
server:
http_listen_port: 3100
grpc_listen_port: 9095
http_server_read_timeout: 600s
http_server_write_timeout: 600s
# -- Limits config
limits_config:
volume_enabled: true
allow_structured_metadata: true
tracing:
enabled: true
ingester:
chunk_encoding: snappy
# -- Storage config. Providing this will automatically populate all necessary storage configs in the templated config.
# storage:
# # Loki requires a bucket for chunks and the ruler. GEL requires a third bucket for the admin API.
# bucketNames:
# chunks: chunks
# ruler: ruler
# admin: admin
# type: s3
# s3:
# s3: "http://loki:loki-secret@test-loki-minio.default.svc.cluster.local:9000"
# endpoint: <endpoint>
# region: us-east-1
# secretAccessKey: loki-secret
# accessKeyId: loki
# insecure: true
# gcs:
# chunkBufferSize: 0
# requestTimeout: "0s"
# enableHttp2: true
# filesystem:
# chunks_directory: /var/loki/chunks
# rules_directory: /var/loki/rules
# admin_api_directory: /var/loki/admin
# -- Check https://grafana.com/docs/loki/latest/configuration/#schema_config for more info on how to configure schemas
schemaConfig:
configs:
- from: "2024-06-01"
store: tsdb
object_store: s3
schema: v13
index:
prefix: loki_index_
period: 24h
memcached:
chunk_cache:
enabled: true
host: ""
service: "memcached-client"
batch_size: 256
parallelism: 10
results_cache:
enabled: false
host: ""
service: "memcached-client"
timeout: "500ms"
default_validity: "12h"
######################################################################################################################
#
# Gateway and Ingress
#
# By default this chart will deploy a Nginx container to act as a gateway which handles routing of traffic
# and can also do auth.
#
# If you would prefer you can optionally disable this and enable using k8s ingress to do the incoming routing.
#
######################################################################################################################
# Configuration for the gateway
gateway:
enabled: true
replicas: 1
containerPort: 8080
# -- Enable logging of 2xx and 3xx HTTP requests
verboseLogging: true
deploymentStrategy:
type: RollingUpdate
nodeSelector:
node_pool: "node-test"
affinity: {}
# Gateway service configuration
service:
port: 80
type: ClusterIP
######################################################################################################################
#
# Simple Scalable Deployment (SSD) Mode
#
# For small to medium size Loki deployments up to around 1 TB/day
#
######################################################################################################################
# Configuration for the write pod(s)
write:
replicas: 1
nodeSelector:
node_pool: "node-test"
affinity: {}
# -- Configuration for the read pod(s)
read:
replicas: 1
nodeSelector:
node_pool: "node-test"
affinity: {}
# -- Configuration for the backend pod(s)
backend:
replicas: 1
nodeSelector:
node_pool: "node-test"
affinity: {}
######################################################################################################################
#
# Subchart configurations
#
######################################################################################################################
minio:
enabled: true
replicas: 1
# Minio requires 2 to 16 drives for erasure code (drivesPerNode * replicas)
# https://docs.min.io/docs/minio-erasure-code-quickstart-guide
# Since we only have 1 replica, that means 2 drives must be used.
drivesPerNode: 2
rootUser: enterprise-logs
rootPassword: supersecret
buckets:
- name: chunks
policy: none
purge: false
- name: ruler
policy: none
purge: false
- name: admin
policy: none
purge: false
persistence:
size: 5Gi
annotations: {}
resources: {}
# requests:
# cpu: 100m
# memory: 128Mi
nodeSelector:
node_pool: "node-test"
#Testing
test:
enabled: false
lokiCanary:
enabled: false
# Zero out replica counts of other deployment modes
singleBinary:
replicas: 0
ingester:
replicas: 0
querier:
replicas: 0
queryFrontend:
replicas: 0
queryScheduler:
replicas: 0
distributor:
replicas: 0
compactor:
replicas: 0
indexGateway:
replicas: 0
bloomCompactor:
replicas: 0
bloomGateway:
replicas: 0
1 Like
Have you find any solution ? Actually I am also using loki with azureblob storage and my application logs are not send to loki .
emma.cn
January 30, 2025, 10:30am
3
No, still facing the problem
I faced this problem in my simple docker-compose local configuration!
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
grpc_server_max_concurrent_streams: 1000
common:
instance_addr: 127.0.0.1
path_prefix: /tmp/loki
storage:
filesystem:
chunks_directory: /tmp/loki/chunks
rules_directory: /tmp/loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 100
limits_config:
metric_aggregation_enabled: true
allow_structured_metadata: false
schema_config:
configs:
- from: 2020-02-02
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
storage_config:
filesystem:
directory: /tmp/loki/chunks
pattern_ingester:
enabled: true
metric_aggregation:
loki_address: localhost:3100
ruler:
alertmanager_url: http://localhost:9093
frontend:
encoding: protobuf
And docker-Compose:
...
loki:
image: grafana/loki:3.4.2
user: "6000:6000"
volumes:
- ./config/loki:/mnt/config/
- "/logs/loki:/tmp/loki"
command: -config.file=/mnt/config/loki_config.yml -target=all
healthcheck:
test: nslookup cadvisor. || exit 1
interval: 30s
timeout: 15s
retries: 2
start_period: 15s
deploy:
replicas: 1
restart_policy:
condition: any
update_config:
parallelism: 1
delay: 40s
failure_action: continue
monitor: 25s
networks:
- pv_net
the logs with debug enabled:
level=info ts=2025-02-27T12:53:41.941179651Z caller=main.go:126 msg="Starting Loki" version="(version=3.4.2, branch=release-3.4.x, revision=4fa045d3)"
level=info ts=2025-02-27T12:53:41.941239131Z caller=main.go:127 msg="Loading configuration file" filename=/mnt/config/loki_config.yml
level=debug ts=2025-02-27T12:53:41.941440958Z caller=modules.go:1754 msg="initializing ingester query tags interceptors"
level=info ts=2025-02-27T12:53:41.944627781Z caller=server.go:351 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9096
level=debug ts=2025-02-27T12:53:41.946152274Z caller=modules.go:1018 msg="initializing query frontend tripperware"
level=info ts=2025-02-27T12:53:41.947932213Z caller=table_manager.go:456 index-store=boltdb-shipper-1970-01-01 msg="loading local table index_20145"
level=info ts=2025-02-27T12:53:41.947894202Z caller=table_manager.go:136 index-store=boltdb-shipper-1970-01-01 msg="uploading tables"
level=debug ts=2025-02-27T12:53:41.948054251Z caller=table.go:105 table-name=index_20145 msg="opening locally present files for table index_20145" files="[- compactor-1740619919 d fake/]"
level=debug ts=2025-02-27T12:53:41.948113382Z caller=index_set.go:146 table-name=index_20145 user-id=fake msg="opened 0 local files, now starting sync operation"
level=debug ts=2025-02-27T12:53:41.948129701Z caller=index_set.go:316 table-name=index_20145 user-id=fake msg="syncing files for table index_20145"
level=debug ts=2025-02-27T12:53:41.948495153Z caller=index_set.go:323 table-name=index_20145 user-id=fake msg="updates for table index_20145. toDownload: [], toDelete: []"
level=debug ts=2025-02-27T12:53:41.948511253Z caller=index_set.go:154 table-name=index_20145 user-id=fake msg="finished syncing files"
level=debug ts=2025-02-27T12:53:41.948799851Z caller=index_set.go:146 table-name=index_20145 msg="opened 1 local files, now starting sync operation"
level=debug ts=2025-02-27T12:53:41.948814738Z caller=index_set.go:316 table-name=index_20145 msg="syncing files for table index_20145"
level=debug ts=2025-02-27T12:53:41.948830067Z caller=index_set.go:323 table-name=index_20145 msg="updates for table index_20145. toDownload: [], toDelete: []"
level=debug ts=2025-02-27T12:53:41.948839295Z caller=index_set.go:154 table-name=index_20145 msg="finished syncing files"
level=info ts=2025-02-27T12:53:41.948853611Z caller=table_manager.go:300 index-store=boltdb-shipper-1970-01-01 msg="query readiness setup completed" duration=2.614µs distinct_users_len=0 distinct_users=
level=info ts=2025-02-27T12:53:41.948883738Z caller=shipper.go:165 index-store=boltdb-shipper-1970-01-01 msg="starting index shipper in RW mode"
level=info ts=2025-02-27T12:53:41.949045569Z caller=table_manager.go:256 index-store=boltdb-shipper-1970-01-01 msg="loading table index_20146"
level=info ts=2025-02-27T12:53:41.949486812Z caller=table.go:318 msg="handing over indexes to shipper index_20146"
level=info ts=2025-02-27T12:53:41.949516107Z caller=table.go:334 msg="finished handing over table index_20146"
level=debug ts=2025-02-27T12:53:41.949529051Z caller=table.go:107 msg="snapshotting table index_20146"
level=debug ts=2025-02-27T12:53:41.949538689Z caller=table.go:170 msg="finished snapshotting table index_20146"
level=info ts=2025-02-27T12:53:41.949568355Z caller=index_client.go:91 index-store=boltdb-shipper-1970-01-01 msg="starting boltdb shipper in RW mode"
level=info ts=2025-02-27T12:53:41.949718595Z caller=table_manager.go:187 index-store=boltdb-shipper-1970-01-01 msg="handing over indexes to shipper"
level=info ts=2025-02-27T12:53:41.9498022Z caller=table.go:318 msg="handing over indexes to shipper index_20146"
level=info ts=2025-02-27T12:53:41.949817148Z caller=table.go:334 msg="finished handing over table index_20146"
level=debug ts=2025-02-27T12:53:41.949831946Z caller=table.go:107 msg="snapshotting table index_20146"
level=info ts=2025-02-27T12:53:41.949826185Z caller=table_manager.go:300 index-store=tsdb-2025-02-28 msg="query readiness setup completed" duration=1.563µs distinct_users_len=0 distinct_users=
level=debug ts=2025-02-27T12:53:41.949844209Z caller=table.go:170 msg="finished snapshotting table index_20146"
level=info ts=2025-02-27T12:53:41.949846463Z caller=shipper.go:165 index-store=tsdb-2025-02-28 msg="starting index shipper in RW mode"
level=info ts=2025-02-27T12:53:41.949957651Z caller=table_manager.go:136 index-store=tsdb-2025-02-28 msg="uploading tables"
level=info ts=2025-02-27T12:53:41.950293427Z caller=head_manager.go:308 index-store=tsdb-2025-02-28 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2025-02-27T12:53:41.950317592Z caller=manager.go:263 index-store=tsdb-2025-02-28 component=tsdb-manager msg="building WALs" n=0 ts=2025-02-27T12:53:41.950315288Z
level=debug ts=2025-02-27T12:53:41.950408161Z caller=manager.go:287 index-store=tsdb-2025-02-28 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2025-02-27T12:53:41.950449357Z caller=manager.go:86 index-store=tsdb-2025-02-28 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2025-02-27T12:53:41.950494742Z caller=head_manager.go:308 index-store=tsdb-2025-02-28 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2025-02-27T12:53:41.950505211Z caller=manager.go:263 index-store=tsdb-2025-02-28 component=tsdb-manager msg="building WALs" n=0 ts=2025-02-27T12:53:41.950503929Z
level=debug ts=2025-02-27T12:53:41.950515931Z caller=manager.go:287 index-store=tsdb-2025-02-28 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2025-02-27T12:53:41.954787857Z caller=modules.go:680 msg="initializing pattern ingester tee service..."
level=debug ts=2025-02-27T12:53:41.961485318Z caller=modules.go:1094 msg="initializing query frontend" config="{Handler:{LogQueriesLongerThan:0s LogQueryRequestHeaders: MaxBodySize:10485760 QueryStatsEnabled:false} FrontendV1:{MaxOutstandingPerTenant:2048 QuerierForgetDelay:0s} FrontendV2:{SchedulerAddress: DNSLookupPeriod:10s WorkerConcurrency:5 GRPCClientConfig:{MaxRecvMsgSize:104857600 MaxSendMsgSize:104857600 GRPCCompression: RateLimit:0 RateLimitBurst:0 BackoffOnRatelimits:false BackoffConfig:{MinBackoff:100ms MaxBackoff:10s MaxRetries:10} InitialStreamWindowSize:65535 InitialConnectionWindowSize:65535 TLSEnabled:false TLS:{CertPath: KeyPath: CAPath: ServerName: InsecureSkipVerify:false CipherSuites: MinVersion: Reader:<nil>} ConnectTimeout:5s ConnectBackoffBaseDelay:1s ConnectBackoffMaxDelay:5s Middleware:[] StreamMiddleware:[] CustomCompressors:[]} GracefulShutdownTimeout:5m0s InfNames:[eth0 eth1 lo] Addr:127.0.0.1 Port:0 Encoding:protobuf} CompressResponses:false DownstreamURL: TailProxyURL: TLS:{CertPath: KeyPath: CAPath: ServerName: InsecureSkipVerify:false CipherSuites: MinVersion: Reader:<nil>} SupportParquetEncoding:false}"
level=debug ts=2025-02-27T12:53:41.961551731Z caller=modules.go:1123 msg="using query frontend" version=v2
level=debug ts=2025-02-27T12:53:41.962639891Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=false queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=true
level=debug ts=2025-02-27T12:53:41.962696536Z caller=worker_service.go:32 msg="determining if querier is running as standalone target" runningStandalone=false queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=true
level=info ts=2025-02-27T12:53:41.962752932Z caller=worker.go:130 component=querier msg="Starting querier worker using query-scheduler and scheduler ring for addresses"
level=info ts=2025-02-27T12:53:41.964031355Z caller=mapper.go:47 msg="cleaning up mapped rules directory" path=/tmp/loki/rules-temp
level=debug ts=2025-02-27T12:53:41.969340226Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2025-02-27T12:53:41.969371884Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ring-client waiting_for=analytics
level=debug ts=2025-02-27T12:53:41.969384808Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend-tripperware waiting_for=server
level=debug ts=2025-02-27T12:53:41.969399666Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.969411488Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2025-02-27T12:53:41.969423951Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=analytics
level=debug ts=2025-02-27T12:53:41.969435974Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.969437757Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.969456822Z caller=module_service.go:72 msg="module waiting for initialization" module=rule-evaluator waiting_for=store
level=debug ts=2025-02-27T12:53:41.969465999Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.969469816Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler waiting_for=server
level=info ts=2025-02-27T12:53:41.969478452Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=debug ts=2025-02-27T12:53:41.9694984Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.969493961Z caller=module_service.go:72 msg="module waiting for initialization" module=ruler waiting_for=server
level=debug ts=2025-02-27T12:53:41.969515733Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester-tee waiting_for=analytics
level=debug ts=2025-02-27T12:53:41.969522936Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2025-02-27T12:53:41.969527595Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=pattern-ring-client
level=debug ts=2025-02-27T12:53:41.969537193Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler-ring waiting_for=server
level=debug ts=2025-02-27T12:53:41.969523216Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2025-02-27T12:53:41.969692242Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2025-02-27T12:53:41.969909336Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.969906782Z caller=module_service.go:72 msg="module waiting for initialization" module=ruler waiting_for=store
level=info ts=2025-02-27T12:53:41.969931056Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2025-02-27T12:53:41.96993314Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler waiting_for=analytics
level=info ts=2025-02-27T12:53:41.969950092Z caller=module_service.go:82 msg=starting module=query-frontend-tripperware
level=debug ts=2025-02-27T12:53:41.969962345Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester waiting_for=pattern-ingester-tee
level=debug ts=2025-02-27T12:53:41.969973486Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=query-frontend-tripperware
level=debug ts=2025-02-27T12:53:41.969972354Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=debug ts=2025-02-27T12:53:41.970009884Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2025-02-27T12:53:41.970008231Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=query-scheduler
level=info ts=2025-02-27T12:53:41.970016286Z caller=module_service.go:82 msg=starting module=store
level=info ts=2025-02-27T12:53:41.969979497Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2025-02-27T12:53:41.970016707Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler-ring waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.970018039Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2025-02-27T12:53:41.970034128Z caller=module_service.go:72 msg="module waiting for initialization" module=ruler waiting_for=analytics
level=debug ts=2025-02-27T12:53:41.970037535Z caller=module_service.go:72 msg="module waiting for initialization" module=rule-evaluator waiting_for=analytics
level=info ts=2025-02-27T12:53:41.970044428Z caller=module_service.go:82 msg=starting module=query-scheduler-ring
level=debug ts=2025-02-27T12:53:41.970221168Z caller=mock.go:150 msg=Get key=collectors/scheduler wait_index=0
level=debug ts=2025-02-27T12:53:41.970237287Z caller=mock.go:150 msg=Get key=collectors/scheduler wait_index=0
level=debug ts=2025-02-27T12:53:41.970255001Z caller=mock.go:157 msg="Get - not found" key=collectors/scheduler
level=debug ts=2025-02-27T12:53:41.970284386Z caller=mock.go:157 msg="Get - not found" key=collectors/scheduler
level=info ts=2025-02-27T12:53:41.970294525Z caller=basic_lifecycler.go:299 msg="instance not found in the ring" instance=7ae817229632 ring=scheduler
level=info ts=2025-02-27T12:53:41.970311486Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2025-02-27T12:53:41.97032414Z caller=ring.go:316 msg="ring doesn't exist in KV store yet"
level=debug ts=2025-02-27T12:53:41.970400182Z caller=mock.go:150 msg=Get key=collectors/scheduler wait_index=0
level=debug ts=2025-02-27T12:53:41.970439184Z caller=mock.go:150 msg=Get key=collectors/ring wait_index=0
level=debug ts=2025-02-27T12:53:41.970459863Z caller=mock.go:157 msg="Get - not found" key=collectors/scheduler
level=debug ts=2025-02-27T12:53:41.970477195Z caller=client.go:256 msg="value is nil" key=collectors/scheduler index=1
level=debug ts=2025-02-27T12:53:41.970496642Z caller=mock.go:150 msg=Get key=collectors/scheduler wait_index=1
level=debug ts=2025-02-27T12:53:41.970513172Z caller=mock.go:157 msg="Get - not found" key=collectors/ring
level=info ts=2025-02-27T12:53:41.970541956Z caller=ring.go:316 msg="ring doesn't exist in KV store yet"
level=debug ts=2025-02-27T12:53:41.970583834Z caller=mock.go:150 msg=Get key=collectors/ring wait_index=0
level=debug ts=2025-02-27T12:53:41.97061364Z caller=mock.go:157 msg="Get - not found" key=collectors/ring
level=debug ts=2025-02-27T12:53:41.97062464Z caller=client.go:256 msg="value is nil" key=collectors/ring index=1
level=debug ts=2025-02-27T12:53:41.970640139Z caller=mock.go:150 msg=Get key=collectors/ring wait_index=1
level=debug ts=2025-02-27T12:53:41.970663753Z caller=mock.go:123 msg=CAS key=collectors/scheduler modify_index=0 value="\"E\\xf0D\\nC\\n\\f7ae817229632\\x123\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\""
level=debug ts=2025-02-27T12:53:41.970665827Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2025-02-27T12:53:41.970670266Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=info ts=2025-02-27T12:53:41.970679383Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2025-02-27T12:53:41.970686276Z caller=module_service.go:82 msg=starting module=ingester-querier
level=debug ts=2025-02-27T12:53:41.970708536Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2025-02-27T12:53:41.970716471Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2025-02-27T12:53:41.970719406Z caller=mock.go:196 msg=Get key=collectors/scheduler modify_index=2 value="\"E\\xf0D\\nC\\n\\f7ae817229632\\x123\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\""
level=debug ts=2025-02-27T12:53:41.970723975Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2025-02-27T12:53:41.970739765Z caller=ringmanager.go:186 msg="waiting until scheduler is JOINING in the ring"
level=debug ts=2025-02-27T12:53:41.970745626Z caller=mock.go:150 msg=Get key=collectors/usagestats_token wait_index=0
level=debug ts=2025-02-27T12:53:41.970756356Z caller=mock.go:150 msg=Get key=collectors/scheduler wait_index=2
level=debug ts=2025-02-27T12:53:41.970759111Z caller=mock.go:157 msg="Get - not found" key=collectors/usagestats_token
level=info ts=2025-02-27T12:53:41.970765103Z caller=ringmanager.go:190 msg="scheduler is JOINING in the ring"
level=debug ts=2025-02-27T12:53:41.970781854Z caller=mock.go:150 msg=Get key=collectors/scheduler wait_index=0
level=debug ts=2025-02-27T12:53:41.970790589Z caller=mock.go:196 msg=Get key=collectors/scheduler modify_index=2 value="\"E\\xf0D\\nC\\n\\f7ae817229632\\x123\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\""
level=debug ts=2025-02-27T12:53:41.970805878Z caller=mock.go:123 msg=CAS key=collectors/scheduler modify_index=2 value="\"C\\xf0B\\nA\\n\\f7ae817229632\\x121\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\""
level=info ts=2025-02-27T12:53:41.970820525Z caller=ringmanager.go:199 msg="waiting until scheduler is ACTIVE in the ring"
level=debug ts=2025-02-27T12:53:41.970847426Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=info ts=2025-02-27T12:53:41.970859147Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2025-02-27T12:53:41.970909291Z caller=ingester.go:559 component=ingester msg="recovering from checkpoint"
level=info ts=2025-02-27T12:53:41.971088414Z caller=recovery.go:41 component=ingester msg="no checkpoint found, treating as no-op"
level=debug ts=2025-02-27T12:53:41.971119463Z caller=module_service.go:72 msg="module waiting for initialization" module=rule-evaluator waiting_for=ingester-querier
level=debug ts=2025-02-27T12:53:41.971119693Z caller=mock.go:123 msg=CAS key=collectors/usagestats_token modify_index=0 value="\"{\\\"UID\\\":\\\"e8d9ee74-2534-4f33-86b0-b25dc287\""
level=debug ts=2025-02-27T12:53:41.971128941Z caller=module_service.go:72 msg="module waiting for initialization" module=rule-evaluator waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.971137647Z caller=module_service.go:72 msg="module waiting for initialization" module=rule-evaluator waiting_for=ring
level=debug ts=2025-02-27T12:53:41.971145762Z caller=module_service.go:72 msg="module waiting for initialization" module=rule-evaluator waiting_for=server
level=info ts=2025-02-27T12:53:41.971153447Z caller=module_service.go:82 msg=starting module=rule-evaluator
level=debug ts=2025-02-27T12:53:41.971170367Z caller=module_service.go:72 msg="module waiting for initialization" module=ruler waiting_for=ingester-querier
level=debug ts=2025-02-27T12:53:41.971161863Z caller=mock.go:196 msg=Get key=collectors/scheduler modify_index=3 value="\"C\\xf0B\\nA\\n\\f7ae817229632\\x121\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\""
level=debug ts=2025-02-27T12:53:41.971178773Z caller=module_service.go:72 msg="module waiting for initialization" module=ruler waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.971186958Z caller=module_service.go:72 msg="module waiting for initialization" module=ruler waiting_for=ring
level=debug ts=2025-02-27T12:53:41.971194152Z caller=module_service.go:72 msg="module waiting for initialization" module=ruler waiting_for=rule-evaluator
level=info ts=2025-02-27T12:53:41.971201135Z caller=module_service.go:82 msg=starting module=ruler
level=debug ts=2025-02-27T12:53:41.971208108Z caller=mock.go:150 msg=Get key=collectors/scheduler wait_index=3
level=info ts=2025-02-27T12:53:41.971222135Z caller=ruler.go:533 msg="ruler up and running"
level=debug ts=2025-02-27T12:53:41.971234047Z caller=ruler.go:571 msg="syncing rules" reason=initial
level=debug ts=2025-02-27T12:53:41.971272979Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.971296182Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler waiting_for=query-scheduler-ring
level=debug ts=2025-02-27T12:53:41.971305029Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.971311421Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ring-client waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.971318824Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=ring
level=debug ts=2025-02-27T12:53:41.971323914Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ring-client waiting_for=ring
level=debug ts=2025-02-27T12:53:41.971330857Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2025-02-27T12:53:41.971335536Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ring-client waiting_for=server
level=info ts=2025-02-27T12:53:41.97134307Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2025-02-27T12:53:41.971346647Z caller=module_service.go:82 msg=starting module=pattern-ring-client
level=debug ts=2025-02-27T12:53:41.9714023Z caller=mock.go:150 msg=Get key=collectors/pattern-ring wait_index=0
level=debug ts=2025-02-27T12:53:41.971414303Z caller=mock.go:157 msg="Get - not found" key=collectors/pattern-ring
level=debug ts=2025-02-27T12:53:41.97141285Z caller=mock.go:150 msg=Get key=collectors/compactor wait_index=0
level=debug ts=2025-02-27T12:53:41.971430713Z caller=mock.go:157 msg="Get - not found" key=collectors/compactor
level=info ts=2025-02-27T12:53:41.97145043Z caller=ring.go:316 component=pattern-ring-client msg="ring doesn't exist in KV store yet"
level=info ts=2025-02-27T12:53:41.971458615Z caller=ring.go:316 msg="ring doesn't exist in KV store yet"
level=info ts=2025-02-27T12:53:41.971454287Z caller=ingester.go:575 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=545.338µs errors=false
level=info ts=2025-02-27T12:53:41.971489923Z caller=ingester.go:581 component=ingester msg="recovering from WAL"
level=debug ts=2025-02-27T12:53:41.971499852Z caller=mock.go:150 msg=Get key=collectors/pattern-ring wait_index=0
level=debug ts=2025-02-27T12:53:41.971539116Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.971537312Z caller=mock.go:157 msg="Get - not found" key=collectors/pattern-ring
level=debug ts=2025-02-27T12:53:41.971549144Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=pattern-ingester-tee
level=debug ts=2025-02-27T12:53:41.971531131Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester-tee waiting_for=memberlist-kv
level=debug ts=2025-02-27T12:53:41.971551979Z caller=client.go:256 msg="value is nil" key=collectors/pattern-ring index=4
level=debug ts=2025-02-27T12:53:41.971564873Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester-tee waiting_for=pattern-ring-client
level=debug ts=2025-02-27T12:53:41.971571987Z caller=mock.go:150 msg=Get key=collectors/compactor wait_index=0
level=debug ts=2025-02-27T12:53:41.971570303Z caller=mock.go:150 msg=Get key=collectors/pattern-ring wait_index=4
level=debug ts=2025-02-27T12:53:41.971580823Z caller=mock.go:157 msg="Get - not found" key=collectors/compactor
level=info ts=2025-02-27T12:53:41.971590061Z caller=basic_lifecycler.go:299 msg="instance not found in the ring" instance=7ae817229632 ring=compactor
level=debug ts=2025-02-27T12:53:41.971591583Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester-tee waiting_for=ring
level=info ts=2025-02-27T12:53:41.971597855Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=debug ts=2025-02-27T12:53:41.971598486Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=query-scheduler
level=debug ts=2025-02-27T12:53:41.971604588Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester-tee waiting_for=server
level=info ts=2025-02-27T12:53:41.971615818Z caller=module_service.go:82 msg=starting module=pattern-ingester-tee
level=debug ts=2025-02-27T12:53:41.971637259Z caller=mock.go:123 msg=CAS key=collectors/compactor modify_index=0 value="\"D\\xc0\\nB\\n\\f7ae817229632\\x122\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\\x81\""
level=debug ts=2025-02-27T12:53:41.971668607Z caller=mock.go:192 msg="Get - not found" key=collectors/pattern-ring
level=debug ts=2025-02-27T12:53:41.971676471Z caller=client.go:256 msg="value is nil" key=collectors/pattern-ring index=5
level=debug ts=2025-02-27T12:53:41.971687752Z caller=mock.go:150 msg=Get key=collectors/pattern-ring wait_index=5
level=info ts=2025-02-27T12:53:41.971695807Z caller=compactor.go:414 msg="waiting until compactor is JOINING in the ring"
level=debug ts=2025-02-27T12:53:41.971499291Z caller=mock.go:150 msg=Get key=collectors/compactor wait_index=0
level=debug ts=2025-02-27T12:53:41.9717775Z caller=mock.go:196 msg=Get key=collectors/compactor modify_index=5 value="\"D\\xc0\\nB\\n\\f7ae817229632\\x122\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\\x81\""
level=debug ts=2025-02-27T12:53:41.97183682Z caller=mock.go:150 msg=Get key=collectors/compactor wait_index=5
level=debug ts=2025-02-27T12:53:41.971925907Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=pattern-ring-client
level=debug ts=2025-02-27T12:53:41.971942006Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=ring
level=debug ts=2025-02-27T12:53:41.971931126Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester waiting_for=pattern-ring-client
level=debug ts=2025-02-27T12:53:41.971953117Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=server
level=info ts=2025-02-27T12:53:41.971964318Z caller=module_service.go:82 msg=starting module=distributor
level=debug ts=2025-02-27T12:53:41.971964398Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester waiting_for=ring
level=debug ts=2025-02-27T12:53:41.972055128Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester waiting_for=server
level=debug ts=2025-02-27T12:53:41.972069535Z caller=module_service.go:72 msg="module waiting for initialization" module=pattern-ingester waiting_for=analytics
level=info ts=2025-02-27T12:53:41.972083381Z caller=module_service.go:82 msg=starting module=pattern-ingester
level=error ts=2025-02-27T12:53:41.972080946Z caller=ratestore.go:109 msg="error getting ingester clients" err="empty ring"
level=debug ts=2025-02-27T12:53:41.972095313Z caller=mock.go:150 msg=Get key=collectors/distributor wait_index=0
level=debug ts=2025-02-27T12:53:41.972113397Z caller=mock.go:157 msg="Get - not found" key=collectors/distributor
level=debug ts=2025-02-27T12:53:41.972125188Z caller=mock.go:150 msg=Get key=collectors/distributor wait_index=0
level=debug ts=2025-02-27T12:53:41.972135888Z caller=mock.go:157 msg="Get - not found" key=collectors/distributor
level=info ts=2025-02-27T12:53:41.972137691Z caller=ring.go:316 component=distributor msg="ring doesn't exist in KV store yet"
level=info ts=2025-02-27T12:53:41.972143252Z caller=basic_lifecycler.go:299 component=distributor msg="instance not found in the ring" instance=7ae817229632 ring=distributor
level=debug ts=2025-02-27T12:53:41.972169902Z caller=mock.go:123 msg=CAS key=collectors/distributor modify_index=0 value="\"@\\xb0\\n>\\n\\f7ae817229632\\x12.\\n\\v127.0.0.1:0\\x10\\xd5\\xc0\\x81\\xbe\\x062\""
level=info ts=2025-02-27T12:53:41.972167938Z caller=lifecycler.go:675 component=pattern-ingester msg="not loading tokens from file, tokens file path is empty"
level=debug ts=2025-02-27T12:53:41.97216884Z caller=mock.go:150 msg=Get key=collectors/distributor wait_index=0
level=debug ts=2025-02-27T12:53:41.97220102Z caller=mock.go:196 msg=Get key=collectors/distributor modify_index=6 value="\"@\\xb0\\n>\\n\\f7ae817229632\\x12.\\n\\v127.0.0.1:0\\x10\\xd5\\xc0\\x81\\xbe\\x062\""
level=debug ts=2025-02-27T12:53:41.972232598Z caller=mock.go:150 msg=Get key=collectors/distributor wait_index=6
level=info ts=2025-02-27T12:53:41.972370396Z caller=ingester.go:282 component=pattern-ingester msg="sleeping for initial delay before starting periodic flushing" delay=41.587090446s
level=debug ts=2025-02-27T12:53:41.972386295Z caller=mock.go:150 msg=Get key=collectors/pattern-ring wait_index=0
level=debug ts=2025-02-27T12:53:41.972414868Z caller=mock.go:157 msg="Get - not found" key=collectors/pattern-ring
level=info ts=2025-02-27T12:53:41.972431719Z caller=lifecycler.go:702 component=pattern-ingester msg="instance not found in ring, adding with no tokens" ring=pattern-ingester
level=debug ts=2025-02-27T12:53:41.972517519Z caller=mock.go:123 msg=CAS key=collectors/pattern-ring modify_index=0 value="\">\\xa8\\n<\\n\\f7ae817229632\\x12,\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\\x81\""
level=debug ts=2025-02-27T12:53:41.972563055Z caller=mock.go:196 msg=Get key=collectors/pattern-ring modify_index=7 value="\">\\xa8\\n<\\n\\f7ae817229632\\x12,\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\\x81\""
level=debug ts=2025-02-27T12:53:41.972590296Z caller=lifecycler.go:540 component=pattern-ingester msg="JoinAfter expired" ring=pattern-ingester
level=info ts=2025-02-27T12:53:41.972606886Z caller=lifecycler.go:544 component=pattern-ingester msg="auto-joining cluster after timeout" ring=pattern-ingester
level=debug ts=2025-02-27T12:53:41.972627795Z caller=mock.go:150 msg=Get key=collectors/pattern-ring wait_index=7
level=debug ts=2025-02-27T12:53:41.97263039Z caller=mock.go:150 msg=Get key=collectors/pattern-ring wait_index=0
level=debug ts=2025-02-27T12:53:41.972648374Z caller=mock.go:196 msg=Get key=collectors/pattern-ring modify_index=7 value="\">\\xa8\\n<\\n\\f7ae817229632\\x12,\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\\x81\""
level=debug ts=2025-02-27T12:53:41.972790139Z caller=mock.go:123 msg=CAS key=collectors/pattern-ring modify_index=7 value="\"\\xbd\\x05\\xf4\\xbc\\x02\\n\\xba\\x05\\n\\f7ae817229632\\x12\\xa9\\x05\\n\\x0e127.0.0.1:909\""
level=debug ts=2025-02-27T12:53:41.972833038Z caller=mock.go:196 msg=Get key=collectors/pattern-ring modify_index=8 value="\"\\xbd\\x05\\xf4\\xbc\\x02\\n\\xba\\x05\\n\\f7ae817229632\\x12\\xa9\\x05\\n\\x0e127.0.0.1:909\""
level=info ts=2025-02-27T12:53:41.972893651Z caller=ingester.go:597 component=ingester msg="WAL segment recovery finished" elapsed=1.986685ms errors=false
level=info ts=2025-02-27T12:53:41.972917806Z caller=ingester.go:545 component=ingester msg="closing recoverer"
level=info ts=2025-02-27T12:53:41.972930349Z caller=ingester.go:553 component=ingester msg="WAL recovery finished" time=2.022803ms
level=debug ts=2025-02-27T12:53:41.97295198Z caller=mock.go:150 msg=Get key=collectors/pattern-ring wait_index=8
level=info ts=2025-02-27T12:53:41.973035906Z caller=lifecycler.go:675 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2025-02-27T12:53:41.973044753Z caller=wal.go:157 msg=started component=wal
level=debug ts=2025-02-27T12:53:41.973076272Z caller=mock.go:150 msg=Get key=collectors/ring wait_index=0
level=debug ts=2025-02-27T12:53:41.973089617Z caller=mock.go:157 msg="Get - not found" key=collectors/ring
level=info ts=2025-02-27T12:53:41.973100577Z caller=lifecycler.go:702 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=debug ts=2025-02-27T12:53:41.973147124Z caller=mock.go:123 msg=CAS key=collectors/ring modify_index=0 value="\">\\xa8\\n<\\n\\f7ae817229632\\x12,\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\\x81\""
level=info ts=2025-02-27T12:53:41.973145962Z caller=ingester.go:766 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=11.140718057s
level=debug ts=2025-02-27T12:53:41.973190024Z caller=lifecycler.go:540 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2025-02-27T12:53:41.973202316Z caller=lifecycler.go:544 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2025-02-27T12:53:41.97321984Z caller=mock.go:150 msg=Get key=collectors/ring wait_index=0
level=debug ts=2025-02-27T12:53:41.973231291Z caller=mock.go:196 msg=Get key=collectors/ring modify_index=9 value="\">\\xa8\\n<\\n\\f7ae817229632\\x12,\\n\\x0e127.0.0.1:9096\\x10\\xd5\\xc0\\x81\""
BTW, this error do not cause any interruption to the loki service. Just like a warning!
