loki:
storage:
bucketNames:
chunks: "Bucket"
ruler: "Bucket"
admin: "Bucket"
type: s3
s3:
endpoint: http://s3.ap-south-1.amazonaws.com
region: ap-south-1
secretAccessKey: " "
accessKeyId: " "
s3ForcePathStyle: false
insecure: false
auth_enabled: false
replication_factor: 1
ring:
kvstore:
store: memberlist
memberlist:
join_members:
- loki-read.loki.svc.cluster.local:7946
- loki-write.loki.svc.cluster.local:7946
logs of read pod
kubectl logs loki-read-7dfcdf7d7d-qjfn9 -n loki
level=info ts=2024-05-03T05:00:21.913266169Z caller=main.go:120 msg=“Starting Loki” version=“(version=3.0.0, branch=HEAD, revision=b4f7181c7a)”
level=info ts=2024-05-03T05:00:21.916854977Z caller=server.go:354 msg=“server listening on addresses” http=[::]:3100 grpc=[::]:9095
level=error ts=2024-05-03T05:00:21.924537234Z caller=resolver.go:100 msg=“failed to lookup SRV records” host=_memcached-client._tcp.loki-results-cache.loki.svc err=“lookup _memcached-client._tcp.loki-results-cache.loki.svc on 172.20.0.10:53: no such host”
level=error ts=2024-05-03T05:00:21.9414662Z caller=resolver.go:100 msg=“failed to lookup SRV records” host=_memcached-client._tcp.loki-results-cache.loki.svc err=“lookup _memcached-client._tcp.loki-results-cache.loki.svc on 172.20.0.10:53: no such host”
level=error ts=2024-05-03T05:00:21.95188291Z caller=resolver.go:100 msg=“failed to lookup SRV records” host=_memcached-client._tcp.loki-results-cache.loki.svc err=“lookup _memcached-client._tcp.loki-results-cache.loki.svc on 172.20.0.10:53: no such host”
level=error ts=2024-05-03T05:00:21.958085664Z caller=resolver.go:100 msg=“failed to lookup SRV records” host=_memcached-client._tcp.loki-results-cache.loki.svc err=“lookup _memcached-client._tcp.loki-results-cache.loki.svc on 172.20.0.10:53: no such host”
level=error ts=2024-05-03T05:00:21.967755317Z caller=resolver.go:100 msg=“failed to lookup SRV records” host=_memcached-client._tcp.loki-results-cache.loki.svc err=“lookup _memcached-client._tcp.loki-results-cache.loki.svc on 172.20.0.10:53: no such host”
level=info ts=2024-05-03T05:00:21.995597704Z caller=memberlist_client.go:435 msg=“Using memberlist cluster label and node name” cluster_label= node=loki-read-7dfcdf7d7d-qjfn9-b9d77169
level=info ts=2024-05-03T05:00:21.998882535Z caller=memberlist_client.go:541 msg=“memberlist fast-join starting” nodes_found=1 to_join=4
level=error ts=2024-05-03T05:00:22.005464684Z caller=resolver.go:100 msg=“failed to lookup SRV records” host=_memcached-client._tcp.loki-chunks-cache.loki.svc err=“lookup _memcached-client._tcp.loki-chunks-cache.loki.svc on 172.20.0.10:53: no such host”
level=error ts=2024-05-03T05:00:22.020526685Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:22.020566731Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=info ts=2024-05-03T05:00:22.022042908Z caller=worker.go:113 component=querier msg=“Starting querier worker using query-scheduler and scheduler ring for addresses”
level=info ts=2024-05-03T05:00:22.025506537Z caller=module_service.go:82 msg=starting module=cache-generation-loader
level=info ts=2024-05-03T05:00:22.025540593Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2024-05-03T05:00:22.025699367Z caller=module_service.go:82 msg=starting module=server
level=info ts=2024-05-03T05:00:22.025915747Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2024-05-03T05:00:22.026127402Z caller=module_service.go:82 msg=starting module=query-frontend-tripperware
level=info ts=2024-05-03T05:00:22.026142391Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2024-05-03T05:00:22.02617229Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2024-05-03T05:00:22.02621226Z caller=module_service.go:82 msg=starting module=query-scheduler-ring
ts=2024-05-03T05:00:22.042624682Z caller=memberlist_logger.go:74 level=warn msg=“Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:22.042659192Z caller=memberlist_client.go:561 msg=“memberlist fast-join finished” joined_nodes=0 elapsed_time=43.78115ms
level=info ts=2024-05-03T05:00:22.04268047Z caller=memberlist_client.go:573 phase=startup msg=“joining memberlist cluster” join_members=loki-memberlist
level=info ts=2024-05-03T05:00:22.042704961Z caller=ring.go:276 msg=“ring doesn’t exist in KV store yet”
level=info ts=2024-05-03T05:00:22.042775903Z caller=module_service.go:82 msg=starting module=query-frontend
level=info ts=2024-05-03T05:00:22.042890248Z caller=ring.go:276 msg=“ring doesn’t exist in KV store yet”
level=info ts=2024-05-03T05:00:22.043008709Z caller=module_service.go:82 msg=starting module=ingester-querier
level=info ts=2024-05-03T05:00:22.043050901Z caller=module_service.go:82 msg=starting module=store
level=info ts=2024-05-03T05:00:22.043076133Z caller=module_service.go:82 msg=starting module=querier
level=info ts=2024-05-03T05:00:22.043184378Z caller=loki.go:503 msg=“Loki started” startup_time=177.269775ms
ts=2024-05-03T05:00:22.048666383Z caller=memberlist_logger.go:74 level=warn msg=“Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:22.04869854Z caller=memberlist_client.go:607 phase=startup msg=“joining memberlist cluster” attempts=1 max_attempts=10 err=“1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host\n\n”
ts=2024-05-03T05:00:23.91310312Z caller=memberlist_logger.go:74 level=warn msg=“Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:23.913211397Z caller=memberlist_client.go:607 phase=startup msg=“joining memberlist cluster” attempts=2 max_attempts=10 err=“1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host\n\n”
level=error ts=2024-05-03T05:00:25.043529168Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:27.021049825Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:27.021084895Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
ts=2024-05-03T05:00:27.691069198Z caller=memberlist_logger.go:74 level=warn msg=“Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:27.691113646Z caller=memberlist_client.go:607 phase=startup msg=“joining memberlist cluster” attempts=3 max_attempts=10 err=“1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host\n\n”
level=error ts=2024-05-03T05:00:28.043224932Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:31.043400698Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
ts=2024-05-03T05:00:31.919875719Z caller=memberlist_logger.go:74 level=warn msg=“Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:31.919980194Z caller=memberlist_client.go:607 phase=startup msg=“joining memberlist cluster” attempts=4 max_attempts=10 err=“1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host\n\n”
level=error ts=2024-05-03T05:00:32.032834433Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:32.032883901Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:00:32.04335263Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:34.044047401Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:37.029791873Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:37.029866599Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:00:37.044042014Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:40.043462634Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:42.020616931Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:42.02066837Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:00:42.043295521Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:43.043847986Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
ts=2024-05-03T05:00:45.194977608Z caller=memberlist_logger.go:74 level=warn msg=“Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:45.195010672Z caller=memberlist_client.go:607 phase=startup msg=“joining memberlist cluster” attempts=5 max_attempts=10 err=“1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host\n\n”
level=error ts=2024-05-03T05:00:46.043960338Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:47.020127767Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:47.020159723Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:00:49.043148318Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:00:51.41707594Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:00:52.020005308Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:52.020080472Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:00:52.043526028Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:52.043526016Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:55.045255949Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:00:57.020410013Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:00:57.020545725Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:00:58.043947081Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:01.043679765Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:01:01.417796777Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:01:02.041341908Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:02.041372616Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:02.043450342Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:04.043852505Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:07.021042307Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:07.021073547Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:07.04354805Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:10.044200932Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:01:11.416615553Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:01:12.020785703Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:12.020815475Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:12.043029813Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:13.044188682Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:16.049128103Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
ts=2024-05-03T05:01:16.38990575Z caller=memberlist_logger.go:74 level=warn msg=“Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:16.389939679Z caller=memberlist_client.go:607 phase=startup msg=“joining memberlist cluster” attempts=6 max_attempts=10 err=“1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host\n\n”
level=error ts=2024-05-03T05:01:17.020264104Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:17.0203072Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:19.044261104Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:01:21.416832134Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:01:22.018568451Z caller=resolver.go:100 msg=“failed to lookup SRV records” host=_memcached-client._tcp.loki-chunks-cache.loki.svc err=“lookup _memcached-client._tcp.loki-chunks-cache.loki.svc on 172.20.0.10:53: no such host”
level=error ts=2024-05-03T05:01:22.019599887Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:22.019662245Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:22.043343302Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:22.043377806Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:25.044142303Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:27.020303754Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:27.020334865Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:28.04382703Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:31.043174121Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:01:31.416622052Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:01:32.021249418Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:32.021279317Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:32.043623344Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:34.043205258Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:37.020947965Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:37.021053303Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:37.044069525Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:01:39.848429529Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:01:40.054183592Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:01:41.417075227Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:01:42.020541889Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:42.020571181Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:42.043060619Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:43.043764084Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:46.043374925Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:47.024430274Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:47.02450942Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:49.043501413Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:01:51.417273743Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:01:52.020779429Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:52.020810869Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:52.043112446Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:52.043225774Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:55.043478455Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:01:57.021070437Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:01:57.021100309Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:01:58.043547292Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
ts=2024-05-03T05:02:00.521200447Z caller=memberlist_logger.go:74 level=warn msg=“Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:02:00.521227785Z caller=memberlist_client.go:607 phase=startup msg=“joining memberlist cluster” attempts=7 max_attempts=10 err=“1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 172.20.0.10:53: no such host\n\n”
level=error ts=2024-05-03T05:02:01.044157222Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:02:01.417310151Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:02:02.021237281Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:02:02.021278725Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:02:02.043612437Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:02:04.043204078Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:02:07.021084235Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:02:07.021114948Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:02:07.043172984Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:02:10.043463982Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:02:11.424244737Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:02:12.020800015Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:02:12.02084635Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:02:12.043085634Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:02:13.043285362Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:02:16.043142076Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:02:17.023823783Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:02:17.023860975Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:02:19.043742941Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=info ts=2024-05-03T05:02:21.416675249Z caller=frontend.go:445 msg=“not ready: number of schedulers this worker is connected to is 0”
level=error ts=2024-05-03T05:02:22.065350778Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:02:22.073315084Z caller=ring_watcher.go:56 component=frontend-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:02:22.084160824Z caller=resolver.go:100 msg=“failed to lookup SRV records” host=_memcached-client._tcp.loki-chunks-cache.loki.svc err=“lookup _memcached-client._tcp.loki-chunks-cache.loki.svc on 172.20.0.10:53: no such host”
level=error ts=2024-05-03T05:02:22.107324972Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:02:22.10738352Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:02:25.044040937Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
level=error ts=2024-05-03T05:02:27.020559493Z caller=resolver.go:87 index-store=tsdb-2024-04-01 msg=“failed to lookup IP addresses” host=loki-backend-headless.loki.svc.cluster.local err=“lookup loki-backend-headless.loki.svc.cluster.local on 172.20.0.10:53: no such host”
level=warn ts=2024-05-03T05:02:27.020752424Z caller=resolver.go:134 index-store=tsdb-2024-04-01 msg=“IP address lookup yielded no results. No host found or no addresses found” host=loki-backend-headless.loki.svc.cluster.local
level=error ts=2024-05-03T05:02:28.043328018Z caller=ring_watcher.go:56 component=querier component=querier-scheduler-worker msg=“error getting addresses from ring” err=“empty ring”
loki Read is in Running with 0/1
Normal NotTriggerScaleUp 3m35s (x31 over 8m37s) cluster-autoscaler pod didn’t trigger scale-up: 1 max node group size reached
Warning FailedScheduling 3m29s (x2 over 8m40s) default-scheduler 0/7 nodes are available: pod has unbound immediate PersistentVolumeClaims. preemption: 0/7 nodes are available: 7 Preemption is not helpful for scheduling…