I am trying to deploy this grafana / loki stack :
Docker-compose.yml :
---
version: "3.7"
x-loki: &defaultloki
image: grafana/loki:main
volumes:
- ./monitor/loki/:/opt/loki/
networks:
- loki
services:
grafana:
container_name: grafana
image: grafana/grafana:latest
restart: unless-stopped
user: "${UID:-1001}:${GID:-1001}"
volumes:
- ./monitor/grafana/grafana.ini:/etc/grafana/grafana.ini
- grafana_data:/var/lib/grafana
networks:
- loki
ports:
- 3000:3000
environment:
- GF_SMTP_ENABLED=true
- GF_SMTP_HOST=mail.jurihub.fr:587
- GF_SMTP_USER=grafana@jurihub.fr
- GF_SMTP_PASSWORD=${GRAFANA_SMTP_PASSWORD}
- GF_SMTP_FROM_ADDRESS=grafana@jurihub.fr
- GF_DATABASE_TYPE=mysql
- GF_DATABASE_HOST=grafana_mysql:3306
- GF_DATABASE_NAME=grafana
- GF_DATABASE_USER=grafana
- GF_DATABASE_PASSWORD=${GRAFANA_MYSQL_PASSWORD}
depends_on:
- grafana_mysql
grafana_mysql:
container_name: grafana_mysql
restart: unless-stopped
image: mysql:8.0
user: "${UID:-1001}:${GID:-1001}"
volumes:
- grafana_mysql:/var/lib/mysql
networks:
- loki
environment:
MYSQL_ROOT_PASSWORD: ${GRAFANA_MYSQL_ROOT_PASSWORD}
MYSQL_DATABASE: grafana
MYSQL_USER: grafana
MYSQL_PASSWORD: ${GRAFANA_MYSQL_PASSWORD}
# Read-path nodes.
loki-read1:
<<: *defaultloki
command: "-config.expand-env=true -config.file=/opt/loki/loki.yml -target=read -querier.frontend-address=loki-frontend"
ports:
- 3100:3100
loki-read2:
<<: *defaultloki
command: "-config.expand-env=true -config.file=/opt/loki/loki.yml -target=read -querier.frontend-address=loki-frontend"
ports:
- 3101:3100
loki-read3:
<<: *defaultloki
command: "-config.expand-env=true -config.file=/opt/loki/loki.yml -target=read -querier.frontend-address=loki-frontend"
ports:
- 3102:3100
# Write-path nodes.
loki-write1:
<<: *defaultloki
command: "-config.expand-env=true -config.file=/opt/loki/loki.yml -target=write -ingester.lifecycler.ID=write1"
ports:
- 3200:3100
loki-write2:
<<: *defaultloki
command: "-config.expand-env=true -config.file=/opt/loki/loki.yml -target=write -ingester.lifecycler.ID=write2"
ports:
- 3201:3100
loki-write3:
<<: *defaultloki
command: "-config.expand-env=true -config.file=/opt/loki/loki.yml -target=write -ingester.lifecycler.ID=write3"
ports:
- 3202:3100
loki-frontend:
<<: *defaultloki
command: "-config.expand-env=true -config.file=/opt/loki/loki.yml -target=query-frontend"
ports:
- 3301:3100
# Network
networks:
loki:
volumes:
grafana_mysql:
grafana_data:
./monitor/loki/loki.yml :
auth_enabled: false
server:
log_level: debug
http_listen_port: 3100
http_server_read_timeout: 300s # allow longer time span queries
http_server_write_timeout: 300s # allow longer time span queries
memberlist:
join_members:
- loki-write1
- loki-write2
- loki-write3
ingester:
wal:
enabled: false
lifecycler:
unregister_on_shutdown: false
join_after: 60s
observe_period: 5s
final_sleep: 0s
schema_config:
configs:
- from: 2020-08-01
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
ruler:
storage:
type: local
local:
directory: /tmp/loki/rules
limits_config:
unordered_writes: true
query_range:
cache_results: false
storage_config:
boltdb_shipper:
cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
shared_store: filesystem
common:
path_prefix: /tmp/loki
ring:
kvstore:
store: memberlist
compactor_address: 0.0.0.0
Containers seem to connect with each other, but my query-frontend cannot read the cluster config, and therefore cannot communicate with readers (as far as I understand). There isn’t any line about this in loki documentation, and I already passed a lot of time on this…
The writers logs are :
...
ts=2023-04-13T16:15:11.9263426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 192.168.32.8:7946"
ts=2023-04-13T16:15:11.9264365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.32.8:39622"
ts=2023-04-13T16:15:11.9281191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.32.5:56550"
ts=2023-04-13T16:15:11.9281737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.32.3:43104"
ts=2023-04-13T16:15:11.9282043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 192.168.32.4:7946"
ts=2023-04-13T16:15:11.9302569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.32.3:43138"
...
level=info ts=2023-04-13T16:15:12.6244388Z caller=memberlist_client.go:595 msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=10.2433ms
ts=2023-04-13T16:15:47.3962164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: d3b28b760ea5-80264658 192.168.32.5:7946"
level=debug ts=2023-04-13T16:15:52.6204249Z caller=reporter.go:202 msg="failed to read cluster seed file" err="open loki_cluster_seed.json: no such file or directory"
ts=2023-04-13T16:16:08.2724812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.32.9:32942"
ts=2023-04-13T16:16:09.7169489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.32.3:44154"
The query-frontend logs are (readers logs are similar) :
...
level=info ts=2023-04-13T16:15:10.2026513Z caller=module_service.go:82 msg=initialising module=query-frontend
level=info ts=2023-04-13T16:15:10.2027464Z caller=loki.go:502 msg="Loki started"
level=debug ts=2023-04-13T16:15:11.8768536Z caller=reporter.go:202 msg="failed to read cluster seed file" err="open loki_cluster_seed.json: no such file or directory"
I tried to launch containers as root on my local computer but it wasn’t working neither.
Any help appreciated,
thanks !