Hi,
We’re using docker-compose to stand up our Loki stack using Simple Scalable Deployment (not K8s). We currently have our Ruler storage set up to use local filesystem because it just works, however, we’d prefer to use our on-prem s3 storage. I’ve noticed by browsing the community forums here that rules need to be put into s3 via Loki http API, so while exploring the API, we noticed that although we can get some endpoints to work fine, like /loki/api/v1/query_range and /metrics, /ready, /memberlist, /ring, etc… we cannot get /loki/api/v1/rules to list the current rules we have in local storage, nor can we use that endpoint to add rules. Instead, that endpoint returns a 404.
We can switch our Ruler storage to s3, but can’t get rules into it unless we can get the API working.
Below is our docker-compose and loki configs. Any help on getting the endpoint working would be great.
---
auth_enabled: false
server:
http_listen_port: 3100
http_server_read_timeout: 10m
http_server_write_timeout: 10m
http_server_idle_timeout: 10m
grpc_listen_port: 9096
grpc_server_max_recv_msg_size: 10485760
grpc_server_max_send_msg_size: 10485760
memberlist:
join_members: #["read", "write", "backend"]
- loki:7946
#bind_addr: ['0.0.0.0']
bind_port: 7946
gossip_interval: 2s
limits_config:
allow_structured_metadata: false
ingestion_rate_mb: 15
ingestion_burst_size_mb: 30
per_stream_rate_limit: 10MB
per_stream_rate_limit_burst: 20MB
split_queries_by_interval: 10m
ingester_client:
remote_timeout: 10s
query_range:
cache_results: true
parallelise_shardable_queries: false
schema_config:
configs:
- from: 2021-08-01
store: boltdb-shipper
object_store: s3
schema: v11
index:
prefix: index_
period: 24h
- from : 2025-02-13
store: tsdb
object_store: s3
schema: v13
index:
prefix: index_
period: 24h
common:
path_prefix: /loki
replication_factor: 1
compactor_address: http://backend:3100
storage:
s3:
endpoint: https://xxxx.xx.xx
insecure: true
bucketnames: se-loki
access_key_id: XXXXXXX
secret_access_key: XXXXX
region: XXXX
s3forcepathstyle: true
ring:
kvstore:
store: memberlist
ruler:
storage:
#type: s3
type: local
s3:
bucketnames: se-loki-ruler
local:
directory: /etc/loki/rules
evaluation_interval: 1m
alertmanager_url: http://XXXXX.XXX:9093
enable_api: true
ring:
kvstore:
store: memberlist
---
networks:
loki:
services:
read:
image: xx.xx.xx.xx/se-loki-read:3.3.2
deploy:
mode: replicated
replicas: 3
endpoint_mode: dnsrr
command: "-config.file=/etc/loki/config.yaml -target=read"
ports:
- 3101-3103:3100
- 7946
- 9095
volumes:
- ./config/config.yaml:/etc/loki/config.yaml
healthcheck:
test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1" ]
interval: 10s
timeout: 5s
retries: 5
networks: &loki-dns
loki:
aliases:
- loki
logging:
driver: loki
options:
loki-url: "http://xxx.xx.xxx:3100/loki/api/v1/push"
write:
image: xxxxxx/xxx/se-loki-write:3.3.2
deploy:
mode: replicated
replicas: 3
endpoint_mode: dnsrr
command: "-config.file=/etc/loki/config.yaml -target=write"
ports:
- 3104-3106:3100
- 7946
- 9095
volumes:
- ./config/config.yaml:/etc/loki/config.yaml
healthcheck:
test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1" ]
interval: 10s
timeout: 5s
retries: 5
networks:
<<: *loki-dns
logging:
driver: loki
options:
loki-url: "http://xxxx.xx.xxx:3100/loki/api/v1/push"
grafana:
image: xxxxx/xxxxx/se-loki-grafana:latest
environment:
- GF_PATHS_PROVISIONING=/etc/grafana/provisioning
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
depends_on:
- gateway
entrypoint:
- sh
- -euc
- |
mkdir -p /etc/grafana/provisioning/datasources
cat <<EOF > /etc/grafana/provisioning/datasources/ds.yaml
apiVersion: 1
datasources:
- name: Loki
type: loki
access: proxy
url: http://gateway:3100
jsonData:
httpHeaderName1: "X-Scope-OrgID"
manageAlerts: false
secureJsonData:
httpHeaderValue1: "tenant1"
EOF
/run.sh
ports:
- "3000:3000"
healthcheck:
test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1" ]
interval: 10s
timeout: 5s
retries: 5
networks:
- loki
logging:
driver: loki
options:
loki-url: "xxxxx.xx.xxx:3100/loki/api/v1/push"
backend:
image: xxxx.xxxxx.xx/se-loki-backend:3.3.2
volumes:
- ./config/config.yaml:/etc/loki/config.yaml
- ./rules/rules_puppet.yaml:/etc/loki/rules/fake/rules_puppet.yaml
ports:
- "3100"
- "7946"
command: "-config.file=/etc/loki/config.yaml -target=backend -legacy-read-mode=false"
depends_on:
- gateway
networks:
- loki
logging:
driver: loki
options:
loki-url: "http://xxxx.xx.xxx:3100/loki/api/v1/push"
gateway:
image: xxx.xx.xxx/se-loki-nginx:latest
depends_on:
- read
- write
entrypoint:
- sh
- -euc
- |
cat <<EOF > /etc/nginx/nginx.conf
user nginx;
worker_processes 5; ## Default: 1
events {
worker_connections 1000;
}
http {
resolver 127.0.0.11;
server {
listen 3100;
location = / {
return 200 'OK';
#auth_basic off;
}
location = /api/prom/push {
proxy_pass http://write:3100\$$request_uri;
}
location = /api/prom/tail {
proxy_pass http://read:3100\$$request_uri;
proxy_set_header Upgrade \$$http_upgrade;
proxy_set_header Connection "upgrade";
}
location ~ /api/prom/.* {
proxy_pass http://read:3100\$$request_uri;
}
location = /loki/api/v1/push {
proxy_pass http://write:3100\$$request_uri;
}
location = /loki/api/v1/tail {
proxy_pass http://read:3100\$$request_uri;
proxy_set_header Upgrade \$$http_upgrade;
proxy_set_header Connection "upgrade";
}
location ~ /loki/api/.* {
proxy_pass http://read:3100\$$request_uri;
}
}
}
EOF
/docker-entrypoint.sh nginx -g "daemon off;"
ports:
- "3100:3100"
healthcheck:
test: ["CMD", "service", "nginx", "status"]
interval: 10s
timeout: 5s
retries: 5
networks:
- loki
logging:
driver: loki
options:
loki-url: "http://xxx.xx.xxx3100/loki/api/v1/push"