I am struggling to get the micro services going and executing some requests result in odd results.
I have the following docker-compose.yml
version: '3.2'
services:
consul:
image: bitnami/consul:latest
ports:
- "8500:8500"
# read path
loki-query-frontend:
image: grafana/loki
ports:
- "3100:3100"
deploy:
replicas: 2
resources:
limits:
memory: 600M
reservations:
memory: 300M
volumes:
- type: bind
source: ../internal/loki-config.yaml
target: /etc/loki-config.yaml
command: -config.file=/etc/loki-config.yaml -target=query-frontend
loki-querier:
image: grafana/loki
ports:
- "3101:3100"
deploy:
replicas: 2
resources:
limits:
memory: 600M
reservations:
memory: 300M
volumes:
- type: bind
source: ../internal/loki-config.yaml
target: /etc/loki-config.yaml
command: -config.file=/etc/loki-config.yaml -target=querier
# write path
# distributor
loki-distributor:
image: grafana/loki
ports:
- "3140:3100"
deploy:
replicas: 2
resources:
limits:
memory: 600M
reservations:
memory: 300M
volumes:
- type: bind
source: ../internal/loki-config.yaml
target: /etc/loki-config.yaml
command: -config.file=/etc/loki-config.yaml -target=distributor
loki-ingester:
image: grafana/loki
ports:
- "9096:9096"
deploy:
replicas: 2
resources:
limits:
memory: 600M
reservations:
memory: 300M
volumes:
- type: bind
source: ../internal/loki-config.yaml
target: /etc/loki-config.yaml
command: -config.file=/etc/loki-config.yaml -target=ingester
# other
loki-ruler:
image: grafana/loki
deploy:
replicas: 2
resources:
limits:
memory: 600M
reservations:
memory: 300M
volumes:
- type: bind
source: ../internal/loki-config.yaml
target: /etc/loki-config.yaml
command: -config.file=/etc/loki-config.yaml -target=ruler
loki-table-manager:
image: grafana/loki
deploy:
replicas: 1
resources:
limits:
memory: 600M
reservations:
memory: 300M
volumes:
- type: bind
source: ../internal/loki-config.yaml
target: /etc/loki-config.yaml
command: -config.file=/etc/loki-config.yaml -target=table-manager
and the following loki-config:
auth_enabled: true
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
grpc_server_max_recv_msg_size: 8388608
grpc_server_max_send_msg_size: 8388608
distributor:
ring:
kvstore:
store: 'consul'
prefix: loki/collectors/
consul:
host: 'consul:8500'
http_client_timeout: '20s'
consistent_reads: false
watch_rate_limit: 1
watch_burst_size: 1
ingester:
wal:
enabled: true
dir: /tmp/wal
lifecycler:
ring:
kvstore:
store: 'consul'
prefix: loki/collectors/
consul:
host: 'consul:8500'
http_client_timeout: '30s'
consistent_reads: true
replication_factor: 1
num_tokens: 512
heartbeat_period: '20s'
join_after: '20s'
observe_period: 10s
min_ready_duration: 0s
final_sleep: 5s
chunk_idle_period: 30m # Any chunk not receiving new logs in this time will be flushed
max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h
chunk_target_size: 262144 # Loki will attempt to build chunks up to 256kB, flushing first if chunk_idle_period or max_chunk_age is reached first
chunk_retain_period: 1m # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
max_transfer_retries: 0 # Chunk transfers disabled
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: s3
schema: v11
index:
prefix: loki_index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: /tmp/loki/boltdb-shipper-active
cache_location: /tmp/loki/boltdb-shipper-cache
cache_ttl: 168h # Can be increased for faster performance over longer query periods, uses more disk space
shared_store: s3
aws:
bucketnames: fr-loki-log-storage
endpoint: s3.eu-west-1.amazonaws.com
region: eu-west-1
access_key_id: ***
secret_access_key: ***
sse_encryption: true
compactor:
working_directory: /tmp/loki/boltdb-shipper-compactor
shared_store: s3
limits_config:
enforce_metric_name: false
reject_old_samples: false
reject_old_samples_max_age: 168h
max_cache_freshness_per_query: 10m
ingestion_rate_mb: 200
per_stream_rate_limit: "50MB"
chunk_store_config:
max_look_back_period: 0s
table_manager:
retention_deletes_enabled: false
retention_period: 0s
ruler:
storage:
type: s3
s3:
bucketnames: fr-loki-ruler-storage
region: eu-west-1
access_key_id: ***
secret_access_key: ***
insecure: false
sse_encryption: true
http_config:
idle_conn_timeout: 90s
response_header_timeout: 0s
insecure_skip_verify: false
s3forcepathstyle: false
rule_path: /tmp/loki/rules-temp
alertmanager_url: <cortex_url>
ring:
kvstore:
store: inmemory
enable_api: true
I can execute a push request and metrics request. However the following return a 404:
curl -X GET http://localhost:3140/ruler/ring
curl -X GET http://localhost:3140/api/prom/label
and this one is wierd (to me anyway):
curl -X GET http://localhost:3100/ready
Query Frontend not ready: not ready: number of queriers connected to query-frontend is 0
note that the following seems to be fine:
curl -X GET http://localhost:3140/ready
ready
I am not sure if I need to expose the same port for every service and also do I need to expose ports on all the services?