Using the loki-distributed helm chart to install loki as well as the promtail helm chart. The loki services do not appear to be pushing their data to the s3 bucket and also do not appear to be sharing state with each other as their caches/index directories are empty.
Loki Config is shown here:
auth_enabled: false
server:
http_listen_port: 3100
distributor:
ring:
kvstore:
store: memberlist
memberlist:
join_members:
- loki-distributed-memberlist
ingester:
lifecycler:
ring:
kvstore:
store: memberlist
replication_factor: 1
chunk_idle_period: 30m
chunk_block_size: 262144
chunk_encoding: snappy
chunk_retain_period: 1m
max_transfer_retries: 0
limits_config:
enforce_metric_name: false
reject_old_samples: true
reject_old_samples_max_age: 168h
max_cache_freshness_per_query: 10m
schema_config:
configs:
- from: 2020-09-07
store: boltdb-shipper
object_store: aws
schema: v11
index:
prefix: loki_index_
period: 24h
storage_config:
aws:
s3: s3://${s3_bucket_region}
bucketnames: ${s3_bucket_name}
access_key_id: ${access_key_id}
secret_access_key: ${secret_access_key}
boltdb_shipper:
active_index_directory: /var/loki/index
shared_store: s3
cache_location: /var/loki/cache
chunk_store_config:
max_look_back_period: 0s
table_manager:
retention_deletes_enabled: false
retention_period: 0s
query_range:
align_queries_with_step: true
max_retries: 5
split_queries_by_interval: 15m
cache_results: true
results_cache:
cache:
enable_fifocache: true
fifocache:
max_size_items: 1024
validity: 24h
frontend_worker:
frontend_address: loki-distributed-query-frontend:9095
frontend:
log_queries_longer_than: 5s
compress_responses: true
tail_proxy_url: http://loki-distributed-querier:3100
compactor:
shared_store: s3
ruler:
storage:
type: s3
s3:
s3: s3://${s3_bucket_region}
bucketnames: ${s3_bucket_name}
access_key_id: ${access_key_id}
secret_access_key: ${secret_access_key}
ring:
kvstore:
store: memberlist
rule_path: /tmp/loki/scratch
alertmanager_url: https://alertmanager.xx
external_url: https://alertmanager.xx