I have installed manually loki-stack with promtail in my DEV EKS cluster
helm upgrade --install loki grafana/loki-stack \
--namespace infra \
--create-namespace \
--set grafana.enabled=true \
--set prometheus.enabled=true \
--set loki.persistence.enabled=true \
--set loki.persistence.storageClassName=gp2 \
--set loki.persistence.size=10Gi
and
helm upgrade --install promtail grafana/promtail \
--namespace infra \
--set config.lokiAddress=http://loki.infra:3100/loki/api/v1/push \
--set config.filepath=/var/log/containers/*.log
I tried to replicate the same with Terraform to deploy to my UAT EKS cluster
locals {
facts = var.tags
}
#########################################
### Deploy Loki
resource "helm_release" "loki" {
name = "loki"
repository = "https://grafana.github.io/helm-charts"
chart = "loki-stack"
namespace = "infra"
values = [
jsonencode({
grafana = {
enabled = true
additionalDataSources = [{
name = "Loki"
type = "loki"
url = "http://loki.infra.svc.cluster.local:3100"
access = "proxy"
isDefault = true
}]
}
prometheus = {
enabled = true
persistence = {
enabled = true
storageClassName = "gp2"
size = "8Gi"
}
}
alertmanager = {
enabled = true
persistence = {
enabled = true
storageClassName = "gp2"
size = "2Gi"
}
}
loki = {
persistence = {
enabled = true
storageClassName = "gp2"
size = var.loki_storage_size
}
limits_config = {
retention_period = var.loki_log_retention
compactor_retention_enabled = true
}
image = {
registry = "docker.io"
repository = "grafana/loki"
tag = "2.9.3" # must be this version because the bundled one does not work (bugs)
}
}
})
]
}
#########################################
### Deploy Promtail for log shipping
resource "helm_release" "promtail" {
name = "promtail"
repository = "https://grafana.github.io/helm-charts"
chart = "promtail"
namespace = "infra"
values = [
jsonencode({
config: {
lokiAddress: "http://loki.infra:3100/loki/api/v1/push"
filepath: "/var/log/containers/*.log"
}
})
]
depends_on = [helm_release.loki]
}
but the problem is that the alertmanager and prometheus PVCs from the Terraform (UAT EKS) are not being bound for they are missing storage class definition 
$ kubectl -n infra get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE
data-rabbit-rabbitmq-0 Bound pvc-ffea3acd-fdd2-40dc-bef9-f5d45e9c3b26 8Gi RWO gp2 <unset> 56m
data-rabbit-rabbitmq-1 Bound pvc-8aac39e0-2859-4eeb-809f-b417f6bcedc5 8Gi RWO gp2 <unset> 56m
data-rabbit-rabbitmq-2 Bound pvc-5b5bce04-34f4-4948-9fc8-43a1ebb4e2c2 8Gi RWO gp2 <unset> 56m
loki-prometheus-server Pending <unset> 4m10s
storage-loki-0 Bound pvc-7bb1e4aa-1561-4a59-b4d7-30a3814f6d9b 10Gi RWO gp2 <unset> 53m
storage-loki-alertmanager-0 Pending <unset> 53m
loki-prometheus-server.yaml
$ kubectl get pvc loki-prometheus-server -n infra -o yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
meta.helm.sh/release-name: loki
meta.helm.sh/release-namespace: infra
creationTimestamp: "2024-12-20T08:55:09Z"
finalizers:
- kubernetes.io/pvc-protection
labels:
app: prometheus
app.kubernetes.io/managed-by: Helm
chart: prometheus-19.7.2
component: server
heritage: Helm
release: loki
name: loki-prometheus-server
namespace: infra
resourceVersion: "43914"
uid: ***moderated***
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 8Gi
volumeMode: Filesystem
status:
phase: Pending
storage-loki-alertmanager.yaml
$ kubectl get pvc storage-loki-alertmanager-0 -n infra -o yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: "2024-12-20T08:48:44Z"
finalizers:
- kubernetes.io/pvc-protection
labels:
app.kubernetes.io/instance: loki
app.kubernetes.io/name: alertmanager
name: storage-loki-alertmanager-0
namespace: infra
resourceVersion: "41942"
uid: ***moderated***
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
volumeMode: Filesystem
status:
phase: Pending
which is weird, for the manually deployed helms were deployed fine in the DEV EKS
$ kubectl get pvc loki-prometheus-server -n infra -o yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
meta.helm.sh/release-name: loki
meta.helm.sh/release-namespace: infra
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
volume.beta.kubernetes.io/storage-provisioner: ebs.csi.aws.com
volume.kubernetes.io/selected-node: ***moderated***
volume.kubernetes.io/storage-provisioner: ebs.csi.aws.com
creationTimestamp: "2024-12-17T14:49:20Z"
finalizers:
- kubernetes.io/pvc-protection
labels:
app: prometheus
app.kubernetes.io/managed-by: Helm
chart: prometheus-19.7.2
component: server
heritage: Helm
release: loki
name: loki-prometheus-server
namespace: infra
resourceVersion: "110099267"
uid: ***moderated***
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 8Gi
storageClassName: gp2
volumeMode: Filesystem
volumeName: ***moderated***
status:
accessModes:
- ReadWriteOnce
capacity:
storage: 8Gi
phase: Bound
$ kubectl get pvc storage-loki-alertmanager-0 -n infra -o yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
volume.beta.kubernetes.io/storage-provisioner: ebs.csi.aws.com
volume.kubernetes.io/selected-node: ***moderated***
volume.kubernetes.io/storage-provisioner: ebs.csi.aws.com
creationTimestamp: "2024-12-17T14:49:21Z"
finalizers:
- kubernetes.io/pvc-protection
labels:
app.kubernetes.io/instance: loki
app.kubernetes.io/name: alertmanager
name: storage-loki-alertmanager-0
namespace: infra
resourceVersion: "110099264"
uid: ***moderated***
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
storageClassName: gp2
volumeMode: Filesystem
volumeName: ***moderated***
status:
accessModes:
- ReadWriteOnce
capacity:
storage: 2Gi
phase: Bound