- Grafana agent (config.river), setup with Grafana Cloud,
Installed to GKE Kuberntes cluster.
- I want to filter a log line (“level”:“ACTIVITY”) and send to a different out than the LOKI endpoint in Grafana cloudn
Trying to figure it out from online instructions : loki.process | Grafana Agent documentation but am kind of lost.
Can not find a good example on how to do this.
Example log:
{“level”:“ACTIVITY”, “message”: “various stuff”} (a log line i do not wan to send to Loki)
{"level:“INFO”, “ip”: “1.2.2.3”, “http”:“GET /”} … (all other go to Loki endpoint in Grafana cloud.
Default config:
discovery.kubernetes "pods" {
role = "pod"
}
// Pod
Logs
discovery.relabel "pod_logs" {
targets = discovery.kubernetes.pods.targets
\ rule {
source_labels = ["__meta_kubernetes_namespace"]
action =
"replace"
target_label = "namespace"
}
rule {
source_labels
= ["__meta_kubernetes_pod_name"]
action = "replace"
target_label
= "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
\ action = "replace"
target_label = "container"
}
rule {
source_labels
= ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
separator
= "/"
action = "replace"
replacement = "$1"
target_label
= "job"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
\ action = "keep"
regex = env("HOSTNAME")
}
rule {
source_labels
= ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
\ separator = "/"
action = "replace"
replacement = "/var/log/pods/*$1/*.log"
\ target_label = "__path__"
}
}
local.file_match "pod_logs" {
path_targets
= discovery.relabel.pod_logs.output
}
loki.source.file "pod_logs" {
targets
\ = local.file_match.pod_logs.targets
forward_to = [loki.process.pod_logs.receiver]
}
loki.process
"pod_logs" {
stage.docker {}
forward_to = [loki.write.grafana_cloud_loki.receiver]
}
\
// Grafana Cloud Loki
local.file "loki_host" {
filename = "/etc/grafana-agent-credentials/loki_host"
}
local.file
"loki_username" {
filename = "/etc/grafana-agent-credentials/loki_username"
}
local.file
"loki_password" {
filename = "/etc/grafana-agent-credentials/loki_password"
\ is_secret = true
}
loki.write "grafana_cloud_loki" {
endpoint {
url
= nonsensitive(local.file.loki_host.content) + "/loki/api/v1/push"
basic_auth
{
username = local.file.loki_username.content
password = local.file.loki_password.content
\ }
}
external_labels = {
cluster = "keystrike-development",