Grafana Logstash cloudwatchlog integration

Grafana Loki
#1

Hi,

I am trying to you Logstash to push AWS Cloudwatchlog to Loki. And I am able to do that but only message are getting uploaded I am not able to inmort other details with the log like log_group and message_stream details. Can you please help. I am attaching logstash config below.

Sample Logstash configuration for creating a simple

Beats → Logstash → Elasticsearch pipeline.

input {
cloudwatch_logs {
log_group => [ “/aws/lambda/BCPGHostService”,"/aws/lambda/BCLoggingService" ]
access_key_id => “XXX”
secret_access_key => “XXX”
region => “ap-south-1”
start_position => “end”
type => “lambda”
}

}

filter {
grok {
match => { “message” => “%{TIMESTAMP_ISO8601}\t%{UUID:[lambda][request_id]}\t%{GREEDYDATA:message}” }
overwrite => [ “message” ]
tag_on_failure => []
}

grok {
    match => { "message" => "(?:START|END) RequestId': '%{UUID:[lambda][request_id]}" }
    tag_on_failure => []
}

grok {
    match => { "message" => "REPORT RequestId: %{UUID:[lambda][request_id]}\tDuration: %{BASE16FLOAT:[lambda][duration]} ms\tBilled Duration: %{BASE16FLOAT:[lambda][billed_duration]} ms \tMemory Size: %{BASE10NUM:[lambda][memory_size]} MB\tMax Memory Used: %{BASE10NUM:[lambda][memory_used]} MB" }
    tag_on_failure => []
	
}

mutate {
    convert => {
        "[lambda][duration]" => "integer"
        "[lambda][billed_duration]" => "integer"
        "[lambda][memory_size]" => "integer"
        "[lambda][memory_used]" => "integer"
    }
	add_field => {
        "log_group" => "%{[log_group]}"
    }
}

}

output {
loki {
url => “https://logs-prod-us-central1.grafana.net/loki/api/v1/push”
username => “XXX”
password => “XXX”
batch_size => 11264 #112.64 kilobytes
retries => 5
min_delay => 3
max_delay => 500
message_field => “message”
}
stdout { codec => rubydebug }
}