Link tempo with loki logs

Grafana Tempo
1

I am trying to setup the tempo trace to loki logs in connections.

image
image1425×616 36.9 KB

Can any one please help me with the query for the linking the logs to loki ?
Here is the sample trace json

{
	"batches": [
		{
			"resource": {
				"attributes": [
					{
						"key": "telemetry.sdk.language",
						"value": {
							"stringValue": "nodejs"
						}
					},
					{
						"key": "telemetry.sdk.name",
						"value": {
							"stringValue": "opentelemetry"
						}
					},
					{
						"key": "telemetry.sdk.version",
						"value": {
							"stringValue": "1.15.2"
						}
					},
					{
						"key": "container.id",
						"value": {
							"stringValue": "eb3b3c97119b7ff00c50ee3473aac0fc5fd0692c85c14004d9e7eded64.scope"
						}
					},
					{
						"key": "service.namespace",
						"value": {
							"stringValue": "opentelemetry-poc"
						}
					},
					{
						"key": "host.name",
						"value": {
							"stringValue": "opentelemetry-demo-frontend-787b4d49b7-xtcw2"
						}
					},
					{
						"key": "host.arch",
						"value": {
							"stringValue": "amd64"
						}
					},
					{
						"key": "os.type",
						"value": {
							"stringValue": "linux"
						}
					},
					{
						"key": "os.version",
						"value": {
							"stringValue": "4.18.0-372.71.1.el8_6.x86_64"
						}
					},
					{
						"key": "process.pid",
						"value": {
							"intValue": 17
						}
					},
					{
						"key": "process.executable.name",
						"value": {
							"stringValue": "node"
						}
					},
					{
						"key": "process.executable.path",
						"value": {
							"stringValue": "/usr/local/bin/node"
						}
					},
					{
						"key": "process.command_args",
						"value": {
							"stringValue": "[\"/usr/local/bin/node\",\"--require\",\"./Instrumentation.js\",\"/app/server.js\"]"
						}
					},
					{
						"key": "process.runtime.version",
						"value": {
							"stringValue": "18.18.2"
						}
					},
					{
						"key": "process.runtime.name",
						"value": {
							"stringValue": "nodejs"
						}
					},
					{
						"key": "process.runtime.description",
						"value": {
							"stringValue": "Node.js"
						}
					},
					{
						"key": "process.command",
						"value": {
							"stringValue": "/app/server.js"
						}
					},
					{
						"key": "process.owner",
						"value": {
							"stringValue": "nextjs"
						}
					},
					{
						"key": "k8s.pod.ip",
						"value": {
							"stringValue": "10.128.8.90"
						}
					},
					{
						"key": "k8s.node.name",
						"value": {
							"stringValue": "00-50-56-be-8e-d8"
						}
					},
					{
						"key": "k8s.pod.start_time",
						"value": {
							"stringValue": "2023-12-18 14:33:34 +0000 UTC"
						}
					},
					{
						"key": "k8s.pod.uid",
						"value": {
							"stringValue": "e3919530-a477-4a34-95ed-48d69da35397"
						}
					},
					{
						"key": "k8s.deployment.name",
						"value": {
							"stringValue": "opentelemetry-demo-frontend"
						}
					},
					{
						"key": "service.instance.id",
						"value": {
							"stringValue": "e3919530-a477-4a34-95ed-48d69da35397"
						}
					},
					{
						"key": "service.name",
						"value": {
							"stringValue": "frontend"
						}
					},
					{
						"key": "k8s.namespace.name",
						"value": {
							"stringValue": "opentelemetry-poc"
						}
					},
					{
						"key": "k8s.pod.name",
						"value": {
							"stringValue": "opentelemetry-demo-frontend-787b4d49b7-xtcw2"
						}
					}
				],
				"droppedAttributesCount": 0
			},
			"instrumentationLibrarySpans": [
				{
					"spans": [
						{
							"traceId": "43e54a049bc8b09ca45f06f8560e8ad3",
							"spanId": "836452e26293d56f",
							"parentSpanId": "d7c696f2965fa23f",
							"traceState": "",
							"name": "grpc.oteldemo.AdService/GetAds",
							"kind": "SPAN_KIND_CLIENT",
							"startTimeUnixNano": 1702910384753000000,
							"endTimeUnixNano": 1702910384756656000,
							"attributes": [
								{
									"key": "rpc.system",
									"value": {
										"stringValue": "grpc"
									}
								},
								{
									"key": "rpc.method",
									"value": {
										"stringValue": "GetAds"
									}
								},
								{
									"key": "rpc.service",
									"value": {
										"stringValue": "oteldemo.AdService"
									}
								},
								{
									"key": "net.peer.name",
									"value": {
										"stringValue": "opentelemetry-demo-adservice"
									}
								},
								{
									"key": "net.peer.port",
									"value": {
										"intValue": 8080
									}
								},
								{
									"key": "rpc.grpc.status_code",
									"value": {
										"intValue": 0
									}
								}
							],
							"droppedAttributesCount": 0,
							"droppedEventsCount": 0,
							"droppedLinksCount": 0,
							"status": {
								"code": 0,
								"message": ""
							}
						}
					],
					"instrumentationLibrary": {
						"name": "@opentelemetry/instrumentation-grpc",
						"version": "0.41.2"
					}
				}
			]
		},
		{
			"resource": {
				"attributes": [
					{
						"key": "telemetry.sdk.language",
						"value": {
							"stringValue": "nodejs"
						}
					},
					{
						"key": "telemetry.sdk.name",
						"value": {
							"stringValue": "opentelemetry"
						}
					},
					{
						"key": "telemetry.sdk.version",
						"value": {
							"stringValue": "1.15.2"
						}
					},
					{
						"key": "container.id",
						"value": {
							"stringValue": "eb3b3c97119b7ff00c50ee3473aac0fc5fd0692c85c14004d9e7eded64.scope"
						}
					},
					{
						"key": "service.namespace",
						"value": {
							"stringValue": "opentelemetry-poc"
						}
					},
					{
						"key": "host.name",
						"value": {
							"stringValue": "opentelemetry-demo-frontend-787b4d49b7-xtcw2"
						}
					},
					{
						"key": "host.arch",
						"value": {
							"stringValue": "amd64"
						}
					},
					{
						"key": "os.type",
						"value": {
							"stringValue": "linux"
						}
					},
					{
						"key": "os.version",
						"value": {
							"stringValue": "4.18.0-372.71.1.el8_6.x86_64"
						}
					},
					{
						"key": "process.pid",
						"value": {
							"intValue": 17
						}
					},
					{
						"key": "process.executable.name",
						"value": {
							"stringValue": "node"
						}
					},
					{
						"key": "process.executable.path",
						"value": {
							"stringValue": "/usr/local/bin/node"
						}
					},
					{
						"key": "process.command_args",
						"value": {
							"stringValue": "[\"/usr/local/bin/node\",\"--require\",\"./Instrumentation.js\",\"/app/server.js\"]"
						}
					},
					{
						"key": "process.runtime.version",
						"value": {
							"stringValue": "18.18.2"
						}
					},
					{
						"key": "process.runtime.name",
						"value": {
							"stringValue": "nodejs"
						}
					},
					{
						"key": "process.runtime.description",
						"value": {
							"stringValue": "Node.js"
						}
					},
					{
						"key": "process.command",
						"value": {
							"stringValue": "/app/server.js"
						}
					},
					{
						"key": "process.owner",
						"value": {
							"stringValue": "nextjs"
						}
					},
					{
						"key": "k8s.pod.ip",
						"value": {
							"stringValue": "10.128.8.90"
						}
					},
					{
						"key": "k8s.node.name",
						"value": {
							"stringValue": "00-50-56-be-8e-d8"
						}
					},
					{
						"key": "k8s.pod.start_time",
						"value": {
							"stringValue": "2023-12-18 14:33:34 +0000 UTC"
						}
					},
					{
						"key": "k8s.pod.uid",
						"value": {
							"stringValue": "e3919530-a477-4a34-95ed-48d69da35397"
						}
					},
					{
						"key": "k8s.deployment.name",
						"value": {
							"stringValue": "opentelemetry-demo-frontend"
						}
					},
					{
						"key": "service.instance.id",
						"value": {
							"stringValue": "e3919530-a477-4a34-95ed-48d69da35397"
						}
					},
					{
						"key": "service.name",
						"value": {
							"stringValue": "frontend"
						}
					},
					{
						"key": "k8s.namespace.name",
						"value": {
							"stringValue": "opentelemetry-poc"
						}
					},
					{
						"key": "k8s.pod.name",
						"value": {
							"stringValue": "opentelemetry-demo-frontend-787b4d49b7-xtcw2"
						}
					}
				],
				"droppedAttributesCount": 0
			},
			"instrumentationLibrarySpans": [
				{
					"spans": [
						{
							"traceId": "43e54a049bc8b09ca45f06f8560e8ad3",
							"spanId": "d7c696f2965fa23f",
							"parentSpanId": "0000000000000000",
							"traceState": "",
							"name": "HTTP GET",
							"kind": "SPAN_KIND_SERVER",
							"startTimeUnixNano": 1702910384753000000,
							"endTimeUnixNano": 1702910384756981800,
							"attributes": [
								{
									"key": "app.synthetic_request",
									"value": {
										"boolValue": true
									}
								},
								{
									"key": "http.target",
									"value": {
										"stringValue": "/api/data"
									}
								},
								{
									"key": "http.user_agent",
									"value": {
										"stringValue": "python-requests/2.31.0"
									}
								},
								{
									"key": "http.flavor",
									"value": {
										"stringValue": "1.1"
									}
								},
								{
									"key": "http.method",
									"value": {
										"stringValue": "GET"
									}
								},
								{
									"key": "http.url",
									"value": {
										"stringValue": "opentelemetry-demo-frontendproxy:8080/api/data"
									}
								},
								{
									"key": "http.status_code",
									"value": {
										"intValue": 200
									}
								}
							],
							"droppedAttributesCount": 0,
							"droppedEventsCount": 0,
							"droppedLinksCount": 0,
							"status": {
								"code": 0,
								"message": ""
							},
							"links": [
								{
									"traceId": "d39d0e3fe09113da71fe2e16a0b8d597",
									"spanId": "c3e4ef1b200c3679",
									"attributes": [],
									"droppedAttributesCount": 0
								}
							]
						}
					],
					"instrumentationLibrary": {
						"name": "frontend",
						"version": ""
					}
				}
			]
		},
		{
			"resource": {
				"attributes": [
					{
						"key": "container.id",
						"value": {
							"stringValue": "7de2f15ffbf92a1e17f5f6b9e0f1aa3179b143f4b3cddc0ca932cdfd26fa5f14"
						}
					},
					{
						"key": "host.arch",
						"value": {
							"stringValue": "amd64"
						}
					},
					{
						"key": "host.name",
						"value": {
							"stringValue": "opentelemetry-demo-adservice-54db87b68f-hbhk4"
						}
					},
					{
						"key": "os.description",
						"value": {
							"stringValue": "Linux 4.18.0-372.71.1.el8_6.x86_64"
						}
					},
					{
						"key": "os.type",
						"value": {
							"stringValue": "linux"
						}
					},
					{
						"key": "process.command_line",
						"value": {
							"stringValue": "/opt/java/openjdk/bin/java -javaagent:/usr/src/app/opentelemetry-javaagent.jar oteldemo.AdService"
						}
					},
					{
						"key": "process.executable.path",
						"value": {
							"stringValue": "/opt/java/openjdk/bin/java"
						}
					},
					{
						"key": "process.pid",
						"value": {
							"intValue": 1
						}
					},
					{
						"key": "process.runtime.description",
						"value": {
							"stringValue": "Eclipse Adoptium OpenJDK 64-Bit Server VM 17.0.8.1+1"
						}
					},
					{
						"key": "process.runtime.name",
						"value": {
							"stringValue": "OpenJDK Runtime Environment"
						}
					},
					{
						"key": "process.runtime.version",
						"value": {
							"stringValue": "17.0.8.1+1"
						}
					},
					{
						"key": "service.namespace",
						"value": {
							"stringValue": "opentelemetry-poc"
						}
					},
					{
						"key": "telemetry.auto.version",
						"value": {
							"stringValue": "1.28.0"
						}
					},
					{
						"key": "telemetry.sdk.language",
						"value": {
							"stringValue": "java"
						}
					},
					{
						"key": "telemetry.sdk.name",
						"value": {
							"stringValue": "opentelemetry"
						}
					},
					{
						"key": "telemetry.sdk.version",
						"value": {
							"stringValue": "1.28.0"
						}
					},
					{
						"key": "k8s.pod.ip",
						"value": {
							"stringValue": "10.130.2.185"
						}
					},
					{
						"key": "k8s.node.name",
						"value": {
							"stringValue": "00-50-56-be-9f-af"
						}
					},
					{
						"key": "k8s.pod.start_time",
						"value": {
							"stringValue": "2023-12-18 14:27:57 +0000 UTC"
						}
					},
					{
						"key": "k8s.pod.uid",
						"value": {
							"stringValue": "c1cbba51-e5fb-42b3-9d8c-bf523d4c989d"
						}
					},
					{
						"key": "k8s.deployment.name",
						"value": {
							"stringValue": "opentelemetry-demo-adservice"
						}
					},
					{
						"key": "service.instance.id",
						"value": {
							"stringValue": "c1cbba51-e5fb-42b3-9d8c-bf523d4c989d"
						}
					},
					{
						"key": "service.name",
						"value": {
							"stringValue": "adservice"
						}
					},
					{
						"key": "k8s.namespace.name",
						"value": {
							"stringValue": "opentelemetry-poc"
						}
					},
					{
						"key": "k8s.pod.name",
						"value": {
							"stringValue": "opentelemetry-demo-adservice-54db87b68f-hbhk4"
						}
					}
				],
				"droppedAttributesCount": 0
			},
			"instrumentationLibrarySpans": [
				{
					"spans": [
						{
							"traceId": "43e54a049bc8b09ca45f06f8560e8ad3",
							"spanId": "18d4fa0ec560e730",
							"parentSpanId": "836452e26293d56f",
							"traceState": "",
							"name": "oteldemo.AdService/GetAds",
							"kind": "SPAN_KIND_SERVER",
							"startTimeUnixNano": 1702910384755497000,
							"endTimeUnixNano": 1702910384756744700,
							"attributes": [
								{
									"key": "rpc.grpc.status_code",
									"value": {
										"intValue": 0
									}
								},
								{
									"key": "net.sock.peer.port",
									"value": {
										"intValue": 59466
									}
								},
								{
									"key": "rpc.method",
									"value": {
										"stringValue": "GetAds"
									}
								},
								{
									"key": "thread.id",
									"value": {
										"intValue": 36
									}
								},
								{
									"key": "app.ads.ad_response_type",
									"value": {
										"stringValue": "RANDOM"
									}
								},
								{
									"key": "app.ads.count",
									"value": {
										"intValue": 2
									}
								},
								{
									"key": "net.host.port",
									"value": {
										"intValue": 8080
									}
								},
								{
									"key": "net.host.name",
									"value": {
										"stringValue": "opentelemetry-demo-adservice"
									}
								},
								{
									"key": "rpc.service",
									"value": {
										"stringValue": "oteldemo.AdService"
									}
								},
								{
									"key": "net.sock.peer.addr",
									"value": {
										"stringValue": "10.128.8.90"
									}
								},
								{
									"key": "app.ads.contextKeys.count",
									"value": {
										"intValue": 0
									}
								},
								{
									"key": "app.ads.contextKeys",
									"value": {
										"stringValue": "[]"
									}
								},
								{
									"key": "app.ads.ad_request_type",
									"value": {
										"stringValue": "NOT_TARGETED"
									}
								},
								{
									"key": "rpc.system",
									"value": {
										"stringValue": "grpc"
									}
								},
								{
									"key": "thread.name",
									"value": {
										"stringValue": "grpc-default-executor-0"
									}
								}
							],
							"droppedAttributesCount": 0,
							"droppedEventsCount": 0,
							"droppedLinksCount": 0,
							"status": {
								"code": 0,
								"message": ""
							},
							"events": [
								{
									"timeUnixNano": 1702910384755771100,
									"attributes": [
										{
											"key": "message",
											"value": {
												"stringValue": {
													"Type": "STRING",
													"Value": "message"
												}
											}
										},
										{
											"key": "message.id",
											"value": {
												"intValue": 1
											}
										},
										{
											"key": "message.type",
											"value": {
												"stringValue": "RECEIVED"
											}
										}
									],
									"droppedAttributesCount": 0,
									"name": ""
								},
								{
									"timeUnixNano": 1702910384756583400,
									"attributes": [
										{
											"key": "message",
											"value": {
												"stringValue": {
													"Type": "STRING",
													"Value": "message"
												}
											}
										},
										{
											"key": "message.id",
											"value": {
												"intValue": 2
											}
										},
										{
											"key": "message.type",
											"value": {
												"stringValue": "SENT"
											}
										}
									],
									"droppedAttributesCount": 0,
									"name": ""
								}
							]
						},
						{
							"traceId": "43e54a049bc8b09ca45f06f8560e8ad3",
							"spanId": "ea0f1a48b40587ba",
							"parentSpanId": "18d4fa0ec560e730",
							"traceState": "",
							"name": "getRandomAds",
							"kind": "SPAN_KIND_INTERNAL",
							"startTimeUnixNano": 1702910384756238600,
							"endTimeUnixNano": 1702910384756318500,
							"attributes": [
								{
									"key": "thread.id",
									"value": {
										"intValue": 36
									}
								},
								{
									"key": "app.ads.count",
									"value": {
										"intValue": 2
									}
								},
								{
									"key": "thread.name",
									"value": {
										"stringValue": "grpc-default-executor-0"
									}
								}
							],
							"droppedAttributesCount": 0,
							"droppedEventsCount": 0,
							"droppedLinksCount": 0,
							"status": {
								"code": 0,
								"message": ""
							}
						}
					],
					"instrumentationLibrary": {
						"name": "io.opentelemetry.grpc-1.6",
						"version": "1.28.0-alpha"
					}
				}
			]
		}
	]
}
2

That depends what is saved in your logs. For example you can use:

{appname=~".+"} |~ "\"traceID\":\"$${__trace.traceId}\"

if you have Loki logs with appname label + trace id in the logs in the format "traceID":"<trace-id>"

3

I have tried this but it gives an error .

image
image1887×637 43.9 KB

Below is the extract of the sample loki logs.

"
{
“body”: “EmptyCartAsync called with userId={userId}”,
“traceid”: “024d6178134eaca9e25ff3e6fc26fc1a”,
“spanid”: “28b493373a024684”,
“severity”: “Information”,
“flags”: 1,
“attributes”: {
“userId”: “bb41a576-9db8-11ee-829b-0a580a8202be”
},
“resources”: {
“container.id”: “efb2e07d9a554bd6e58994de9fa4843593292102dfa07542d2cde2767083bcce”,
“k8s.deployment.name”: “opentelemetry-demo-cartservice”,
“k8s.namespace.name”: “opentelemetry-poc”,
“k8s.node.name”: “00-50-56-be-9f-af”,
“k8s.pod.ip”: “10.130.2.186”,
“k8s.pod.name”: “opentelemetry-demo-cartservice-7dbf75896f-5vz9s”,
“k8s.pod.start_time”: “2023-12-18 14:28:22 +0000 UTC”,
“k8s.pod.uid”: “e04d08c7-a8ac-4d3e-b56a-256154085f56”,
“service.name”: “cartservice”,
“service.namespace”: “opentelemetry-poc”,
“telemetry.sdk.language”: “dotnet”,
“telemetry.sdk.name”: “opentelemetry”,
“telemetry.sdk.version”: “1.6.0”
}
}

4

Of course, I don’t write you copy&paste solution from my head. Please edit it and fix any syntax issue, so it will create proper search. Please use also proper formatting for logs, traces for better readibility.

5

Thanks @jangaraj , but i have tried with this as well “{job=${service.namespace}/${service.name}}” |= "${traceid}"
But still not able to resolve it

6

I was able to change it accordingly. It is working now. Thank you

7

This topic was automatically closed 365 days after the last reply. New replies are no longer allowed.