Can anyone describe why there are two hidden, duplicate items showing up on the chart?
- What Grafana version and what operating system are you using?
Grafana v12.0.2 (5bda17e7c1) in Docker container
- What are you trying to achieve?
Get a single item on the chart for each of two different metrics, by aggregating the sum() of metric values, across multiple label values.
- How are you trying to achieve it?
For an Access Point’s SNMP metrics, I am querying the inbound errors and outbound errors, as two metric queries. For each of the two counter metrics, I am using a Sum() aggregation to combine the values, across all Network Interfaces (eg. br0, ath0, eth0, wifi0, wifi1 etc.) into a single value.
What happened?
The chart is showing two different lines / colors, for each of the two metrics I’m querying. However, the duplicate items on the chart, don’t actually seem to have any values. There are two duplicate “ghost” items showing up in the chart.
The yellow and orange duplicate items are invisible, while the green and blue lines are actually visible and have values.
- What did you expect to happen?
Each of the two metrics should have been aggregated into a single item on the chart.
- Can you copy/paste the configuration(s) that you are having problems with?
{
"request": {
"url": "api/ds/query?ds_type=prometheus&requestId=SQR116",
"method": "POST",
"data": {
"queries": [
{
"refId": "A",
"expr": "sum(rate(interface_InboundErrors{agent_host=\"10.0.0.45\"}[$__rate_interval]))",
"range": true,
"instant": false,
"datasource": {
"type": "prometheus",
"uid": "ceqh1o5jdp8g0a"
},
"editorMode": "builder",
"legendFormat": "__auto",
"useBackend": false,
"disableTextWrap": false,
"fullMetaSearch": false,
"includeNullMetadata": true,
"exemplar": false,
"requestId": "4A",
"utcOffsetSec": -21600,
"scopes": [],
"adhocFilters": [],
"interval": "",
"datasourceId": 2,
"intervalMs": 15000,
"maxDataPoints": 1386
},
{
"refId": "A-Instant",
"expr": "sum(rate(interface_InboundErrors{agent_host=\"10.0.0.45\"}[$__rate_interval]))",
"range": false,
"instant": true,
"datasource": {
"type": "prometheus",
"uid": "ceqh1o5jdp8g0a"
},
"editorMode": "builder",
"legendFormat": "__auto",
"useBackend": false,
"disableTextWrap": false,
"fullMetaSearch": false,
"includeNullMetadata": true,
"exemplar": false,
"requestId": "4A",
"utcOffsetSec": -21600,
"scopes": [],
"adhocFilters": [],
"interval": "",
"datasourceId": 2,
"intervalMs": 15000,
"maxDataPoints": 1386
},
{
"refId": "B",
"expr": "sum(rate(interface_OutboundErrors{agent_host=\"10.0.0.45\"}[$__rate_interval]))",
"range": true,
"instant": false,
"key": "Q-2cd8320d-76e6-498f-a257-5fda32a81b69-1",
"datasource": {
"type": "prometheus",
"uid": "ceqh1o5jdp8g0a"
},
"editorMode": "builder",
"legendFormat": "__auto",
"useBackend": false,
"disableTextWrap": false,
"fullMetaSearch": false,
"includeNullMetadata": true,
"exemplar": false,
"requestId": "4B",
"utcOffsetSec": -21600,
"scopes": [],
"adhocFilters": [],
"interval": "",
"datasourceId": 2,
"intervalMs": 15000,
"maxDataPoints": 1386
},
{
"refId": "B-Instant",
"expr": "sum(rate(interface_OutboundErrors{agent_host=\"10.0.0.45\"}[$__rate_interval]))",
"range": false,
"instant": true,
"key": "Q-2cd8320d-76e6-498f-a257-5fda32a81b69-1",
"datasource": {
"type": "prometheus",
"uid": "ceqh1o5jdp8g0a"
},
"editorMode": "builder",
"legendFormat": "__auto",
"useBackend": false,
"disableTextWrap": false,
"fullMetaSearch": false,
"includeNullMetadata": true,
"exemplar": false,
"requestId": "4B",
"utcOffsetSec": -21600,
"scopes": [],
"adhocFilters": [],
"interval": "",
"datasourceId": 2,
"intervalMs": 15000,
"maxDataPoints": 1386
}
],
"from": "1751433070386",
"to": "1751436670386"
},
"hideFromInspector": false
},
"response": {
"results": {
"A": {
"status": 200,
"frames": [
{
"schema": {
"refId": "A",
"meta": {
"type": "timeseries-multi",
"typeVersion": [
0,
1
],
"custom": {
"resultType": "matrix"
},
"notices": [
{
"severity": "warning",
"text": "PromQL info: metric might not be a counter, name does not end in _total/_sum/_count/_bucket: \"interface_InboundErrors\" (1:10)"
}
],
"executedQueryString": "Expr: sum(rate(interface_InboundErrors{agent_host=\"10.0.0.45\"}[1m0s]))\nStep: 15s"
},
"fields": [
{
"name": "Time",
"type": "time",
"typeInfo": {
"frame": "time.Time"
},
"config": {
"interval": 15000
}
},
{
"name": "Value",
"type": "number",
"typeInfo": {
"frame": "float64"
},
"labels": {},
"config": {
"displayNameFromDS": "sum(rate(interface_InboundErrors{agent_host=\"10.0.0.45\"}[1m0s]))"
}
}
]
},
"data": {
"values": [
[
1751434890000,
1751434905000,
1751434920000,
1751434935000,
1751434950000,
1751434965000,
1751434980000,
1751434995000,
1751435010000,
1751435025000,
1751435040000,
1751435055000,
1751435070000,
1751435085000,
1751435100000,
1751435115000,
1751435130000,
1751435145000,
1751435160000,
1751435175000,
1751435190000,
1751435205000,
1751435220000,
1751435235000,
1751435250000,
1751435265000,
1751435280000,
1751435295000,
1751435310000,
1751435325000,
1751435340000,
1751435355000,
1751435370000,
1751435385000,
1751435400000,
1751435415000,
1751435430000,
1751435445000,
1751435460000,
1751435475000,
1751435490000,
1751435505000,
1751435520000,
1751435535000,
1751435550000,
1751435565000,
1751435580000,
1751435595000,
1751435610000,
1751435625000,
1751435640000,
1751435655000,
1751435670000,
1751435685000,
1751435700000,
1751435715000,
1751435730000,
1751435745000,
1751435760000,
1751435775000,
1751435790000,
1751435805000,
1751435820000,
1751435835000,
1751435850000,
1751435865000,
1751435880000,
1751435895000,
1751435910000,
1751435925000,
1751435940000,
1751435955000,
1751435970000,
1751435985000,
1751436000000,
1751436015000,
1751436030000,
1751436045000,
1751436060000,
1751436075000,
1751436090000,
1751436105000,
1751436120000,
1751436135000,
1751436150000,
1751436165000,
1751436180000,
1751436195000,
1751436210000,
1751436225000,
1751436240000,
1751436255000,
1751436270000,
1751436285000,
1751436300000,
1751436315000,
1751436330000,
1751436345000,
1751436360000,
1751436375000,
1751436390000,
1751436405000,
1751436420000,
1751436435000,
1751436450000,
1751436465000,
1751436480000,
1751436495000,
1751436510000,
1751436525000,
1751436540000,
1751436555000,
1751436570000,
1751436585000,
1751436600000,
1751436615000,
1751436630000,
1751436645000,
1751436660000
],
[
39.01952,
74.36634,
94.66666666666666,
107.84444444444443,
86.79999999999998,
88.46666666666665,
70.33333333333331,
107.53333333333333,
109.15555555555554,
142.04444444444442,
119.39999999999999,
124.44444444444444,
92.19999999999999,
89.55555555555554,
80.17777777777776,
99.84444444444443,
91.08888888888889,
104.68888888888887,
89.17777777777776,
102.13333333333331,
81.3111111111111,
89.04444444444442,
76.77777777777777,
87.79999999999998,
86.02222222222221,
114.08888888888887,
99.79999999999998,
110.53333333333332,
82.28888888888888,
99.55555555555554,
79.66666666666666,
108.06666666666665,
97.1111111111111,
125.6222222222222,
97.08888888888887,
107.79999999999998,
90.6222222222222,
103.28888888888888,
90.57777777777777,
103.04444444444444,
90.35555555555554,
89.95555555555555,
67.44444444444443,
83.15555555555554,
79.22222222222221,
83.5111111111111,
69.13333333333333,
119.6222222222222,
122.77777777777776,
129.57777777777775,
82.33333333333333,
83.3111111111111,
72.97777777777777,
75.64444444444443,
61.35555555555555,
68.13333333333333,
61.66666666666666,
69.77777777777776,
61.133333333333326,
87.57777777777777,
93.3111111111111,
121.39999999999999,
113.37777777777777,
132.46666666666664,
113.71111111111111,
108.84444444444443,
79.73333333333332,
91.33333333333333,
80.22222222222221,
92.39999999999999,
75.37777777777777,
91.79999999999998,
84.15555555555554,
93.75555555555555,
76.84444444444443,
75.06333051864362,
59.22222222222222,
65.17053660704364,
58.55815814036179,
74.95555555555555,
70.87454161573507,
84.79999999999998,
69.4222222222222,
73.46666666666665,
61.199999999999996,
72.86666666666666,
67.17777777777778,
78.1111111111111,
63.377777777777766,
80.77777777777777,
66.24444444444444,
88.51111111111109,
71.3111111111111,
83.33333333333331,
66.22222222222221,
77.66666666666666,
65.57777777777777,
81.84444444444443,
68.46666666666667,
90.39999999999999,
70.33333333333331,
85.42222222222222,
62.79999999999999,
82.82222222222221,
64.91111111111111,
84.42222222222222,
71.06666666666665,
88.24444444444443,
67.93333333333332,
84.73333333333332,
65.53333333333332,
90,
65.04444444444444,
84.19999999999999,
61.04444444444444,
96.53333333333333,
89.35555555555554,
114.4222222222222,
80.3111111111111
]
]
}
}
],
"refId": "A"
},
"A-Instant": {
"status": 200,
"frames": [
{
"schema": {
"refId": "A-Instant",
"meta": {
"type": "numeric-multi",
"typeVersion": [
0,
1
],
"custom": {
"resultType": "vector"
},
"notices": [
{
"severity": "warning",
"text": "PromQL info: metric might not be a counter, name does not end in _total/_sum/_count/_bucket: \"interface_InboundErrors\" (1:10)"
}
],
"executedQueryString": "Expr: sum(rate(interface_InboundErrors{agent_host=\"10.0.0.45\"}[1m0s]))\nStep: 15s"
},
"fields": [
{
"name": "Time",
"type": "time",
"typeInfo": {
"frame": "time.Time"
},
"config": {
"interval": 15000
}
},
{
"name": "Value",
"type": "number",
"typeInfo": {
"frame": "float64"
},
"labels": {},
"config": {
"displayNameFromDS": "sum(rate(interface_InboundErrors{agent_host=\"10.0.0.45\"}[1m0s]))"
}
}
]
},
"data": {
"values": [
[
1751436670386
],
[
91.06666666666666
]
]
}
}
],
"refId": "A-Instant"
},
"B": {
"status": 200,
"frames": [
{
"schema": {
"refId": "B",
"meta": {
"type": "timeseries-multi",
"typeVersion": [
0,
1
],
"custom": {
"resultType": "matrix"
},
"notices": [
{
"severity": "warning",
"text": "PromQL info: metric might not be a counter, name does not end in _total/_sum/_count/_bucket: \"interface_OutboundErrors\" (1:10)"
}
],
"executedQueryString": "Expr: sum(rate(interface_OutboundErrors{agent_host=\"10.0.0.45\"}[1m0s]))\nStep: 15s"
},
"fields": [
{
"name": "Time",
"type": "time",
"typeInfo": {
"frame": "time.Time"
},
"config": {
"interval": 15000
}
},
{
"name": "Value",
"type": "number",
"typeInfo": {
"frame": "float64"
},
"labels": {},
"config": {
"displayNameFromDS": "sum(rate(interface_OutboundErrors{agent_host=\"10.0.0.45\"}[1m0s]))"
}
}
]
},
"data": {
"values": [
[
1751434890000,
1751434905000,
1751434920000,
1751434935000,
1751434950000,
1751434965000,
1751434980000,
1751434995000,
1751435010000,
1751435025000,
1751435040000,
1751435055000,
1751435070000,
1751435085000,
1751435100000,
1751435115000,
1751435130000,
1751435145000,
1751435160000,
1751435175000,
1751435190000,
1751435205000,
1751435220000,
1751435235000,
1751435250000,
1751435265000,
1751435280000,
1751435295000,
1751435310000,
1751435325000,
1751435340000,
1751435355000,
1751435370000,
1751435385000,
1751435400000,
1751435415000,
1751435430000,
1751435445000,
1751435460000,
1751435475000,
1751435490000,
1751435505000,
1751435520000,
1751435535000,
1751435550000,
1751435565000,
1751435580000,
1751435595000,
1751435610000,
1751435625000,
1751435640000,
1751435655000,
1751435670000,
1751435685000,
1751435700000,
1751435715000,
1751435730000,
1751435745000,
1751435760000,
1751435775000,
1751435790000,
1751435805000,
1751435820000,
1751435835000,
1751435850000,
1751435865000,
1751435880000,
1751435895000,
1751435910000,
1751435925000,
1751435940000,
1751435955000,
1751435970000,
1751435985000,
1751436000000,
1751436015000,
1751436030000,
1751436045000,
1751436060000,
1751436075000,
1751436090000,
1751436105000,
1751436120000,
1751436135000,
1751436150000,
1751436165000,
1751436180000,
1751436195000,
1751436210000,
1751436225000,
1751436240000,
1751436255000,
1751436270000,
1751436285000,
1751436300000,
1751436315000,
1751436330000,
1751436345000,
1751436360000,
1751436375000,
1751436390000,
1751436405000,
1751436420000,
1751436435000,
1751436450000,
1751436465000,
1751436480000,
1751436495000,
1751436510000,
1751436525000,
1751436540000,
1751436555000,
1751436570000,
1751436585000,
1751436600000,
1751436615000,
1751436630000,
1751436645000,
1751436660000
],
[
0.13548444444444444,
0.22741999999999998,
0.22222222222222218,
0.22222222222222218,
0.1333333333333333,
0.39999999999999997,
0.39999999999999997,
0.6444444444444444,
0.7999999999999998,
1.0666666666666664,
1.1111111111111112,
0.6666666666666665,
0.39999999999999997,
0.2666666666666666,
0.31111111111111106,
0.39999999999999997,
0.31111111111111106,
0.37777777777777777,
0.28888888888888886,
0.3555555555555555,
0.5333333333333332,
0.6666666666666665,
0.6444444444444444,
0.6444444444444444,
0.5111111111111111,
2.1999999999999997,
2.133333333333333,
2.133333333333333,
0.4888888888888889,
0.39999999999999997,
0.39999999999999997,
0.42222222222222217,
0.3555555555555555,
0.6888888888888888,
0.5777777777777777,
0.6222222222222222,
0.33333333333333326,
0.42222222222222217,
0.39999999999999997,
0.31111111111111106,
0.28888888888888886,
0.28888888888888886,
0.39999999999999997,
0.28888888888888886,
0.33333333333333326,
0.17777777777777776,
0.19999999999999998,
0.15555555555555553,
0.28888888888888886,
0.22222222222222218,
0.31111111111111106,
0.2666666666666666,
0.31111111111111106,
0.3555555555555555,
0.31111111111111106,
0.4444444444444444,
0.3555555555555555,
0.6444444444444444,
0.4666666666666667,
0.5111111111111111,
0.3555555555555555,
0.4444444444444444,
0.42222222222222217,
0.33333333333333326,
0.31111111111111106,
0.4666666666666667,
0.37777777777777777,
0.42222222222222217,
0.15555555555555553,
0.39999999999999997,
0.28888888888888886,
0.5111111111111111,
0.8888888888888888,
1.111111111111111,
0.9777777777777776,
0.39998222301231057,
0.17777777777777776,
0.19997778024663923,
0.20000888928396818,
0.5333333333333332,
0.4889432159128792,
0.5333333333333333,
0.2444444444444444,
0.2666666666666666,
0.24444444444444444,
0.26666666666666666,
0.22222222222222218,
0.33333333333333326,
0.3555555555555555,
0.26666666666666666,
0.17777777777777776,
0.2666666666666666,
0.37777777777777777,
0.6888888888888888,
0.7333333333333333,
0.8222222222222221,
0.4666666666666667,
0.4444444444444444,
0.39999999999999997,
0.4888888888888889,
0.6222222222222221,
0.7555555555555555,
0.7333333333333332,
0.5333333333333332,
0.42222222222222217,
0.4444444444444444,
0.4888888888888888,
0.4666666666666667,
0.33333333333333326,
0.4888888888888889,
0.5555555555555554,
0.8444444444444443,
0.5999999999999999,
0.7777777777777777,
0.4888888888888889,
0.5555555555555554,
0.24444444444444444,
0.39999999999999997,
0.37777777777777777
]
]
}
}
],
"refId": "B"
},
"B-Instant": {
"status": 200,
"frames": [
{
"schema": {
"refId": "B-Instant",
"meta": {
"type": "numeric-multi",
"typeVersion": [
0,
1
],
"custom": {
"resultType": "vector"
},
"notices": [
{
"severity": "warning",
"text": "PromQL info: metric might not be a counter, name does not end in _total/_sum/_count/_bucket: \"interface_OutboundErrors\" (1:10)"
}
],
"executedQueryString": "Expr: sum(rate(interface_OutboundErrors{agent_host=\"10.0.0.45\"}[1m0s]))\nStep: 15s"
},
"fields": [
{
"name": "Time",
"type": "time",
"typeInfo": {
"frame": "time.Time"
},
"config": {
"interval": 15000
}
},
{
"name": "Value",
"type": "number",
"typeInfo": {
"frame": "float64"
},
"labels": {},
"config": {
"displayNameFromDS": "sum(rate(interface_OutboundErrors{agent_host=\"10.0.0.45\"}[1m0s]))"
}
}
]
},
"data": {
"values": [
[
1751436670386
],
[
0.46666666666666656
]
]
}
}
],
"refId": "B-Instant"
}
}
}
}
- Did you receive any errors in the Grafana UI or in related logs? If so, please tell us exactly what they were.
Not that I’m aware of.
- Did you follow any online instructions? If so, what is the URL?
No
