I can see two options:
- One that I mentioned above. Ship Grafana logs to Loki (or other place that is supported by Grafana). Then Create an alert rule that queries the data source for logs
Notify for alerts failed
and send it to an email.
- Use infinity data source, setup to query Grafana API. Here is an example rule that I managed to run in my environment.
{
"apiVersion": 1,
"groups": [
{
"orgId": 1,
"name": "Default",
"folder": "Grafana Alerting",
"interval": "1m",
"rules": [
{
"uid": "eeazviqm7uwowd",
"title": "Failing Integrations",
"condition": "A",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 600,
"to": 0
},
"datasourceUid": "ceazv0jwh9ts0b",
"model": {
"columns": [],
"computed_columns": [
{
"selector": "1",
"text": "value",
"type": "string"
}
],
"datasource": {
"type": "yesoreyeram-infinity-datasource",
"uid": "ceazv0jwh9ts0b"
},
"filterExpression": "lastNotifyAttemptError != \"\"",
"filters": [],
"format": "table",
"global_query_id": "",
"instant": true,
"intervalMs": 1000,
"maxDataPoints": 43200,
"parser": "backend",
"refId": "A",
"root_selector": "(\n $map($, function($item) {\n $merge([\n $map($item.integrations, function($integration) {\n $merge([\n {\n \"name\": $item.name,\n \"integrationName\": $integration.name,\n \"lastNotifyAttemptError\": $integration.lastNotifyAttemptError\n }\n ])\n })\n ])\n })\n)",
"source": "url",
"type": "json",
"url": "http://localhost:3000/api/alertmanager/grafana/config/api/v1/receivers",
"url_options": {
"data": "",
"method": "GET"
}
}
}
],
"noDataState": "NoData",
"execErrState": "Error",
"annotations": {},
"labels": {},
"isPaused": false,
"notification_settings": {
"receiver": "Black hole"
}
}
]
}
]
}
Make sure you change the URL to point to public Grafana URL