How to automate the creation of alert rules and contact points?

Hi,

I am using Jenkins to automate the creation of an alert rule. I am able to at least gain access via the api however I am getting a Bad Request 400 on my pipeline. I am not sure of how one is one supposed to format the alert rules in json.

This is what I got so far.

pipeline {
    agent any

    environment {
        GRAFANA_TOKEN = credentials('grafana-service-account')
        GRAFANA_URL = "http://sjgrafana.duckdns.org/api/v1/provisioning/alert-rules"
    }

    stages {
        stage('Post Alert Rules') {
            steps {
                script {
                    def alertRulesJson = '''{
                        "apiVersion": "1",
                        "groups": [
                            {
                                "orgId": 1,
                                "name": "VNC_Client_Group",
                                "folder": "VNC_Client",
                                "interval": "1m",
                                "rules": [
                                    {
                                        "uid": "fe9v2mqgegs1sd",
                                        "title": "Cadvisor or ApacheEx down",
                                        "condition": "C",
                                        "data": [
                                            {
                                                "refId": "A",
                                                "relativeTimeRange": {
                                                    "from": 600,
                                                    "to": 0
                                                },
                                                "datasourceUid": "be9v1cz8wgf0ga",
                                                "model": {
                                                    "editorMode": "code",
                                                    "expr": "up{instance=\"172.20.20.3:9117\"}",
                                                    "instant": true,
                                                    "intervalMs": 1000,
                                                    "legendFormat": "__auto",
                                                    "maxDataPoints": 43200,
                                                    "range": false
                                                }
                                            },
                                            {
                                                "refId": "B",
                                                "relativeTimeRange": {
                                                    "from": 600,
                                                    "to": 0
                                                },
                                                "datasourceUid": "be9v1cz8wgf0ga",
                                                "model": {
                                                    "editorMode": "code",
                                                    "expr": "up{instance=\"172.20.20.5:8080\"}",
                                                    "instant": true,
                                                    "intervalMs": 1000,
                                                    "legendFormat": "__auto",
                                                    "maxDataPoints": 43200,
                                                    "range": false
                                                }
                                            },
                                            {
                                                "refId": "C",
                                                "datasourceUid": "__expr__",
                                                "model": {
                                                    "conditions": [
                                                        {
                                                            "evaluator": {
                                                                "params": [0, 0],
                                                                "type": "gt"
                                                            },
                                                            "operator": {
                                                                "type": "and"
                                                            },
                                                            "query": {
                                                                "params": []
                                                            },
                                                            "reducer": {
                                                                "params": [],
                                                                "type": "avg"
                                                            },
                                                            "type": "query"
                                                        }
                                                    ],
                                                    "datasource": {
                                                        "name": "Expression",
                                                        "type": "__expr__",
                                                        "uid": "__expr__"
                                                    },
                                                    "expression": "($A == 0) || ($B == 0)",
                                                    "intervalMs": 1000,
                                                    "maxDataPoints": 43200,
                                                    "refId": "C",
                                                    "type": "math"
                                                }
                                            }
                                        ],
                                        "noDataState": "NoData",
                                        "execErrState": "Error",
                                        "for": "1m",
                                        "isPaused": false,
                                        "notification_settings": {
                                            "receiver": "JenkinsCall"
                                        }
                                    },
                                    {
                                        "uid": "ae9v2qlx6vgn4c",
                                        "title": "Memory Usage Exceeds 90%",
                                        "condition": "C",
                                        "data": [
                                            {
                                                "refId": "A",
                                                "relativeTimeRange": {
                                                    "from": 600,
                                                    "to": 0
                                                },
                                                "datasourceUid": "be9v1cz8wgf0ga",
                                                "model": {
                                                    "editorMode": "code",
                                                    "expr": "(container_memory_usage_bytes{name=~\"clab-labenv-cadvisor|clab-labenv-server|clab-labenv-iol|clab-labenv-vncclient\"} / container_spec_memory_limit_bytes{name=~\"clab-labenv-cadvisor|clab-labenv-server|clab-labenv-iol|clab-labenv-vncclient\"}) * 100",
                                                    "instant": true,
                                                    "intervalMs": 1000,
                                                    "legendFormat": "__auto",
                                                    "maxDataPoints": 43200,
                                                    "range": false
                                                }
                                            },
                                            {
                                                "refId": "C",
                                                "datasourceUid": "__expr__",
                                                "model": {
                                                    "conditions": [
                                                        {
                                                            "evaluator": {
                                                                "params": [90],
                                                                "type": "gt"
                                                            },
                                                            "operator": {
                                                                "type": "and"
                                                            },
                                                            "query": {
                                                                "params": ["C"]
                                                            },
                                                            "reducer": {
                                                                "params": [],
                                                                "type": "last"
                                                            },
                                                            "type": "query"
                                                        }
                                                    ],
                                                    "datasource": {
                                                        "type": "__expr__",
                                                        "uid": "__expr__"
                                                    },
                                                    "expression": "A",
                                                    "intervalMs": 1000,
                                                    "maxDataPoints": 43200,
                                                    "refId": "C",
                                                    "type": "threshold"
                                                }
                                            }
                                        ],
                                        "noDataState": "NoData",
                                        "execErrState": "Error",
                                        "for": "1m",
                                        "isPaused": false,
                                        "notification_settings": {
                                            "receiver": "grafana-default-email"
                                        }
                                    }
                                ]
                            }
                        ]
                    }'''

                    sh returnStdout: true, script: """
                    curl -v -X POST ${GRAFANA_URL} \
                        -H "Content-Type: application/json" \
                        -H "Authorization: Bearer ${GRAFANA_TOKEN}" \
                        -d '${alertRulesJson}'
                    """
                }
            }
        }
    }
}

This is the exported alert rule that is in yaml:

apiVersion: 1
groups:
    - orgId: 1
      name: VNC_Client_Group
      folder: VNC_Client
      interval: 1m
      rules:
        - uid: fe9v2mqgegs1sd
          title: Cadvisor or ApacheEx down
          condition: C
          data:
            - refId: A
              relativeTimeRange:
                from: 600
                to: 0
              datasourceUid: be9v1cz8wgf0ga
              model:
                editorMode: code
                expr: up{instance="172.20.20.3:9117"}
                instant: true
                intervalMs: 1000
                legendFormat: __auto
                maxDataPoints: 43200
                range: false
                refId: A
            - refId: B
              relativeTimeRange:
                from: 600
                to: 0
              datasourceUid: be9v1cz8wgf0ga
              model:
                datasource:
                    type: prometheus
                    uid: be9v1cz8wgf0ga
                editorMode: code
                expr: up{instance="172.20.20.5:8080"}
                hide: false
                instant: true
                intervalMs: 1000
                legendFormat: __auto
                maxDataPoints: 43200
                range: false
                refId: B
            - refId: C
              datasourceUid: __expr__
              model:
                conditions:
                    - evaluator:
                        params:
                            - 0
                            - 0
                        type: gt
                      operator:
                        type: and
                      query:
                        params: []
                      reducer:
                        params: []
                        type: avg
                      type: query
                datasource:
                    name: Expression
                    type: __expr__
                    uid: __expr__
                expression: ($A == 0) || ($B == 0)
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                refId: C
                type: math
          noDataState: NoData
          execErrState: Error
          for: 1m
          isPaused: false
          notification_settings:
            receiver: JenkinsCall
        - uid: ae9v2qlx6vgn4c
          title: Memory Usage Exceeds 90%
          condition: C
          data:
            - refId: A
              relativeTimeRange:
                from: 600
                to: 0
              datasourceUid: be9v1cz8wgf0ga
              model:
                editorMode: code
                expr: (container_memory_usage_bytes{name=~"clab-labenv-cadvisor|clab-labenv-server|clab-labenv-iol|clab-labenv-vncclient"} / container_spec_memory_limit_bytes{name=~"clab-labenv-cadvisor|clab-labenv-server|clab-labenv-iol|clab-labenv-vncclient"}) * 100
                instant: true
                intervalMs: 1000
                legendFormat: __auto
                maxDataPoints: 43200
                range: false
                refId: A
            - refId: C
              datasourceUid: __expr__
              model:
                conditions:
                    - evaluator:
                        params:
                            - 90
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                            - C
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: __expr__
                expression: A
                intervalMs: 1000
                maxDataPoints: 43200
                refId: C
                type: threshold
          noDataState: NoData
          execErrState: Error
          for: 1m
          isPaused: false
          notification_settings:
            receiver: grafana-default-email

Hi, I think that you need a different request body. Following the path from the docs you’ll notice that you’ll need to use ProvisionedAlertRule body (also remember about the header if you want to update the alerts).