Add two SNMPinterfacedown alerts with different severities

snmp-observ-lib/alerts.libsonnet

@@ -156,18 +156,42 @@ local xtd = import 'github.com/jsonnet-libs/xtd/main.libsonnet';
                       and (%s) != 2
                     |||
                     % [
-                      this.signals.interface.ifOperStatus.withFilteringSelectorMixin(this.config.alertInterfaceDownSelector).asRuleExpression(),
+                      this.signals.interface.ifOperStatus.withFilteringSelectorMixin(this.config.alertInterfaceDownSelectorCritical).asRuleExpression(),
                       this.signals.interface.ifAdminStatus.asRuleExpression(),
                     ],
               labels: {
-                severity: this.config.alertInterfaceDownSeverity,
+                severity: 'critical',
+              },
+              annotations: {
+                summary: 'Critical network interface is down on SNMP device.',
+                description: |||
+                  A critical network interface {{$labels.ifName}} ({{$labels.ifAlias}}) on {{$labels.%s}} is down. 
+                  Note that only interfaces with ifAdminStatus = `up` and matching `%s` are being checked and considered critical.
+                ||| % [instanceLabel, this.config.alertInterfaceDownSelectorCritical],
+              },
+              'for': '5m',
+              keep_firing_for: '5m',
+            },
+            {
+              alert: 'SNMPInterfaceDown',
+              expr: |||
+                      (%s) == 2
+                      # only alert if interface is adminatratively up:
+                      and (%s) != 2
+                    |||
+                    % [
+                      this.signals.interface.ifOperStatus.withFilteringSelectorMixin(this.config.alertInterfaceDownSelectorWarning).asRuleExpression(),
+                      this.signals.interface.ifAdminStatus.asRuleExpression(),
+                    ],
+              labels: {
+                severity: 'warning',
               },
               annotations: {
                 summary: 'Network interface is down on SNMP device.',
                 description: |||
                   Network interface {{$labels.ifName}} ({{$labels.ifAlias}}) on {{$labels.%s}} is down. 
                   Only interfaces with ifAdminStatus = `up` and matching `%s` are being checked.
-                ||| % [instanceLabel, this.config.alertInterfaceDownSelector],
+                ||| % [instanceLabel, this.config.alertInterfaceDownSelectorWarning],
               },
               'for': '5m',
               keep_firing_for: '5m',
@@ -239,7 +263,6 @@ local xtd = import 'github.com/jsonnet-libs/xtd/main.libsonnet';
                   Network interface {{ $labels.ifName }} ({{$labels.ifAlias}}) is flapping on {{ $labels.%s }}. It has changed its status more than 5 times in the last 5 minutes.
                 ||| % [instanceLabel],
               },
-              'for': '0',
               keep_firing_for: '5m',
             },
           ],

snmp-observ-lib/config.libsonnet

@@ -15,9 +15,12 @@
   // pick vendors you have. See README for full list.
   metricsSource: ['generic', 'cisco', 'mikrotik', 'juniper'],
 
-  //only fire alerts 'interface is down' for the following selector:
-  alertInterfaceDownSelector: 'ifAlias=~".*(?i:(uplink|internet|WAN)|ISP).*"',
-  alertInterfaceDownSeverity: 'warning',
+  // only fire alerts 'interface is down' for the following selector:
+  // deprecated, use alertInterfaceDownSelectorWarning, alertInterfaceDownSelectorCritical, alertInterfaceDownSelectorInfo instead
+  alertInterfaceDownSelector: self.alertInterfaceDownSelectorWarning,
+  alertInterfaceDownSelectorWarning: 'ifAlias=~".*(?i:(uplink|internet|WAN|ISP)).*"',
+  // change according to your org naming conventions
+  alertInterfaceDownSelectorCritical: 'ifAlias=~".*(?i:(critical)).*"',
 
   // Enable to workaround issue with counters on Cisco NX-OS overloading issue.
   // This will clamp max interface traffic possible to 1000Gbps.