refactor!: move alert setup from Grafana to Alertmanager

external/modules/ntfy/main.tf

@@ -1,7 +1,7 @@
 resource "kubernetes_secret" "ntfy_auth" {
   metadata {
-    name      = "ntfy.auth"
-    namespace = "global-secrets"
+    name      = "webhook-transformer"
+    namespace = "monitoring-system"
 
     annotations = {
       "app.kubernetes.io/managed-by" = "Terraform"
@@ -9,8 +9,7 @@ resource "kubernetes_secret" "ntfy_auth" {
   }
 
   data = {
-    url      = var.auth.url
-    username = var.auth.username
-    password = var.auth.password
+    NTFY_URL   = var.auth.url
+    NTFY_TOPIC = var.auth.topic
   }
 }

external/modules/ntfy/variables.tf

@@ -1,7 +1,6 @@
 variable "auth" {
   type = object({
-    url      = string
-    username = string
-    password = string
+    url   = string
+    topic = string
   })
 }

external/namespaces.yml

@@ -10,6 +10,6 @@
         - cert-manager
         - cloudflared
         - external-dns
-        - global-secrets
         - k8up-operator
+        - monitoring-system
         - zerotier

external/terraform.tfvars.example

@@ -9,10 +9,8 @@ cloudflare_api_key = "foobarkey"
 zerotier_central_token = "foobartoken"
 
 ntfy = {
-  # https://ntfy.sh/app or your own instance
-  url = "https://ntfy.sh/random_topic_name_here_a8sd7fkjxlkcjasdw33813"
-  # Optional, required if the ntfy instance has access control enabled
-  username = ""
-  # Optional, required if the ntfy instance has access control enabled
-  password = ""
+  # https://ntfy.sh or your own instance
+  url = "https://ntfy.sh"
+  # Your topic name
+  topic = "random_topic_name_here_a8sd7fkjxlkcjasdw33813"
 }

external/variables.tf

@@ -17,9 +17,8 @@ variable "zerotier_central_token" {
 
 variable "ntfy" {
   type = object({
-    url      = string
-    username = string
-    password = string
+    url   = string
+    topic = string
   })
 
   sensitive = true

platform/global-secrets/files/secret-generator/config.yaml

@@ -37,10 +37,3 @@
     - key: PAPERLESS_ADMIN_PASSWORD
       length: 32
       special: true
-
-# ntfy
-- name: ntfy-relay.auth
-  data:
-    - key: password
-      length: 32
-      special: true

platform/grafana/templates/secret.yaml

@@ -14,7 +14,3 @@ spec:
       remoteRef:
         key: dex.grafana
         property: client_secret
-    - secretKey: NTFY_RELAY_PASSWORD
-      remoteRef:
-        key: ntfy-relay.auth
-        property: password

platform/grafana/values.yaml

@@ -31,21 +31,3 @@ grafana:
       auth_url: https://dex.khuedoan.com/auth
       token_url: https://dex.khuedoan.com/token
       api_url: https://dex.khuedoan.com/userinfo
-  alerting:
-    contactpoints.yaml:
-      secret:
-        contactPoints:
-          - name: ntfy
-            receivers:
-              - uid: ntfy-relay
-                type: webhook
-                settings:
-                  url: http://ntfy-relay.ntfy-relay
-                  username: admin
-                  password: $__env{NTFY_RELAY_PASSWORD}
-    policies.yaml:
-      policies:
-        - receiver: ntfy
-          group_by:
-            - grafana_folder
-            - alertname

platform/ntfy-relay/Chart.yaml

@@ -1,7 +0,0 @@
-apiVersion: v2
-name: ntfy-relay
-version: 0.0.0
-dependencies:
-  - name: app-template
-    version: 2.5.0
-    repository: https://bjw-s.github.io/helm-charts

platform/ntfy-relay/templates/secret.yaml

@@ -1,26 +0,0 @@
-apiVersion: external-secrets.io/v1beta1
-kind: ExternalSecret
-metadata:
-  name: {{ .Release.Name }}-secret
-  namespace: {{ .Release.Namespace }}
-spec:
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: global-secrets
-  data:
-    - secretKey: NTFY_URL
-      remoteRef:
-        key: ntfy.auth
-        property: url
-    - secretKey: NTFY_BAUTH_USER
-      remoteRef:
-        key: ntfy.auth
-        property: username
-    - secretKey: NTFY_BAUTH_PASS
-      remoteRef:
-        key: ntfy.auth
-        property: password
-    - secretKey: BAUTH_PASS
-      remoteRef:
-        key: ntfy-relay.auth
-        property: password

platform/ntfy-relay/values.yaml

@@ -1,19 +0,0 @@
-app-template:
-  controllers:
-    main:
-      containers:
-        main:
-          image:
-            repository: docker.io/kittyandrew/grafana-to-ntfy
-            tag: latest
-          env:
-            BAUTH_USER: admin
-          envFrom:
-            - secret: "{{ .Release.Name }}-secret"
-  service:
-    main:
-      ports:
-        http:
-          port: 80
-          targetPort: 8080
-          protocol: HTTP

system/monitoring-system/files/webhook-transformer/alertmanager-to-ntfy.jsonnet

@@ -0,0 +1,8 @@
+{
+  "topic": env.NTFY_TOPIC,
+  "title": body.alerts[0].labels.alertname, // TODO support multiple alerts
+  "message": body.alerts[0].annotations.description,
+  "tags": [],
+  "priority": 3,
+  "actions": []
+}

system/monitoring-system/templates/configmap.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: webhook-transformer
+  namespace: {{ .Release.Namespace }}
+data:
+{{ (.Files.Glob "files/webhook-transformer/*").AsConfig | indent 2 }}

system/monitoring-system/values.yaml

@@ -13,3 +13,39 @@ kube-prometheus-stack:
       serviceMonitorSelectorNilUsesHelmValues: false
       podMonitorSelectorNilUsesHelmValues: false
       probeSelectorNilUsesHelmValues: false
+  alertmanager:
+    alertmanagerSpec:
+      containers:
+        - name: ntfy-relay
+          image: ghcr.io/khuedoan/webhook-transformer:v0.0.3
+          args:
+            - --port=8081
+            - --config=/config/alertmanager-to-ntfy.jsonnet
+            - --upstream-host=https://ntfy.sh
+          envFrom:
+            - secretRef:
+                name: webhook-transformer
+          volumeMounts:
+            - name: config
+              mountPath: /config
+      volumes:
+        - name: config
+          configMap:
+            name: webhook-transformer
+    config:
+      route:
+        receiver: ntfy
+        group_by:
+          - namespace
+        group_wait: 30s
+        group_interval: 5m
+        repeat_interval: 12h
+        routes:
+          - receiver: ntfy
+            matchers:
+              - alertname = "Watchdog"
+      receivers:
+        - name: ntfy
+          webhook_configs:
+            - url: http://localhost:8081
+              send_resolved: true