From a7cdb0055092d5a2ef9e6833a1e12fedd8ef2069 Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Thu, 25 Jan 2024 00:44:46 +0700 Subject: [PATCH] refactor!: move alert setup from Grafana to Alertmanager --- external/modules/ntfy/main.tf | 9 +++-- external/modules/ntfy/variables.tf | 5 ++- external/namespaces.yml | 2 +- external/terraform.tfvars.example | 10 +++--- external/variables.tf | 5 ++- .../files/secret-generator/config.yaml | 7 ---- platform/grafana/templates/secret.yaml | 4 --- platform/grafana/values.yaml | 18 ---------- platform/ntfy-relay/Chart.yaml | 7 ---- platform/ntfy-relay/templates/secret.yaml | 26 -------------- platform/ntfy-relay/values.yaml | 19 ---------- .../alertmanager-to-ntfy.jsonnet | 8 +++++ .../templates/configmap.yaml | 7 ++++ system/monitoring-system/values.yaml | 36 +++++++++++++++++++ 14 files changed, 64 insertions(+), 99 deletions(-) delete mode 100644 platform/ntfy-relay/Chart.yaml delete mode 100644 platform/ntfy-relay/templates/secret.yaml delete mode 100644 platform/ntfy-relay/values.yaml create mode 100644 system/monitoring-system/files/webhook-transformer/alertmanager-to-ntfy.jsonnet create mode 100644 system/monitoring-system/templates/configmap.yaml diff --git a/external/modules/ntfy/main.tf b/external/modules/ntfy/main.tf index 88d8b238..8dc73f51 100644 --- a/external/modules/ntfy/main.tf +++ b/external/modules/ntfy/main.tf @@ -1,7 +1,7 @@ resource "kubernetes_secret" "ntfy_auth" { metadata { - name = "ntfy.auth" - namespace = "global-secrets" + name = "webhook-transformer" + namespace = "monitoring-system" annotations = { "app.kubernetes.io/managed-by" = "Terraform" @@ -9,8 +9,7 @@ resource "kubernetes_secret" "ntfy_auth" { } data = { - url = var.auth.url - username = var.auth.username - password = var.auth.password + NTFY_URL = var.auth.url + NTFY_TOPIC = var.auth.topic } } diff --git a/external/modules/ntfy/variables.tf b/external/modules/ntfy/variables.tf index 1f43c4eb..803d9e6d 100644 --- a/external/modules/ntfy/variables.tf +++ b/external/modules/ntfy/variables.tf @@ -1,7 +1,6 @@ variable "auth" { type = object({ - url = string - username = string - password = string + url = string + topic = string }) } diff --git a/external/namespaces.yml b/external/namespaces.yml index 7e1767c2..32cb75db 100644 --- a/external/namespaces.yml +++ b/external/namespaces.yml @@ -10,6 +10,6 @@ - cert-manager - cloudflared - external-dns - - global-secrets - k8up-operator + - monitoring-system - zerotier diff --git a/external/terraform.tfvars.example b/external/terraform.tfvars.example index 553fe87f..a7950cbc 100644 --- a/external/terraform.tfvars.example +++ b/external/terraform.tfvars.example @@ -9,10 +9,8 @@ cloudflare_api_key = "foobarkey" zerotier_central_token = "foobartoken" ntfy = { - # https://ntfy.sh/app or your own instance - url = "https://ntfy.sh/random_topic_name_here_a8sd7fkjxlkcjasdw33813" - # Optional, required if the ntfy instance has access control enabled - username = "" - # Optional, required if the ntfy instance has access control enabled - password = "" + # https://ntfy.sh or your own instance + url = "https://ntfy.sh" + # Your topic name + topic = "random_topic_name_here_a8sd7fkjxlkcjasdw33813" } diff --git a/external/variables.tf b/external/variables.tf index 147df0ff..0a169e50 100644 --- a/external/variables.tf +++ b/external/variables.tf @@ -17,9 +17,8 @@ variable "zerotier_central_token" { variable "ntfy" { type = object({ - url = string - username = string - password = string + url = string + topic = string }) sensitive = true diff --git a/platform/global-secrets/files/secret-generator/config.yaml b/platform/global-secrets/files/secret-generator/config.yaml index 53c7773d..8bf673b8 100644 --- a/platform/global-secrets/files/secret-generator/config.yaml +++ b/platform/global-secrets/files/secret-generator/config.yaml @@ -37,10 +37,3 @@ - key: PAPERLESS_ADMIN_PASSWORD length: 32 special: true - -# ntfy -- name: ntfy-relay.auth - data: - - key: password - length: 32 - special: true diff --git a/platform/grafana/templates/secret.yaml b/platform/grafana/templates/secret.yaml index ad9276e1..701c2305 100644 --- a/platform/grafana/templates/secret.yaml +++ b/platform/grafana/templates/secret.yaml @@ -14,7 +14,3 @@ spec: remoteRef: key: dex.grafana property: client_secret - - secretKey: NTFY_RELAY_PASSWORD - remoteRef: - key: ntfy-relay.auth - property: password diff --git a/platform/grafana/values.yaml b/platform/grafana/values.yaml index 7202ee1c..474736e0 100644 --- a/platform/grafana/values.yaml +++ b/platform/grafana/values.yaml @@ -31,21 +31,3 @@ grafana: auth_url: https://dex.khuedoan.com/auth token_url: https://dex.khuedoan.com/token api_url: https://dex.khuedoan.com/userinfo - alerting: - contactpoints.yaml: - secret: - contactPoints: - - name: ntfy - receivers: - - uid: ntfy-relay - type: webhook - settings: - url: http://ntfy-relay.ntfy-relay - username: admin - password: $__env{NTFY_RELAY_PASSWORD} - policies.yaml: - policies: - - receiver: ntfy - group_by: - - grafana_folder - - alertname diff --git a/platform/ntfy-relay/Chart.yaml b/platform/ntfy-relay/Chart.yaml deleted file mode 100644 index cadfd9da..00000000 --- a/platform/ntfy-relay/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v2 -name: ntfy-relay -version: 0.0.0 -dependencies: - - name: app-template - version: 2.5.0 - repository: https://bjw-s.github.io/helm-charts diff --git a/platform/ntfy-relay/templates/secret.yaml b/platform/ntfy-relay/templates/secret.yaml deleted file mode 100644 index cca3eca0..00000000 --- a/platform/ntfy-relay/templates/secret.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: {{ .Release.Name }}-secret - namespace: {{ .Release.Namespace }} -spec: - secretStoreRef: - kind: ClusterSecretStore - name: global-secrets - data: - - secretKey: NTFY_URL - remoteRef: - key: ntfy.auth - property: url - - secretKey: NTFY_BAUTH_USER - remoteRef: - key: ntfy.auth - property: username - - secretKey: NTFY_BAUTH_PASS - remoteRef: - key: ntfy.auth - property: password - - secretKey: BAUTH_PASS - remoteRef: - key: ntfy-relay.auth - property: password diff --git a/platform/ntfy-relay/values.yaml b/platform/ntfy-relay/values.yaml deleted file mode 100644 index 7dba782f..00000000 --- a/platform/ntfy-relay/values.yaml +++ /dev/null @@ -1,19 +0,0 @@ -app-template: - controllers: - main: - containers: - main: - image: - repository: docker.io/kittyandrew/grafana-to-ntfy - tag: latest - env: - BAUTH_USER: admin - envFrom: - - secret: "{{ .Release.Name }}-secret" - service: - main: - ports: - http: - port: 80 - targetPort: 8080 - protocol: HTTP diff --git a/system/monitoring-system/files/webhook-transformer/alertmanager-to-ntfy.jsonnet b/system/monitoring-system/files/webhook-transformer/alertmanager-to-ntfy.jsonnet new file mode 100644 index 00000000..4b74b08c --- /dev/null +++ b/system/monitoring-system/files/webhook-transformer/alertmanager-to-ntfy.jsonnet @@ -0,0 +1,8 @@ +{ + "topic": env.NTFY_TOPIC, + "title": body.alerts[0].labels.alertname, // TODO support multiple alerts + "message": body.alerts[0].annotations.description, + "tags": [], + "priority": 3, + "actions": [] +} diff --git a/system/monitoring-system/templates/configmap.yaml b/system/monitoring-system/templates/configmap.yaml new file mode 100644 index 00000000..bcb8258a --- /dev/null +++ b/system/monitoring-system/templates/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: webhook-transformer + namespace: {{ .Release.Namespace }} +data: +{{ (.Files.Glob "files/webhook-transformer/*").AsConfig | indent 2 }} diff --git a/system/monitoring-system/values.yaml b/system/monitoring-system/values.yaml index 863d5596..ba7b7893 100644 --- a/system/monitoring-system/values.yaml +++ b/system/monitoring-system/values.yaml @@ -13,3 +13,39 @@ kube-prometheus-stack: serviceMonitorSelectorNilUsesHelmValues: false podMonitorSelectorNilUsesHelmValues: false probeSelectorNilUsesHelmValues: false + alertmanager: + alertmanagerSpec: + containers: + - name: ntfy-relay + image: ghcr.io/khuedoan/webhook-transformer:v0.0.3 + args: + - --port=8081 + - --config=/config/alertmanager-to-ntfy.jsonnet + - --upstream-host=https://ntfy.sh + envFrom: + - secretRef: + name: webhook-transformer + volumeMounts: + - name: config + mountPath: /config + volumes: + - name: config + configMap: + name: webhook-transformer + config: + route: + receiver: ntfy + group_by: + - namespace + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + routes: + - receiver: ntfy + matchers: + - alertname = "Watchdog" + receivers: + - name: ntfy + webhook_configs: + - url: http://localhost:8081 + send_resolved: true