Added PermanentError type, IsPermanent() helper. Permanent errors are logged and not retried

backend/internal/notifications/discord.go

@@ -101,8 +101,12 @@ func SendDiscordNotification(webhookURL, message string, meta AlertMetadata) (bo
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		err := fmt.Errorf("discord webhook failed: HTTP %d", resp.StatusCode)
 		log.Printf("Discord webhook failed: HTTP %d", resp.StatusCode)
-		return false, fmt.Errorf("discord webhook failed: HTTP %d", resp.StatusCode)
+		if isPermanentStatusCode(resp.StatusCode) {
+			return false, &PermanentError{Err: err}
+		}
+		return false, err
 	}
 
 	return true, nil

backend/internal/notifications/email.go

@@ -97,8 +97,12 @@ func SendEmailNotification(apiKey, fromAddress, toAddress, message string, meta
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		err := fmt.Errorf("resend API failed: HTTP %d", resp.StatusCode)
 		log.Printf("Resend API failed: HTTP %d", resp.StatusCode)
-		return false, fmt.Errorf("resend API failed: HTTP %d", resp.StatusCode)
+		if isPermanentStatusCode(resp.StatusCode) {
+			return false, &PermanentError{Err: err}
+		}
+		return false, err
 	}
 
 	return true, nil

backend/internal/notifications/notifier.go

@@ -1,6 +1,10 @@
 package notifications
 
-import "context"
+import (
+	"context"
+	"errors"
+	"net/http"
+)
 
 // AlertMetadata holds context about the alert being sent.
 type AlertMetadata struct {
@@ -13,3 +17,22 @@ type AlertMetadata struct {
 type Notifier interface {
 	Send(ctx context.Context, message string, meta AlertMetadata) error
 }
+
+// PermanentError wraps errors that should not be retried (e.g. 401, 403, 404).
+type PermanentError struct{ Err error }
+
+func (e *PermanentError) Error() string { return e.Err.Error() }
+func (e *PermanentError) Unwrap() error { return e.Err }
+
+func IsPermanent(err error) bool {
+	var p *PermanentError
+	return errors.As(err, &p)
+}
+
+func isPermanentStatusCode(code int) bool {
+	return code == http.StatusUnauthorized ||
+		code == http.StatusForbidden ||
+		code == http.StatusNotFound ||
+		code == http.StatusMethodNotAllowed ||
+		code == http.StatusGone
+}

backend/internal/notifications/slack.go

@@ -87,8 +87,12 @@ func SendSlackNotification(webhookURL, message string, meta AlertMetadata) (bool
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		err := fmt.Errorf("slack webhook failed: HTTP %d", resp.StatusCode)
 		log.Printf("Slack webhook failed: HTTP %d", resp.StatusCode)
-		return false, fmt.Errorf("slack webhook failed: HTTP %d", resp.StatusCode)
+		if isPermanentStatusCode(resp.StatusCode) {
+			return false, &PermanentError{Err: err}
+		}
+		return false, err
 	}
 
 	return true, nil

backend/internal/notifications/telegram.go

@@ -63,8 +63,12 @@ func SendTelegramNotification(botToken, chatID, message string, meta AlertMetada
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		err := fmt.Errorf("telegram API failed: HTTP %d", resp.StatusCode)
 		log.Printf("Telegram API failed: HTTP %d", resp.StatusCode)
-		return false, fmt.Errorf("telegram API failed: HTTP %d", resp.StatusCode)
+		if isPermanentStatusCode(resp.StatusCode) {
+			return false, &PermanentError{Err: err}
+		}
+		return false, err
 	}
 
 	return true, nil

backend/internal/services/evaluator.go

@@ -261,8 +261,14 @@ func sendWithRetry(ctx context.Context, n notifications.Notifier, message string
 		}
 
 		if err := n.Send(ctx, message, meta); err != nil {
-			log.Printf("Notification attempt %d/%d failed: %v", attempt+1, notificationMaxRetries, err)
 			lastErr = err
+
+			if notifications.IsPermanent(err) {
+				log.Printf("Permanent notification failure, skipping retries: %v", err)
+				return err
+			}
+
+			log.Printf("Notification attempt %d/%d failed: %v", attempt+1, notificationMaxRetries, err)
 			continue
 		}