telecom-platform/apps/api-server/internal/monitoring/alerting.go

package monitoring

import (
	"context"
	"fmt"
	"log"
	"maps"
	"sync"
	"time"

	"github.com/nutcas3/telecom-platform/apps/api-server/internal/websocket"
)

// AlertSeverity represents the severity level of an alert
type AlertSeverity string

const (
	SeverityInfo     AlertSeverity = "info"
	SeverityWarning  AlertSeverity = "warning"
	SeverityCritical AlertSeverity = "critical"
)

// AlertStatus represents the status of an alert
type AlertStatus string

const (
	StatusFiring   AlertStatus = "firing"
	StatusResolved AlertStatus = "resolved"
	StatusSilenced AlertStatus = "silenced"
)

// Alert represents a monitoring alert
type Alert struct {
	ID            string            `json:"id"`
	Name          string            `json:"name"`
	Description   string            `json:"description"`
	Severity      AlertSeverity     `json:"severity"`
	Status        AlertStatus       `json:"status"`
	Timestamp     time.Time         `json:"timestamp"`
	Labels        map[string]string `json:"labels"`
	Annotations   map[string]string `json:"annotations"`
	LastEvaluated time.Time         `json:"last_evaluated"`
}

// AlertRule defines a rule for generating alerts
type AlertRule struct {
	ID          string            `json:"id"`
	Name        string            `json:"name"`
	Description string            `json:"description"`
	Severity    AlertSeverity     `json:"severity"`
	Query       string            `json:"query"`
	Threshold   float64           `json:"threshold"`
	Duration    time.Duration     `json:"duration"`
	Labels      map[string]string `json:"labels"`
	Enabled     bool              `json:"enabled"`
}

// AlertManager manages alerts and alert rules
type AlertManager struct {
	rules     map[string]*AlertRule
	alerts    map[string]*Alert
	mutex     sync.RWMutex
	evaluator *AlertEvaluator
	notifier  AlertNotifier
}

// AlertEvaluator evaluates alert rules
type AlertEvaluator struct {
	healthMonitor *HealthMonitor
}

// AlertNotifier sends alert notifications
type AlertNotifier interface {
	SendAlert(alert *Alert) error
	ResolveAlert(alertID string) error
}

// WebSocketNotifier sends alerts via WebSocket
type WebSocketNotifier struct{}

func (wsn *WebSocketNotifier) SendAlert(alert *Alert) error {
	// Broadcast alert via WebSocket
	websocket.BroadcastAlertUpdate(alert.ID, string(alert.Severity), alert.Description, string(alert.Status))
	return nil
}

func (wsn *WebSocketNotifier) ResolveAlert(alertID string) error {
	// Broadcast alert resolution via WebSocket
	websocket.BroadcastAlertUpdate(alertID, "info", "Alert resolved", "resolved")
	return nil
}

// LogNotifier sends alerts to logs
type LogNotifier struct{}

func (ln *LogNotifier) SendAlert(alert *Alert) error {
	log.Printf("ALERT [%s] %s: %s", alert.Severity, alert.Name, alert.Description)
	return nil
}

func (ln *LogNotifier) ResolveAlert(alertID string) error {
	log.Printf("ALERT RESOLVED: %s", alertID)
	return nil
}

// CompositeNotifier sends alerts via multiple channels
type CompositeNotifier struct {
	notifiers []AlertNotifier
}

func (cn *CompositeNotifier) SendAlert(alert *Alert) error {
	for _, notifier := range cn.notifiers {
		if err := notifier.SendAlert(alert); err != nil {
			log.Printf("Failed to send alert via notifier: %v", err)
		}
	}
	return nil
}

func (cn *CompositeNotifier) ResolveAlert(alertID string) error {
	for _, notifier := range cn.notifiers {
		if err := notifier.ResolveAlert(alertID); err != nil {
			log.Printf("Failed to resolve alert via notifier: %v", err)
		}
	}
	return nil
}

// NewAlertManager creates a new alert manager
func NewAlertManager(healthMonitor *HealthMonitor) *AlertManager {
	evaluator := &AlertEvaluator{healthMonitor: healthMonitor}

	// Create composite notifier with WebSocket and log notifiers
	notifier := &CompositeNotifier{
		notifiers: []AlertNotifier{
			&WebSocketNotifier{},
			&LogNotifier{},
		},
	}

	return &AlertManager{
		rules:     make(map[string]*AlertRule),
		alerts:    make(map[string]*Alert),
		evaluator: evaluator,
		notifier:  notifier,
	}
}

// AddRule adds an alert rule
func (am *AlertManager) AddRule(rule *AlertRule) {
	am.mutex.Lock()
	defer am.mutex.Unlock()
	am.rules[rule.ID] = rule
}

// RemoveRule removes an alert rule
func (am *AlertManager) RemoveRule(ruleID string) {
	am.mutex.Lock()
	defer am.mutex.Unlock()
	delete(am.rules, ruleID)

	// Also remove any active alerts for this rule
	delete(am.alerts, ruleID)
}

// GetRules returns all alert rules
func (am *AlertManager) GetRules() map[string]*AlertRule {
	am.mutex.RLock()
	defer am.mutex.RUnlock()

	rules := make(map[string]*AlertRule)
	maps.Copy(rules, am.rules)
	return rules
}

// GetAlerts returns all active alerts
func (am *AlertManager) GetAlerts() map[string]*Alert {
	am.mutex.RLock()
	defer am.mutex.RUnlock()

	alerts := make(map[string]*Alert)
	maps.Copy(alerts, am.alerts)
	return alerts
}

// EvaluateRules evaluates all alert rules
func (am *AlertManager) EvaluateRules(ctx context.Context) {
	am.mutex.RLock()
	rules := make(map[string]*AlertRule)
	maps.Copy(rules, am.rules)
	am.mutex.RUnlock()

	for _, rule := range rules {
		if !rule.Enabled {
			continue
		}

		shouldAlert, value, err := am.evaluator.EvaluateRule(ctx, rule)
		if err != nil {
			log.Printf("Error evaluating rule %s: %v", rule.ID, err)
			continue
		}

		am.handleRuleEvaluation(rule, shouldAlert, value)
	}
}

// handleRuleEvaluation handles the result of a rule evaluation
func (am *AlertManager) handleRuleEvaluation(rule *AlertRule, shouldAlert bool, value float64) {
	am.mutex.Lock()
	defer am.mutex.Unlock()

	alert, exists := am.alerts[rule.ID]

	if shouldAlert && (!exists || alert.Status == StatusResolved) {
		// Create or update alert
		newAlert := &Alert{
			ID:            rule.ID,
			Name:          rule.Name,
			Description:   fmt.Sprintf("%s (current: %.2f, threshold: %.2f)", rule.Description, value, rule.Threshold),
			Severity:      rule.Severity,
			Status:        StatusFiring,
			Timestamp:     time.Now(),
			LastEvaluated: time.Now(),
			Labels:        rule.Labels,
			Annotations: map[string]string{
				"threshold": fmt.Sprintf("%.2f", rule.Threshold),
				"value":     fmt.Sprintf("%.2f", value),
				"query":     rule.Query,
			},
		}

		am.alerts[rule.ID] = newAlert

		// Send notification
		if err := am.notifier.SendAlert(newAlert); err != nil {
			log.Printf("Failed to send alert notification: %v", err)
		}

	} else if !shouldAlert && exists && alert.Status == StatusFiring {
		// Resolve alert
		alert.Status = StatusResolved
		alert.LastEvaluated = time.Now()

		// Send resolution notification
		if err := am.notifier.ResolveAlert(alert.ID); err != nil {
			log.Printf("Failed to send alert resolution: %v", err)
		}
	}
}

// Start starts the alert manager evaluation loop
func (am *AlertManager) Start(ctx context.Context, interval time.Duration) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			am.EvaluateRules(ctx)
		}
	}
}

// EvaluateRule evaluates a single alert rule
func (ae *AlertEvaluator) EvaluateRule(ctx context.Context, rule *AlertRule) (bool, float64, error) {
	switch rule.Query {
	case "health_check":
		return ae.evaluateHealthCheck(ctx, rule)
	case "system_memory":
		return ae.evaluateSystemMemory(ctx, rule)
	case "system_goroutines":
		return ae.evaluateSystemGoroutines(ctx, rule)
	case "database_connections":
		return ae.evaluateDatabaseConnections(ctx, rule)
	default:
		return false, 0, fmt.Errorf("unknown alert query: %s", rule.Query)
	}
}

// evaluateHealthCheck evaluates health check based alerts
func (ae *AlertEvaluator) evaluateHealthCheck(ctx context.Context, rule *AlertRule) (bool, float64, error) {
	if ae.healthMonitor == nil {
		return false, 0, fmt.Errorf("health monitor not available")
	}

	health := ae.healthMonitor.CheckHealth(ctx)

	// Convert health status to numeric value for threshold comparison
	var value float64
	switch health.Status {
	case StatusHealthy:
		value = 0
	case StatusDegraded:
		value = 1
	case StatusUnhealthy:
		value = 2
	}

	// Alert if health status meets or exceeds threshold
	return value >= rule.Threshold, value, nil
}

// evaluateSystemMemory evaluates system memory usage
func (ae *AlertEvaluator) evaluateSystemMemory(ctx context.Context, rule *AlertRule) (bool, float64, error) {
	systemChecker := NewSystemHealthChecker()
	check := systemChecker.CheckHealth(ctx)

	// Extract memory usage percentage from metadata
	memoryUsagePercent := 0.0
	if metadata, ok := check.Metadata["memory_usage_percent"].(float64); ok {
		memoryUsagePercent = metadata
	}

	return memoryUsagePercent >= rule.Threshold, memoryUsagePercent, nil
}

// evaluateSystemGoroutines evaluates goroutine count
func (ae *AlertEvaluator) evaluateSystemGoroutines(ctx context.Context, rule *AlertRule) (bool, float64, error) {
	systemChecker := NewSystemHealthChecker()
	check := systemChecker.CheckHealth(ctx)

	// Extract goroutine count from metadata
	goroutines := 0.0
	if metadata, ok := check.Metadata["goroutines"].(int); ok {
		goroutines = float64(metadata)
	}

	return goroutines >= rule.Threshold, goroutines, nil
}

// evaluateDatabaseConnections evaluates database connection pool
func (ae *AlertEvaluator) evaluateDatabaseConnections(ctx context.Context, rule *AlertRule) (bool, float64, error) {
	if ae.healthMonitor == nil {
		return false, 0, fmt.Errorf("health monitor not available")
	}

	// Get database health check result which contains connection pool metadata
	health := ae.healthMonitor.CheckHealth(ctx)
	dbCheck, exists := health.Checks["database"]
	if !exists {
		return false, 0, fmt.Errorf("database health check not registered")
	}

	// Calculate connection pool utilization percentage
	openConns, ok1 := dbCheck.Metadata["open_connections"].(int)
	inUse, ok2 := dbCheck.Metadata["in_use"].(int)
	if !ok1 || !ok2 || openConns == 0 {
		return false, 0, nil
	}

	utilization := float64(inUse) / float64(openConns) * 100
	return utilization >= rule.Threshold, utilization, nil
}

// Global alert manager instance
var globalAlertManager *AlertManager

// InitializeAlertManager initializes the global alert manager
func InitializeAlertManager(healthMonitor *HealthMonitor) {
	globalAlertManager = NewAlertManager(healthMonitor)

	// Add default alert rules
	globalAlertManager.AddRule(&AlertRule{
		ID:          "system_health",
		Name:        "System Health Check",
		Description: "System health is degraded or unhealthy",
		Severity:    SeverityWarning,
		Query:       "health_check",
		Threshold:   1, // Alert if status is degraded (1) or unhealthy (2)
		Duration:    time.Minute * 5,
		Labels: map[string]string{
			"component": "system",
			"check":     "health",
		},
		Enabled: true,
	})

	globalAlertManager.AddRule(&AlertRule{
		ID:          "high_memory_usage",
		Name:        "High Memory Usage",
		Description: "System memory usage is high",
		Severity:    SeverityWarning,
		Query:       "system_memory",
		Threshold:   80, // Alert if memory usage > 80%
		Duration:    time.Minute * 5,
		Labels: map[string]string{
			"component": "system",
			"metric":    "memory",
		},
		Enabled: true,
	})

	globalAlertManager.AddRule(&AlertRule{
		ID:          "high_goroutine_count",
		Name:        "High Goroutine Count",
		Description: "System has too many goroutines",
		Severity:    SeverityWarning,
		Query:       "system_goroutines",
		Threshold:   1000, // Alert if goroutines > 1000
		Duration:    time.Minute * 5,
		Labels: map[string]string{
			"component": "system",
			"metric":    "goroutines",
		},
		Enabled: true,
	})
}

// GetAlertManager returns the global alert manager
func GetAlertManager() *AlertManager {
	return globalAlertManager
}

// StartAlertManager starts the global alert manager
func StartAlertManager(ctx context.Context) {
	if globalAlertManager != nil {
		go globalAlertManager.Start(ctx, time.Minute*1) // Evaluate every minute
	}
}