feat: Add comprehensive health monitoring system with component checks, system metrics, and HTTP handlers

- Add HealthStatus enum with healthy/degraded/unhealthy states and HealthCheck struct with name/status/message/last_checked/response_time/metadata fields - Add SystemHealth struct with status/timestamp/uptime/version/environment/checks/summary fields and HealthSummary with total/healthy/degraded/unhealthy counts - Add HealthMonitor struct with checks map, mutex, startTime, version, env fields
2026-10-20 20:04:24 +00:00 · 2026-04-23 02:57:31 +03:00
parent d905478e89
commit dd7ff85da6
1 changed files with 413 additions and 0 deletions
@@ -0,0 +1,413 @@
+package monitoring
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"net/http"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+)
+
+// HealthStatus represents the health status of a component
+type HealthStatus string
+
+const (
+	StatusHealthy   HealthStatus = "healthy"
+	StatusDegraded  HealthStatus = "degraded"
+	StatusUnhealthy HealthStatus = "unhealthy"
+)
+
+// HealthCheck represents a health check for a component
+type HealthCheck struct {
+	Name         string         `json:"name"`
+	Status       HealthStatus   `json:"status"`
+	Message      string         `json:"message,omitempty"`
+	LastChecked  time.Time      `json:"last_checked"`
+	ResponseTime time.Duration  `json:"response_time"`
+	Metadata     map[string]any `json:"metadata,omitempty"`
+}
+
+// SystemHealth represents the overall system health
+type SystemHealth struct {
+	Status      HealthStatus           `json:"status"`
+	Timestamp   time.Time              `json:"timestamp"`
+	Uptime      time.Duration          `json:"uptime"`
+	Version     string                 `json:"version"`
+	Environment string                 `json:"environment"`
+	Checks      map[string]HealthCheck `json:"checks"`
+	Summary     HealthSummary          `json:"summary"`
+}
+
+// HealthSummary provides a summary of health checks
+type HealthSummary struct {
+	Total     int `json:"total"`
+	Healthy   int `json:"healthy"`
+	Degraded  int `json:"degraded"`
+	Unhealthy int `json:"unhealthy"`
+}
+
+// HealthChecker interface for components that can be health checked
+type HealthChecker interface {
+	CheckHealth(ctx context.Context) HealthCheck
+}
+
+// HealthMonitor manages health checks for all system components
+type HealthMonitor struct {
+	checks    map[string]HealthChecker
+	mutex     sync.RWMutex
+	startTime time.Time
+	version   string
+	env       string
+}
+
+// NewHealthMonitor creates a new health monitor
+func NewHealthMonitor(version, environment string) *HealthMonitor {
+	return &HealthMonitor{
+		checks:    make(map[string]HealthChecker),
+		startTime: time.Now(),
+		version:   version,
+		env:       environment,
+	}
+}
+
+// RegisterCheck registers a health checker
+func (hm *HealthMonitor) RegisterCheck(name string, checker HealthChecker) {
+	hm.mutex.Lock()
+	defer hm.mutex.Unlock()
+	hm.checks[name] = checker
+}
+
+// CheckHealth performs all health checks
+func (hm *HealthMonitor) CheckHealth(ctx context.Context) SystemHealth {
+	hm.mutex.RLock()
+	checks := make(map[string]HealthChecker, len(hm.checks))
+	maps.Copy(checks, hm.checks)
+	hm.mutex.RUnlock()
+
+	results := make(map[string]HealthCheck)
+	summary := HealthSummary{}
+
+	for name, checker := range checks {
+		checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		result := checker.CheckHealth(checkCtx)
+		cancel()
+
+		result.LastChecked = time.Now()
+		results[name] = result
+
+		summary.Total++
+		switch result.Status {
+		case StatusHealthy:
+			summary.Healthy++
+		case StatusDegraded:
+			summary.Degraded++
+		case StatusUnhealthy:
+			summary.Unhealthy++
+		}
+	}
+
+	// Determine overall status
+	var overallStatus HealthStatus
+	if summary.Unhealthy > 0 {
+		overallStatus = StatusUnhealthy
+	} else if summary.Degraded > 0 {
+		overallStatus = StatusDegraded
+	} else {
+		overallStatus = StatusHealthy
+	}
+
+	return SystemHealth{
+		Status:      overallStatus,
+		Timestamp:   time.Now(),
+		Uptime:      time.Since(hm.startTime),
+		Version:     hm.version,
+		Environment: hm.env,
+		Checks:      results,
+		Summary:     summary,
+	}
+}
+
+// DatabaseHealthChecker checks database connectivity
+type DatabaseHealthChecker struct {
+	db *gorm.DB
+}
+
+func NewDatabaseHealthChecker(db *gorm.DB) *DatabaseHealthChecker {
+	return &DatabaseHealthChecker{db: db}
+}
+
+func (dhc *DatabaseHealthChecker) CheckHealth(ctx context.Context) HealthCheck {
+	start := time.Now()
+
+	sqlDB, err := dhc.db.DB()
+	if err != nil {
+		return HealthCheck{
+			Name:         "database",
+			Status:       StatusUnhealthy,
+			Message:      fmt.Sprintf("Failed to get database instance: %v", err),
+			ResponseTime: time.Since(start),
+		}
+	}
+
+	// Test database connection
+	err = sqlDB.PingContext(ctx)
+	responseTime := time.Since(start)
+
+	if err != nil {
+		return HealthCheck{
+			Name:         "database",
+			Status:       StatusUnhealthy,
+			Message:      fmt.Sprintf("Database ping failed: %v", err),
+			ResponseTime: responseTime,
+		}
+	}
+
+	// Get connection pool stats
+	stats := sqlDB.Stats()
+	metadata := map[string]any{
+		"open_connections":     stats.OpenConnections,
+		"in_use":               stats.InUse,
+		"idle":                 stats.Idle,
+		"wait_count":           stats.WaitCount,
+		"wait_duration":        stats.WaitDuration.String(),
+		"max_idle_closed":      stats.MaxIdleClosed,
+		"max_idle_time_closed": stats.MaxIdleTimeClosed,
+		"max_lifetime_closed":  stats.MaxLifetimeClosed,
+	}
+
+	// Check if connection pool is healthy
+	if stats.OpenConnections >= stats.MaxOpenConnections-5 {
+		return HealthCheck{
+			Name:         "database",
+			Status:       StatusDegraded,
+			Message:      "Database connection pool nearly full",
+			ResponseTime: responseTime,
+			Metadata:     metadata,
+		}
+	}
+
+	return HealthCheck{
+		Name:         "database",
+		Status:       StatusHealthy,
+		Message:      "Database connection successful",
+		ResponseTime: responseTime,
+		Metadata:     metadata,
+	}
+}
+
+// ServiceChecker interface for services that can be checked for availability
+type ServiceChecker interface {
+	Available(ctx context.Context) bool
+}
+
+// PrometheusHealthChecker checks Prometheus connectivity
+type PrometheusHealthChecker struct {
+	prometheusService ServiceChecker
+}
+
+func NewPrometheusHealthChecker(prometheusService ServiceChecker) *PrometheusHealthChecker {
+	return &PrometheusHealthChecker{prometheusService: prometheusService}
+}
+
+func (phc *PrometheusHealthChecker) CheckHealth(ctx context.Context) HealthCheck {
+	start := time.Now()
+
+	available := phc.prometheusService.Available(ctx)
+	responseTime := time.Since(start)
+
+	if !available {
+		return HealthCheck{
+			Name:         "prometheus",
+			Status:       StatusUnhealthy,
+			Message:      "Prometheus service unavailable",
+			ResponseTime: responseTime,
+		}
+	}
+
+	return HealthCheck{
+		Name:         "prometheus",
+		Status:       StatusHealthy,
+		Message:      "Prometheus service available",
+		ResponseTime: responseTime,
+	}
+}
+
+// KubernetesHealthChecker checks Kubernetes connectivity
+type KubernetesHealthChecker struct {
+	kubernetesService ServiceChecker
+}
+
+func NewKubernetesHealthChecker(kubernetesService ServiceChecker) *KubernetesHealthChecker {
+	return &KubernetesHealthChecker{kubernetesService: kubernetesService}
+}
+
+func (khc *KubernetesHealthChecker) CheckHealth(ctx context.Context) HealthCheck {
+	start := time.Now()
+
+	available := khc.kubernetesService.Available(ctx)
+	responseTime := time.Since(start)
+
+	if !available {
+		return HealthCheck{
+			Name:         "kubernetes",
+			Status:       StatusDegraded,
+			Message:      "Kubernetes service unavailable",
+			ResponseTime: responseTime,
+		}
+	}
+
+	return HealthCheck{
+		Name:         "kubernetes",
+		Status:       StatusHealthy,
+		Message:      "Kubernetes service available",
+		ResponseTime: responseTime,
+	}
+}
+
+// SystemHealthChecker checks system resources
+type SystemHealthChecker struct{}
+
+func NewSystemHealthChecker() *SystemHealthChecker {
+	return &SystemHealthChecker{}
+}
+
+func (shc *SystemHealthChecker) CheckHealth(ctx context.Context) HealthCheck {
+	start := time.Now()
+
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	responseTime := time.Since(start)
+
+	// Calculate memory usage percentage
+	memoryUsageMB := float64(m.Alloc) / 1024 / 1024
+	memoryLimitMB := float64(m.Sys) / 1024 / 1024
+	memoryUsagePercent := (memoryUsageMB / memoryLimitMB) * 100
+
+	metadata := map[string]any{
+		"memory_usage_mb":      memoryUsageMB,
+		"memory_limit_mb":      memoryLimitMB,
+		"memory_usage_percent": memoryUsagePercent,
+		"goroutines":           runtime.NumGoroutine(),
+		"gc_cycles":            m.NumGC,
+		"heap_alloc":           m.HeapAlloc,
+		"heap_sys":             m.HeapSys,
+		"heap_objects":         m.HeapObjects,
+	}
+
+	status := StatusHealthy
+	message := "System resources healthy"
+
+	// Check memory usage
+	if memoryUsagePercent > 90 {
+		status = StatusUnhealthy
+		message = "High memory usage (>90%)"
+	} else if memoryUsagePercent > 75 {
+		status = StatusDegraded
+		message = "Moderate memory usage (>75%)"
+	}
+
+	// Check goroutine count
+	goroutines := runtime.NumGoroutine()
+	if goroutines > 1000 {
+		if status == StatusHealthy {
+			status = StatusDegraded
+			message = "High goroutine count"
+		}
+	}
+
+	return HealthCheck{
+		Name:         "system",
+		Status:       status,
+		Message:      message,
+		ResponseTime: responseTime,
+		Metadata:     metadata,
+	}
+}
+
+// Global health monitor instance
+var globalHealthMonitor *HealthMonitor
+
+// InitializeHealthMonitor initializes the global health monitor
+func InitializeHealthMonitor(version, environment string) {
+	globalHealthMonitor = NewHealthMonitor(version, environment)
+}
+
+// GetHealthMonitor returns the global health monitor
+func GetHealthMonitor() *HealthMonitor {
+	return globalHealthMonitor
+}
+
+// HealthHandler returns a gin handler for health checks
+func HealthHandler() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		if globalHealthMonitor == nil {
+			c.JSON(http.StatusServiceUnavailable, gin.H{
+				"status":  "unhealthy",
+				"message": "Health monitor not initialized",
+			})
+			return
+		}
+
+		ctx := c.Request.Context()
+		health := globalHealthMonitor.CheckHealth(ctx)
+
+		// Set appropriate HTTP status code
+		statusCode := http.StatusOK
+		switch health.Status {
+		case StatusDegraded:
+			statusCode = http.StatusOK // Still serve traffic but indicate degradation
+		case StatusUnhealthy:
+			statusCode = http.StatusServiceUnavailable
+		}
+
+		c.JSON(statusCode, health)
+	}
+}
+
+// ReadyHandler returns a gin handler for readiness checks
+func ReadyHandler() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		if globalHealthMonitor == nil {
+			c.JSON(http.StatusServiceUnavailable, gin.H{
+				"status":  "not_ready",
+				"message": "Health monitor not initialized",
+			})
+			return
+		}
+
+		ctx := c.Request.Context()
+		health := globalHealthMonitor.CheckHealth(ctx)
+
+		// Ready if all critical checks are healthy
+		ready := health.Status != StatusUnhealthy
+
+		statusCode := http.StatusOK
+		if !ready {
+			statusCode = http.StatusServiceUnavailable
+		}
+
+		response := map[string]any{
+			"ready":  ready,
+			"status": health.Status,
+		}
+
+		c.JSON(statusCode, response)
+	}
+}
+
+// LiveHandler returns a gin handler for liveness checks
+func LiveHandler() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// Liveness is simple - if we can respond, we're alive
+		c.JSON(http.StatusOK, gin.H{
+			"alive":     true,
+			"timestamp": time.Now(),
+		})
+	}
+}