package monitoring import ( "context" "fmt" "maps" "net/http" "runtime" "sync" "time" "github.com/gin-gonic/gin" "gorm.io/gorm" ) // HealthStatus represents the health status of a component type HealthStatus string const ( StatusHealthy HealthStatus = "healthy" StatusDegraded HealthStatus = "degraded" StatusUnhealthy HealthStatus = "unhealthy" ) // HealthCheck represents a health check for a component type HealthCheck struct { Name string `json:"name"` Status HealthStatus `json:"status"` Message string `json:"message,omitempty"` LastChecked time.Time `json:"last_checked"` ResponseTime time.Duration `json:"response_time"` Metadata map[string]any `json:"metadata,omitempty"` } // SystemHealth represents the overall system health type SystemHealth struct { Status HealthStatus `json:"status"` Timestamp time.Time `json:"timestamp"` Uptime time.Duration `json:"uptime"` Version string `json:"version"` Environment string `json:"environment"` Checks map[string]HealthCheck `json:"checks"` Summary HealthSummary `json:"summary"` } // HealthSummary provides a summary of health checks type HealthSummary struct { Total int `json:"total"` Healthy int `json:"healthy"` Degraded int `json:"degraded"` Unhealthy int `json:"unhealthy"` } // HealthChecker interface for components that can be health checked type HealthChecker interface { CheckHealth(ctx context.Context) HealthCheck } // HealthMonitor manages health checks for all system components type HealthMonitor struct { checks map[string]HealthChecker mutex sync.RWMutex startTime time.Time version string env string } // NewHealthMonitor creates a new health monitor func NewHealthMonitor(version, environment string) *HealthMonitor { return &HealthMonitor{ checks: make(map[string]HealthChecker), startTime: time.Now(), version: version, env: environment, } } // RegisterCheck registers a health checker func (hm *HealthMonitor) RegisterCheck(name string, checker HealthChecker) { hm.mutex.Lock() defer hm.mutex.Unlock() hm.checks[name] = checker } // CheckHealth performs all health checks func (hm *HealthMonitor) CheckHealth(ctx context.Context) SystemHealth { hm.mutex.RLock() checks := make(map[string]HealthChecker, len(hm.checks)) maps.Copy(checks, hm.checks) hm.mutex.RUnlock() results := make(map[string]HealthCheck) summary := HealthSummary{} for name, checker := range checks { checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second) // Execute health check with timeout resultChan := make(chan HealthCheck, 1) go func() { resultChan <- checker.CheckHealth(checkCtx) }() var result HealthCheck select { case result = <-resultChan: // Check completed successfully case <-checkCtx.Done(): // Timeout occurred result = HealthCheck{ Name: name, Status: StatusUnhealthy, Message: "Health check timed out", ResponseTime: 10 * time.Second, } } cancel() result.LastChecked = time.Now() results[name] = result summary.Total++ switch result.Status { case StatusHealthy: summary.Healthy++ case StatusDegraded: summary.Degraded++ case StatusUnhealthy: summary.Unhealthy++ } } // Determine overall status var overallStatus HealthStatus if summary.Unhealthy > 0 { overallStatus = StatusUnhealthy } else if summary.Degraded > 0 { overallStatus = StatusDegraded } else { overallStatus = StatusHealthy } return SystemHealth{ Status: overallStatus, Timestamp: time.Now(), Uptime: time.Since(hm.startTime), Version: hm.version, Environment: hm.env, Checks: results, Summary: summary, } } // DatabaseHealthChecker checks database connectivity type DatabaseHealthChecker struct { db *gorm.DB } func NewDatabaseHealthChecker(db *gorm.DB) *DatabaseHealthChecker { return &DatabaseHealthChecker{db: db} } func (dhc *DatabaseHealthChecker) CheckHealth(ctx context.Context) HealthCheck { start := time.Now() sqlDB, err := dhc.db.DB() if err != nil { return HealthCheck{ Name: "database", Status: StatusUnhealthy, Message: fmt.Sprintf("Failed to get database instance: %v", err), ResponseTime: time.Since(start), } } // Test database connection err = sqlDB.PingContext(ctx) responseTime := time.Since(start) if err != nil { return HealthCheck{ Name: "database", Status: StatusUnhealthy, Message: fmt.Sprintf("Database ping failed: %v", err), ResponseTime: responseTime, } } // Get connection pool stats stats := sqlDB.Stats() metadata := map[string]any{ "open_connections": stats.OpenConnections, "in_use": stats.InUse, "idle": stats.Idle, "wait_count": stats.WaitCount, "wait_duration": stats.WaitDuration.String(), "max_idle_closed": stats.MaxIdleClosed, "max_idle_time_closed": stats.MaxIdleTimeClosed, "max_lifetime_closed": stats.MaxLifetimeClosed, } // Check if connection pool is healthy if stats.OpenConnections >= stats.MaxOpenConnections-5 { return HealthCheck{ Name: "database", Status: StatusDegraded, Message: "Database connection pool nearly full", ResponseTime: responseTime, Metadata: metadata, } } return HealthCheck{ Name: "database", Status: StatusHealthy, Message: "Database connection successful", ResponseTime: responseTime, Metadata: metadata, } } // ServiceChecker interface for services that can be checked for availability type ServiceChecker interface { Available(ctx context.Context) bool } // PrometheusHealthChecker checks Prometheus connectivity type PrometheusHealthChecker struct { prometheusService ServiceChecker } func NewPrometheusHealthChecker(prometheusService ServiceChecker) *PrometheusHealthChecker { return &PrometheusHealthChecker{prometheusService: prometheusService} } func (phc *PrometheusHealthChecker) CheckHealth(ctx context.Context) HealthCheck { start := time.Now() available := phc.prometheusService.Available(ctx) responseTime := time.Since(start) if !available { return HealthCheck{ Name: "prometheus", Status: StatusUnhealthy, Message: "Prometheus service unavailable", ResponseTime: responseTime, } } return HealthCheck{ Name: "prometheus", Status: StatusHealthy, Message: "Prometheus service available", ResponseTime: responseTime, } } // KubernetesHealthChecker checks Kubernetes connectivity type KubernetesHealthChecker struct { kubernetesService ServiceChecker } func NewKubernetesHealthChecker(kubernetesService ServiceChecker) *KubernetesHealthChecker { return &KubernetesHealthChecker{kubernetesService: kubernetesService} } func (khc *KubernetesHealthChecker) CheckHealth(ctx context.Context) HealthCheck { start := time.Now() available := khc.kubernetesService.Available(ctx) responseTime := time.Since(start) if !available { return HealthCheck{ Name: "kubernetes", Status: StatusDegraded, Message: "Kubernetes service unavailable", ResponseTime: responseTime, } } return HealthCheck{ Name: "kubernetes", Status: StatusHealthy, Message: "Kubernetes service available", ResponseTime: responseTime, } } // SystemHealthChecker checks system resources type SystemHealthChecker struct{} func NewSystemHealthChecker() *SystemHealthChecker { return &SystemHealthChecker{} } func (shc *SystemHealthChecker) CheckHealth(ctx context.Context) HealthCheck { start := time.Now() var m runtime.MemStats runtime.ReadMemStats(&m) responseTime := time.Since(start) // Calculate memory usage percentage memoryUsageMB := float64(m.Alloc) / 1024 / 1024 memoryLimitMB := float64(m.Sys) / 1024 / 1024 memoryUsagePercent := (memoryUsageMB / memoryLimitMB) * 100 metadata := map[string]any{ "memory_usage_mb": memoryUsageMB, "memory_limit_mb": memoryLimitMB, "memory_usage_percent": memoryUsagePercent, "goroutines": runtime.NumGoroutine(), "gc_cycles": m.NumGC, "heap_alloc": m.HeapAlloc, "heap_sys": m.HeapSys, "heap_objects": m.HeapObjects, } status := StatusHealthy message := "System resources healthy" // Check memory usage if memoryUsagePercent > 90 { status = StatusUnhealthy message = "High memory usage (>90%)" } else if memoryUsagePercent > 75 { status = StatusDegraded message = "Moderate memory usage (>75%)" } // Check goroutine count goroutines := runtime.NumGoroutine() if goroutines > 1000 { if status == StatusHealthy { status = StatusDegraded message = "High goroutine count" } } return HealthCheck{ Name: "system", Status: status, Message: message, ResponseTime: responseTime, Metadata: metadata, } } // Global health monitor instance var globalHealthMonitor *HealthMonitor // InitializeHealthMonitor initializes the global health monitor func InitializeHealthMonitor(version, environment string) { globalHealthMonitor = NewHealthMonitor(version, environment) } // GetHealthMonitor returns the global health monitor func GetHealthMonitor() *HealthMonitor { return globalHealthMonitor } // HealthHandler returns a gin handler for health checks func HealthHandler() gin.HandlerFunc { return func(c *gin.Context) { if globalHealthMonitor == nil { c.JSON(http.StatusServiceUnavailable, gin.H{ "status": "unhealthy", "message": "Health monitor not initialized", }) return } ctx := c.Request.Context() health := globalHealthMonitor.CheckHealth(ctx) // Set appropriate HTTP status code statusCode := http.StatusOK switch health.Status { case StatusDegraded: statusCode = http.StatusOK // Still serve traffic but indicate degradation case StatusUnhealthy: statusCode = http.StatusServiceUnavailable } c.JSON(statusCode, health) } } // ReadyHandler returns a gin handler for readiness checks func ReadyHandler() gin.HandlerFunc { return func(c *gin.Context) { if globalHealthMonitor == nil { c.JSON(http.StatusServiceUnavailable, gin.H{ "status": "not_ready", "message": "Health monitor not initialized", }) return } ctx := c.Request.Context() health := globalHealthMonitor.CheckHealth(ctx) // Ready if all critical checks are healthy ready := health.Status != StatusUnhealthy statusCode := http.StatusOK if !ready { statusCode = http.StatusServiceUnavailable } response := map[string]any{ "ready": ready, "status": health.Status, } c.JSON(statusCode, response) } } // LiveHandler returns a gin handler for liveness checks func LiveHandler() gin.HandlerFunc { return func(c *gin.Context) { // Liveness is simple - if we can respond, we're alive c.JSON(http.StatusOK, gin.H{ "alive": true, "timestamp": time.Now(), }) } }