From dd7ff85da600526b99103dbe47f1578d150e812f Mon Sep 17 00:00:00 2001 From: nutcas3 Date: Thu, 23 Apr 2026 02:57:31 +0300 Subject: [PATCH] feat: Add comprehensive health monitoring system with component checks, system metrics, and HTTP handlers - Add HealthStatus enum with healthy/degraded/unhealthy states and HealthCheck struct with name/status/message/last_checked/response_time/metadata fields - Add SystemHealth struct with status/timestamp/uptime/version/environment/checks/summary fields and HealthSummary with total/healthy/degraded/unhealthy counts - Add HealthMonitor struct with checks map, mutex, startTime, version, env fields --- apps/api-server/internal/monitoring/health.go | 413 ++++++++++++++++++ 1 file changed, 413 insertions(+) create mode 100644 apps/api-server/internal/monitoring/health.go diff --git a/apps/api-server/internal/monitoring/health.go b/apps/api-server/internal/monitoring/health.go new file mode 100644 index 0000000..ea344c0 --- /dev/null +++ b/apps/api-server/internal/monitoring/health.go @@ -0,0 +1,413 @@ +package monitoring + +import ( + "context" + "fmt" + "maps" + "net/http" + "runtime" + "sync" + "time" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" +) + +// HealthStatus represents the health status of a component +type HealthStatus string + +const ( + StatusHealthy HealthStatus = "healthy" + StatusDegraded HealthStatus = "degraded" + StatusUnhealthy HealthStatus = "unhealthy" +) + +// HealthCheck represents a health check for a component +type HealthCheck struct { + Name string `json:"name"` + Status HealthStatus `json:"status"` + Message string `json:"message,omitempty"` + LastChecked time.Time `json:"last_checked"` + ResponseTime time.Duration `json:"response_time"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +// SystemHealth represents the overall system health +type SystemHealth struct { + Status HealthStatus `json:"status"` + Timestamp time.Time `json:"timestamp"` + Uptime time.Duration `json:"uptime"` + Version string `json:"version"` + Environment string `json:"environment"` + Checks map[string]HealthCheck `json:"checks"` + Summary HealthSummary `json:"summary"` +} + +// HealthSummary provides a summary of health checks +type HealthSummary struct { + Total int `json:"total"` + Healthy int `json:"healthy"` + Degraded int `json:"degraded"` + Unhealthy int `json:"unhealthy"` +} + +// HealthChecker interface for components that can be health checked +type HealthChecker interface { + CheckHealth(ctx context.Context) HealthCheck +} + +// HealthMonitor manages health checks for all system components +type HealthMonitor struct { + checks map[string]HealthChecker + mutex sync.RWMutex + startTime time.Time + version string + env string +} + +// NewHealthMonitor creates a new health monitor +func NewHealthMonitor(version, environment string) *HealthMonitor { + return &HealthMonitor{ + checks: make(map[string]HealthChecker), + startTime: time.Now(), + version: version, + env: environment, + } +} + +// RegisterCheck registers a health checker +func (hm *HealthMonitor) RegisterCheck(name string, checker HealthChecker) { + hm.mutex.Lock() + defer hm.mutex.Unlock() + hm.checks[name] = checker +} + +// CheckHealth performs all health checks +func (hm *HealthMonitor) CheckHealth(ctx context.Context) SystemHealth { + hm.mutex.RLock() + checks := make(map[string]HealthChecker, len(hm.checks)) + maps.Copy(checks, hm.checks) + hm.mutex.RUnlock() + + results := make(map[string]HealthCheck) + summary := HealthSummary{} + + for name, checker := range checks { + checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + result := checker.CheckHealth(checkCtx) + cancel() + + result.LastChecked = time.Now() + results[name] = result + + summary.Total++ + switch result.Status { + case StatusHealthy: + summary.Healthy++ + case StatusDegraded: + summary.Degraded++ + case StatusUnhealthy: + summary.Unhealthy++ + } + } + + // Determine overall status + var overallStatus HealthStatus + if summary.Unhealthy > 0 { + overallStatus = StatusUnhealthy + } else if summary.Degraded > 0 { + overallStatus = StatusDegraded + } else { + overallStatus = StatusHealthy + } + + return SystemHealth{ + Status: overallStatus, + Timestamp: time.Now(), + Uptime: time.Since(hm.startTime), + Version: hm.version, + Environment: hm.env, + Checks: results, + Summary: summary, + } +} + +// DatabaseHealthChecker checks database connectivity +type DatabaseHealthChecker struct { + db *gorm.DB +} + +func NewDatabaseHealthChecker(db *gorm.DB) *DatabaseHealthChecker { + return &DatabaseHealthChecker{db: db} +} + +func (dhc *DatabaseHealthChecker) CheckHealth(ctx context.Context) HealthCheck { + start := time.Now() + + sqlDB, err := dhc.db.DB() + if err != nil { + return HealthCheck{ + Name: "database", + Status: StatusUnhealthy, + Message: fmt.Sprintf("Failed to get database instance: %v", err), + ResponseTime: time.Since(start), + } + } + + // Test database connection + err = sqlDB.PingContext(ctx) + responseTime := time.Since(start) + + if err != nil { + return HealthCheck{ + Name: "database", + Status: StatusUnhealthy, + Message: fmt.Sprintf("Database ping failed: %v", err), + ResponseTime: responseTime, + } + } + + // Get connection pool stats + stats := sqlDB.Stats() + metadata := map[string]any{ + "open_connections": stats.OpenConnections, + "in_use": stats.InUse, + "idle": stats.Idle, + "wait_count": stats.WaitCount, + "wait_duration": stats.WaitDuration.String(), + "max_idle_closed": stats.MaxIdleClosed, + "max_idle_time_closed": stats.MaxIdleTimeClosed, + "max_lifetime_closed": stats.MaxLifetimeClosed, + } + + // Check if connection pool is healthy + if stats.OpenConnections >= stats.MaxOpenConnections-5 { + return HealthCheck{ + Name: "database", + Status: StatusDegraded, + Message: "Database connection pool nearly full", + ResponseTime: responseTime, + Metadata: metadata, + } + } + + return HealthCheck{ + Name: "database", + Status: StatusHealthy, + Message: "Database connection successful", + ResponseTime: responseTime, + Metadata: metadata, + } +} + +// ServiceChecker interface for services that can be checked for availability +type ServiceChecker interface { + Available(ctx context.Context) bool +} + +// PrometheusHealthChecker checks Prometheus connectivity +type PrometheusHealthChecker struct { + prometheusService ServiceChecker +} + +func NewPrometheusHealthChecker(prometheusService ServiceChecker) *PrometheusHealthChecker { + return &PrometheusHealthChecker{prometheusService: prometheusService} +} + +func (phc *PrometheusHealthChecker) CheckHealth(ctx context.Context) HealthCheck { + start := time.Now() + + available := phc.prometheusService.Available(ctx) + responseTime := time.Since(start) + + if !available { + return HealthCheck{ + Name: "prometheus", + Status: StatusUnhealthy, + Message: "Prometheus service unavailable", + ResponseTime: responseTime, + } + } + + return HealthCheck{ + Name: "prometheus", + Status: StatusHealthy, + Message: "Prometheus service available", + ResponseTime: responseTime, + } +} + +// KubernetesHealthChecker checks Kubernetes connectivity +type KubernetesHealthChecker struct { + kubernetesService ServiceChecker +} + +func NewKubernetesHealthChecker(kubernetesService ServiceChecker) *KubernetesHealthChecker { + return &KubernetesHealthChecker{kubernetesService: kubernetesService} +} + +func (khc *KubernetesHealthChecker) CheckHealth(ctx context.Context) HealthCheck { + start := time.Now() + + available := khc.kubernetesService.Available(ctx) + responseTime := time.Since(start) + + if !available { + return HealthCheck{ + Name: "kubernetes", + Status: StatusDegraded, + Message: "Kubernetes service unavailable", + ResponseTime: responseTime, + } + } + + return HealthCheck{ + Name: "kubernetes", + Status: StatusHealthy, + Message: "Kubernetes service available", + ResponseTime: responseTime, + } +} + +// SystemHealthChecker checks system resources +type SystemHealthChecker struct{} + +func NewSystemHealthChecker() *SystemHealthChecker { + return &SystemHealthChecker{} +} + +func (shc *SystemHealthChecker) CheckHealth(ctx context.Context) HealthCheck { + start := time.Now() + + var m runtime.MemStats + runtime.ReadMemStats(&m) + + responseTime := time.Since(start) + + // Calculate memory usage percentage + memoryUsageMB := float64(m.Alloc) / 1024 / 1024 + memoryLimitMB := float64(m.Sys) / 1024 / 1024 + memoryUsagePercent := (memoryUsageMB / memoryLimitMB) * 100 + + metadata := map[string]any{ + "memory_usage_mb": memoryUsageMB, + "memory_limit_mb": memoryLimitMB, + "memory_usage_percent": memoryUsagePercent, + "goroutines": runtime.NumGoroutine(), + "gc_cycles": m.NumGC, + "heap_alloc": m.HeapAlloc, + "heap_sys": m.HeapSys, + "heap_objects": m.HeapObjects, + } + + status := StatusHealthy + message := "System resources healthy" + + // Check memory usage + if memoryUsagePercent > 90 { + status = StatusUnhealthy + message = "High memory usage (>90%)" + } else if memoryUsagePercent > 75 { + status = StatusDegraded + message = "Moderate memory usage (>75%)" + } + + // Check goroutine count + goroutines := runtime.NumGoroutine() + if goroutines > 1000 { + if status == StatusHealthy { + status = StatusDegraded + message = "High goroutine count" + } + } + + return HealthCheck{ + Name: "system", + Status: status, + Message: message, + ResponseTime: responseTime, + Metadata: metadata, + } +} + +// Global health monitor instance +var globalHealthMonitor *HealthMonitor + +// InitializeHealthMonitor initializes the global health monitor +func InitializeHealthMonitor(version, environment string) { + globalHealthMonitor = NewHealthMonitor(version, environment) +} + +// GetHealthMonitor returns the global health monitor +func GetHealthMonitor() *HealthMonitor { + return globalHealthMonitor +} + +// HealthHandler returns a gin handler for health checks +func HealthHandler() gin.HandlerFunc { + return func(c *gin.Context) { + if globalHealthMonitor == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{ + "status": "unhealthy", + "message": "Health monitor not initialized", + }) + return + } + + ctx := c.Request.Context() + health := globalHealthMonitor.CheckHealth(ctx) + + // Set appropriate HTTP status code + statusCode := http.StatusOK + switch health.Status { + case StatusDegraded: + statusCode = http.StatusOK // Still serve traffic but indicate degradation + case StatusUnhealthy: + statusCode = http.StatusServiceUnavailable + } + + c.JSON(statusCode, health) + } +} + +// ReadyHandler returns a gin handler for readiness checks +func ReadyHandler() gin.HandlerFunc { + return func(c *gin.Context) { + if globalHealthMonitor == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{ + "status": "not_ready", + "message": "Health monitor not initialized", + }) + return + } + + ctx := c.Request.Context() + health := globalHealthMonitor.CheckHealth(ctx) + + // Ready if all critical checks are healthy + ready := health.Status != StatusUnhealthy + + statusCode := http.StatusOK + if !ready { + statusCode = http.StatusServiceUnavailable + } + + response := map[string]any{ + "ready": ready, + "status": health.Status, + } + + c.JSON(statusCode, response) + } +} + +// LiveHandler returns a gin handler for liveness checks +func LiveHandler() gin.HandlerFunc { + return func(c *gin.Context) { + // Liveness is simple - if we can respond, we're alive + c.JSON(http.StatusOK, gin.H{ + "alive": true, + "timestamp": time.Now(), + }) + } +}