diff --git a/deployments/elk-stack/elasticsearch.yaml b/deployments/elk-stack/elasticsearch.yaml new file mode 100644 index 0000000..810d366 --- /dev/null +++ b/deployments/elk-stack/elasticsearch.yaml @@ -0,0 +1,141 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: elasticsearch + labels: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch-master + namespace: elasticsearch + labels: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master + app.kubernetes.io/part-of: telecom-platform +spec: + serviceName: elasticsearch-master + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master + template: + metadata: + labels: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master + app.kubernetes.io/part-of: telecom-platform + spec: + securityContext: + fsGroup: 1000 + runAsUser: 1000 + initContainers: + - name: increase-vm-max-map + image: busybox:1.35.0 + command: ["sysctl", "-w", "vm.max_map_count=262144"] + securityContext: + privileged: true + - name: increase-fd-limit + image: busybox:1.35.0 + command: ["sh", "-c", "ulimit -n 65536"] + securityContext: + privileged: true + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0 + ports: + - containerPort: 9200 + name: rest + - containerPort: 9300 + name: inter-node + env: + - name: cluster.name + value: telecom-platform-logs + - name: node.name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: discovery.seed_hosts + value: "elasticsearch-master" + - name: cluster.initial_master_nodes + value: "elasticsearch-master-0,elasticsearch-master-1,elasticsearch-master-2" + - name: ES_JAVA_OPTS + value: "-Xms1g -Xmx1g" + - name: xpack.security.enabled + value: "false" + - name: xpack.monitoring.collection.enabled + value: "true" + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + - name: config + mountPath: /usr/share/elasticsearch/config + volumes: + - name: config + configMap: + name: elasticsearch-config + volumeClaimTemplates: + - metadata: + name: data + labels: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master + app.kubernetes.io/part-of: telecom-platform + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "standard" + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch-master + namespace: elasticsearch + labels: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master + app.kubernetes.io/part-of: telecom-platform +spec: + ports: + - port: 9200 + name: rest + - port: 9300 + name: inter-node + selector: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master + clusterIP: None +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elasticsearch-config + namespace: elasticsearch + labels: + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master + app.kubernetes.io/part-of: telecom-platform +data: + elasticsearch.yml: | + cluster.name: telecom-platform-logs + node.name: ${HOSTNAME} + network.host: 0.0.0.0 + discovery.seed_hosts: ["elasticsearch-master"] + cluster.initial_master_nodes: ["elasticsearch-master-0", "elasticsearch-master-1", "elasticsearch-master-2"] + xpack.security.enabled: false + xpack.monitoring.collection.enabled: true + xpack.monitoring.elasticsearch.collection.enabled: true + indices.lifecycle.poll_interval: 1m diff --git a/deployments/elk-stack/kibana.yaml b/deployments/elk-stack/kibana.yaml new file mode 100644 index 0000000..3fc902a --- /dev/null +++ b/deployments/elk-stack/kibana.yaml @@ -0,0 +1,106 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + namespace: elasticsearch + labels: + app.kubernetes.io/name: kibana + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kibana + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform + template: + metadata: + labels: + app.kubernetes.io/name: kibana + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:8.10.0 + ports: + - containerPort: 5601 + name: ui + env: + - name: ELASTICSEARCH_HOSTS + value: "http://elasticsearch-master:9200" + - name: SERVER_NAME + value: kibana + - name: SERVER_HOST + value: "0.0.0.0" + - name: SERVER_PORT + value: "5601" + - name: SERVER_PUBLICBASEURL + value: "http://kibana.telecom.local:5601" + - name: LOGGING_ROOT_LEVEL + value: "info" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + livenessProbe: + httpGet: + path: /api/status + port: ui + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /api/status + port: ui + initialDelaySeconds: 5 + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: kibana + namespace: elasticsearch + labels: + app.kubernetes.io/name: kibana + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform +spec: + ports: + - port: 5601 + name: ui + targetPort: 5601 + selector: + app.kubernetes.io/name: kibana + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform + type: ClusterIP +--- +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: kibana + namespace: elasticsearch + labels: + app.kubernetes.io/name: kibana + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform +spec: + hosts: + - kibana.telecom.local + gateways: + - telecom-platform-gateway + http: + - match: + - uri: + prefix: / + route: + - destination: + host: kibana + port: + number: 5601 + timeout: 30s diff --git a/deployments/elk-stack/logstash.yaml b/deployments/elk-stack/logstash.yaml new file mode 100644 index 0000000..bd910f4 --- /dev/null +++ b/deployments/elk-stack/logstash.yaml @@ -0,0 +1,204 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: logstash + namespace: elasticsearch + labels: + app.kubernetes.io/name: logstash + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: logstash + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform + template: + metadata: + labels: + app.kubernetes.io/name: logstash + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform + spec: + containers: + - name: logstash + image: docker.elastic.co/logstash/logstash:8.10.0 + ports: + - containerPort: 5044 + name: beats + - containerPort: 9600 + name: http + env: + - name: LS_JAVA_OPTS + value: "-Xmx512m -Xms512m" + - name: ELASTICSEARCH_HOSTS + value: "http://elasticsearch-master:9200" + - name: LOG_LEVEL + value: "info" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + volumeMounts: + - name: config + mountPath: /usr/share/logstash/pipeline + - name: settings + mountPath: /usr/share/logstash/config + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 60 + periodSeconds: 10 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 5 + volumes: + - name: config + configMap: + name: logstash-config + - name: settings + configMap: + name: logstash-settings +--- +apiVersion: v1 +kind: Service +metadata: + name: logstash + namespace: elasticsearch + labels: + app.kubernetes.io/name: logstash + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform +spec: + ports: + - port: 5044 + name: beats + targetPort: 5044 + - port: 9600 + name: http + targetPort: 9600 + selector: + app.kubernetes.io/name: logstash + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform + type: ClusterIP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: logstash-config + namespace: elasticsearch + labels: + app.kubernetes.io/name: logstash + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform +data: + logstash.conf: | + input { + beats { + port => 5044 + } + http { + port => 8080 + codec => json + } + } + + filter { + # Parse JSON logs + if [message] { + json { + source => "message" + } + } + + # Add environment tag + mutate { + add_field => { "environment" => "production" } + } + + # Parse timestamp + if [timestamp] { + date { + match => [ "timestamp", "ISO8601" ] + } + } + + # Add telecom platform specific fields + if [service] { + mutate { + add_field => { "[@metadata][service]" => "%{service}" } + } + } + + # GDPR compliance - anonymize sensitive data + if [phone] { + mutate { + replace => { "phone" => "XXX-XXX-XXXX" } + } + } + + if [email] { + mutate { + replace => { "email" => "XXXX@XXXX.com" } + } + } + } + + output { + elasticsearch { + hosts => ["elasticsearch-master:9200"] + index => "telecom-platform-logs-%{+YYYY.MM.dd}" + template_name => "telecom-platform" + template_pattern => "telecom-platform-*" + template => { + "index_patterns" => ["telecom-platform-*"], + "settings" => { + "number_of_shards" => 3, + "number_of_replicas" => 1, + "index.lifecycle.name" => "telecom-platform-policy", + "index.lifecycle.rollover_alias" => "telecom-platform-logs" + }, + "mappings" => { + "properties" => { + "@timestamp" => { "type" => "date" }, + "level" => { "type" => "keyword" }, + "service" => { "type" => "keyword" }, + "environment" => { "type" => "keyword" }, + "message" => { "type" => "text" }, + "user_id" => { "type" => "keyword" }, + "request_id" => { "type" => "keyword" }, + "duration_ms" => { "type" => "long" }, + "status_code" => { "type" => "integer" } + } + } + } + } + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: logstash-settings + namespace: elasticsearch + labels: + app.kubernetes.io/name: logstash + app.kubernetes.io/component: logging + app.kubernetes.io/part-of: telecom-platform +data: + logstash.yml: | + http.host: "0.0.0.0" + xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch-master:9200" ] + xpack.monitoring.enabled: true + path.config: /usr/share/logstash/pipeline + pipeline.workers: 2 + pipeline.batch.size: 125 + pipeline.batch.delay: 50